From 2b89b3c8765130a2ce202265f77f8f9b9d284ee0 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 17:15:16 +0000 Subject: [PATCH 1/9] perf(step2): borrow original blobs instead of copying them Step 2 copied every untokenized original blob into the cregit repo one by one: inflate, hash, deflate, write a loose file. On psi-probe that was half of the consumer thread's time. blobExec --alternates now writes dst/objects/info/alternates naming src's objects before the walk, and copies no original blob. Trees that equal one of src's are not written either. The runner passes it by default. The pack after step 2 then runs git repack -a -d without -l, which copies the borrowed objects in once, sets the alternates file aside, proves dst complete with git fsck --connectivity-only, and only then deletes it. prune and commit-graph follow, as gc does. While dst borrows, the pack is required, not optional. A failed pack puts the file back (dst stays readable through src), writes a PACK-FAILED keep marker and stops the run. Any resume past step 2 packs first, also after a kill during fsck. Default on, because the objects, refs and mapping rows are the same. Opt out with --copy-objects or CREGIT_COPY_OBJECTS=1; ctp.py has no pass-through for runner flags, but the census inherits the environment. --gc none and --mode sharded always copy. --- .github/workflows/ci.yml | 2 +- .../src/main/scala/cregit/blobexec/Main.scala | 37 +++- .../main/scala/cregit/blobexec/Walker.scala | 19 +- .../cregit/blobexec/AlternatesSpec.scala | 198 +++++++++++++++++ run_pipeline_process.sh | 85 +++++++- tests/pack_alternates.sh | 202 ++++++++++++++++++ 6 files changed, 535 insertions(+), 8 deletions(-) create mode 100644 blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala create mode 100755 tests/pack_alternates.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0f4b9ee..b6e89015 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: - name: Pipeline tests (shell) run: | devenv shell -- bash -c 'set -e - for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do + for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough pack_alternates; do bash "tests/$t.sh" done' diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index 8ca84b7e..fc852ad3 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -74,6 +74,9 @@ object Main { | --tokenizer-identity and --memo-dir; a no-op is refused (status ${RetokenizeIneffectiveExitStatus}) | --memo-dir= the tokenizer memo ($$BFG_MEMO_DIR) that --retokenize purges | --tokenizer-worker= tokenize through a pool of persistent workers; needs --pipeline(-trees) + | --alternates write /objects/info/alternates naming 's objects + | and copy no original blob; the caller must then repack + | without -l and delete that file. Not with --shard | --blob-timeout= budget for one run (default ${BlobExec.DefaultTimeoutSeconds}); then its process | tree is killed and the blob excluded | --stall-timeout= exit ${Walker.StalledExitStatus} after this long with no progress (default ${Walker.DefaultStallTimeoutSeconds}); @@ -103,6 +106,7 @@ object Main { retokenize: Set[String] = Set.empty, memoDir: Option[Path] = None, tokenizerWorker: Option[Path] = None, + alternates: Boolean = false, blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, stallExplicit: Boolean = false, @@ -136,6 +140,8 @@ object Main { "into, or the invalidation would leave the real memo's stale entries in place." val PipelinesExclusive = withUsage("Error: --pipeline and --pipeline-trees are mutually exclusive") + val AlternatesUnderShard = withUsage( + "Error: --alternates cannot be used with --shard: shard_merge.py copies each shard's own objects") val ShardWithPipeline = withUsage( "Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees") val MaskWidenedUnderShard = withUsage( @@ -249,6 +255,7 @@ object Main { case "--pipeline" => Right(o.copy(pipeline = true)) case "--pipeline-trees" => Right(o.copy(pipelineTrees = true)) case "--mask-widened" => Right(o.copy(maskWidened = true)) + case "--alternates" => Right(o.copy(alternates = true)) case TokenizerIdentityFlag(spec) => flagValue(TokenizerIdentityFlag, spec)(TokenizerIdentity.parse).map(id => o.copy(tokenizerIdentity = id)) case RetokenizeFlag(spec) => @@ -285,6 +292,7 @@ object Main { (retokenizing && o.shard.isDefined) -> (() => Messages.RetokenizeUnderShard), (!retokenizing && o.memoDir.isDefined) -> (() => Messages.MemoDirWithoutRetokenize), (o.shard.isDefined && o.anyPipeline) -> (() => Messages.ShardWithPipeline), + (o.shard.isDefined && o.alternates) -> (() => Messages.AlternatesUnderShard), o.tokenizerWorker.exists(p => !Files.isRegularFile(p) || !Files.isExecutable(p)) -> (() => Messages.workerNotExecutable(o.tokenizerWorker.get)), (o.tokenizerWorker.isDefined && !o.anyPipeline) -> (() => Messages.WorkerNeedsPipeline) @@ -339,7 +347,7 @@ object Main { println(startLine(o, stallSeconds, incremental)) val src = openSrc(o.src) - val dst = openOrInitDst(o.dst) + val dst = openOrInitDst(o.dst, borrowFrom = Option.when(o.alternates)(src.getObjectsDirectory.toPath)) val mapping = openMapping(o, src, dst) match { case Right(m) => m case Left(refusal) => @@ -436,6 +444,7 @@ object Main { src, dst, mapping, o.mask.r, o.command, o.abortOnError, parallelism, o.pipeline, o.pipelineTrees, o.shard, destinationMayContainObjects = incremental, + borrowOriginalObjects = o.alternates, blobTimeoutSeconds = o.blobTimeoutSeconds, stallTimeoutSeconds = stallSeconds, workerPool = workerPool, @@ -456,6 +465,7 @@ object Main { s"originalBlobDestinationLookups=${stats.originalBlobDestinationLookups} " + s"originalBlobBytesCopied=${stats.originalBlobBytesCopied} " + s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " + + s"originalBlobsBorrowed=${stats.originalBlobsBorrowed} " + s"refsProjected=${stats.refsProjected} " + s"blobsTimedOut=${stats.blobsTimedOut} " + s"blobsOversized=${stats.blobsOversized} " + @@ -480,10 +490,31 @@ object Main { FileRepositoryBuilder.create(gitDir).asInstanceOf[FileRepository] } - private def openOrInitDst(path: Path): FileRepository = { + /** `borrowFrom` names src's objects directory: it is written to dst's + * objects/info/alternates BEFORE dst is opened for the walk, because JGit reads + * that file once, and a walk that skipped a copy must find the object there. */ + private[blobexec] def openOrInitDst(path: Path, borrowFrom: Option[Path] = None): FileRepository = { val exists = Files.isDirectory(path) val repo = FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository] if (!exists) repo.create(true) - repo + borrowFrom match { + case None => repo + case Some(objects) => + val info = repo.getObjectsDirectory.toPath.resolve("info") + repo.close() + Files.createDirectories(info) + writeAlternates(info.resolve("alternates"), objects) + FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository] + } + } + + /** One absolute line, replacing whatever was there: a resume writes the same + * file again. Written to a temporary file and renamed, so git never reads half. */ + private[blobexec] def writeAlternates(file: Path, objects: Path): Unit = { + val line = objects.toAbsolutePath.normalize.toString + "\n" + val tmp = file.resolveSibling(file.getFileName.toString + ".tmp") + Files.write(tmp, line.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + Files.move(tmp, file, java.nio.file.StandardCopyOption.REPLACE_EXISTING, + java.nio.file.StandardCopyOption.ATOMIC_MOVE) } } diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala index 5c1ece55..19aae1c7 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala @@ -35,7 +35,9 @@ final case class WalkStats( originalBlobCacheHits: Long, originalBlobDestinationLookups: Long, originalBlobBytesCopied: Long, - originalBlobBytesAvoided: Long + originalBlobBytesAvoided: Long, + /** Original blobs left in src, which dst reads through objects/info/alternates. */ + originalBlobsBorrowed: Long = 0L ) /** Rebuild `src` history into `dst`, persisting mappings to `mapping`. @@ -53,6 +55,10 @@ final class Walker( shard: Option[(Int, Int)] = None, deduplicateOriginalBlobs: Boolean = true, destinationMayContainObjects: Boolean = true, + // dst's objects/info/alternates names src's objects: an original blob is + // already readable from dst, so it is never copied. The caller must repack dst + // without -l before dropping the alternates file (run_pipeline_process.sh does). + borrowOriginalObjects: Boolean = false, blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, workerPool: Option[TokenizerWorkerPool] = None, @@ -79,6 +85,7 @@ final class Walker( private val originalBlobDestinationLookups = new LongAdder private val originalBlobBytesCopied = new LongAdder private val originalBlobBytesAvoided = new LongAdder + private val originalBlobsBorrowed = new LongAdder // -- stall watchdog ------------------------------------------------------ // `Await.result` is unbounded: a duration cannot tell a wedged run from a large @@ -215,7 +222,8 @@ final class Walker( originalBlobCacheHits = originalBlobCacheHits.sum(), originalBlobDestinationLookups = originalBlobDestinationLookups.sum(), originalBlobBytesCopied = originalBlobBytesCopied.sum(), - originalBlobBytesAvoided = originalBlobBytesAvoided.sum() + originalBlobBytesAvoided = originalBlobBytesAvoided.sum(), + originalBlobsBorrowed = originalBlobsBorrowed.sum() ) } finally revWalk.close() } @@ -1139,6 +1147,13 @@ final class Walker( // Copying a 435 GB repository's unmasked blobs is work, not a stall. progress(s"original blob ${id.name}") + // Borrowed: src holds the blob and dst reads src's objects, so there is + // nothing to write. The pass-through copy was half the consumer's time. + if (borrowOriginalObjects) { + originalBlobsBorrowed.increment() + return + } + if (!deduplicateOriginalBlobs) { val (_, size) = insertBlob(inserter) originalBlobCopies.increment() diff --git a/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala new file mode 100644 index 00000000..08733c87 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala @@ -0,0 +1,198 @@ +package cregit.blobexec + +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.internal.storage.file.FileRepository +import org.eclipse.jgit.lib.ObjectId +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ +import scala.sys.process._ + +/** `--alternates`: dst borrows src's objects instead of copying the original + * blobs, and `git repack -a -d` (no -l) then makes dst whole. The output must be + * the same objects, refs and mapping rows as a run that copies. */ +class AlternatesSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { + + private val workRoot: Path = Files.createTempDirectory("alternates-") + + override def afterAll(): Unit = deleteRecursive(workRoot) + + private def deleteRecursive(p: Path): Unit = + if (Files.exists(p)) { + if (Files.isDirectory(p)) Files.list(p).iterator().asScala.toList.foreach(deleteRecursive) + Files.delete(p) + } + + private val Mask = """\.[ch]$""" + + private def commit(git: Git, files: Map[String, String], message: String): Unit = { + val wt = git.getRepository.getWorkTree.toPath + files.foreach { case (rel, body) => + val p = wt.resolve(rel) + Files.createDirectories(p.getParent) + Files.writeString(p, body) + } + files.keys.foreach(f => git.add().addFilepattern(f).call()) + git.commit().setMessage(message) + .setAuthor("Tester", "t@example.org").setCommitter("Tester", "t@example.org").call() + () + } + + /** Five commits on master plus a side branch. a.c/b.h are tokenized; the rest + * pass through, among them a blob that only an old commit holds. */ + private def sourceRepo(dir: Path): Git = { + Files.createDirectories(dir) + val git = Git.init().setDirectory(dir.toFile).setBare(false).call() + commit(git, Map("a.c" -> "int a;\n", "README.md" -> "readme 1\n", "docs/x.txt" -> "x\n"), "c1") + commit(git, Map("b.h" -> "int b;\n", "README.md" -> "readme 2\n"), "c2") + commit(git, Map("a.c" -> "int a = BOOM;\n", "docs/y.txt" -> "y\n"), "c3") + git.branchCreate().setName("side").call() + commit(git, Map("lib/z.c" -> "int z;\n", "README.md" -> "readme 3\n"), "c4") + commit(git, Map("docs/x.txt" -> "x2\n"), "c5") + git + } + + /** Uppercases its input; fails on a BOOM blob unless `flag` exists. */ + private def command(dir: Path, flag: Path): String = { + val f = dir.resolve("cmd.sh") + Files.writeString(f, + s"""#!/bin/sh + |c=$$(cat) + |case "$$c" in *BOOM*) [ -e '$flag' ] || exit 1 ;; esac + |printf '%s\\n' "$$c" | tr a-z A-Z + |""".stripMargin) + f.toFile.setExecutable(true) + f.toAbsolutePath.toString + } + + private def walk(src: FileRepository, dstPath: Path, db: Path, cmd: String, + borrow: Boolean, abortOnError: Boolean = false): WalkStats = { + val incremental = Files.isDirectory(dstPath) + val dst = Main.openOrInitDst(dstPath, Option.when(borrow)(src.getObjectsDirectory.toPath)) + val mapping = Mapping.open(db, cmd, Mask) + try new Walker(src, dst, mapping, Mask.r, cmd, abortOnError, 2, pipeline = true, + destinationMayContainObjects = incremental, borrowOriginalObjects = borrow, + denylist = BlobDenylist.empty).run() + finally { mapping.close(); dst.close() } + } + + private def git(gitDir: Path, args: String*): (Int, String) = { + val out = new StringBuilder + val rc = Process(Seq("git", "--git-dir", gitDir.toString) ++ args) + .!(ProcessLogger(l => out.append(l).append('\n'), _ => ())) + (rc, out.toString) + } + + private def reachable(gitDir: Path): Vector[String] = { + val (rc, out) = git(gitDir, "rev-list", "--objects", "--all") + rc shouldBe 0 + out.linesIterator.map(_.takeWhile(_ != ' ')).toVector.sorted + } + + private def refs(gitDir: Path): String = git(gitDir, "for-each-ref", "--format=%(objectname) %(refname)")._2 + + private def table(db: Path, sql: String): Vector[String] = { + val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db") + try { + val rs = c.createStatement().executeQuery(sql) + val b = Vector.newBuilder[String] + while (rs.next()) b += (1 to rs.getMetaData.getColumnCount).map(rs.getString).mkString(" ") + b.result() + } finally c.close() + } + + // processed_at is a wall-clock stamp, so it is left out. + private val MappingQueries = Seq( + "SELECT orig_commit, new_commit FROM commit_map ORDER BY 1", + "SELECT orig_blob, path, new_blob FROM blob_map ORDER BY 1, 2", + "SELECT orig_tree, new_tree FROM tree_map ORDER BY 1") + + private def alternates(dst: Path): Path = dst.resolve("objects/info/alternates") + + /** What run_pipeline_process.sh's dissolve_alternates does, minus logging. */ + private def dissolve(dst: Path): Unit = { + git(dst, "repack", "-a", "-d")._1 shouldBe 0 + Files.delete(alternates(dst)) + git(dst, "fsck", "--connectivity-only", "--no-dangling")._1 shouldBe 0 + } + + test("a borrowing walk copies no original blob and, once repacked, equals a copying walk") { + val w = Files.createTempDirectory(workRoot, "equal-") + val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] + val flag = Files.createFile(w.resolve("flag")) + val cmd = command(w, flag) + + val copied = walk(src, w.resolve("copy.git"), w.resolve("copy.db"), cmd, borrow = false) + val borrowed = walk(src, w.resolve("borrow.git"), w.resolve("borrow.db"), cmd, borrow = true) + + copied.originalBlobCopies should be > 0L + copied.originalBlobsBorrowed shouldBe 0L + borrowed.originalBlobCopies shouldBe 0L + borrowed.originalBlobsBorrowed shouldBe copied.originalBlobCopyRequests + borrowed.commitsProcessed shouldBe 5 + + val b = w.resolve("borrow.git") + Files.readString(alternates(b)).trim shouldBe src.getObjectsDirectory.toPath.toAbsolutePath.normalize.toString + + // Without the file, dst is missing the borrowed blobs: the pack is not optional. + Files.move(alternates(b), b.resolve("objects/info/alternates.aside")) + git(b, "fsck", "--connectivity-only", "--no-dangling")._1 should not be 0 + Files.move(b.resolve("objects/info/alternates.aside"), alternates(b)) + + dissolve(b) + reachable(b) shouldBe reachable(w.resolve("copy.git")) + refs(b) shouldBe refs(w.resolve("copy.git")) + for (q <- MappingQueries) + table(w.resolve("borrow.db"), q) shouldBe table(w.resolve("copy.db"), q) + + // A non-bare clone, as step 6 makes, reads every file. + val clone = w.resolve("clone") + Process(Seq("git", "clone", "-q", b.toString, clone.toString)).! shouldBe 0 + Files.readString(clone.resolve("README.md")) shouldBe "readme 3\n" + Files.readString(clone.resolve("a.c")) shouldBe "INT A = BOOM;\n" + } + + test("an interrupted borrowing walk resumes to the same result") { + val w = Files.createTempDirectory(workRoot, "resume-") + val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] + val flag = w.resolve("flag") + val cmd = command(w, flag) + val dst = w.resolve("dst.git") + val db = w.resolve("dst.db") + + // c3's BOOM blob aborts the walk after c1 and c2. + val first = walk(src, dst, db, cmd, borrow = true, abortOnError = true) + first.aborted shouldBe true + table(db, "SELECT count(*) FROM commit_map") shouldBe Vector("2") + + Files.createFile(flag) + val second = walk(src, dst, db, cmd, borrow = true, abortOnError = true) + second.aborted shouldBe false + second.commitsProcessed shouldBe 3 + dissolve(dst) + + // Reference: one uninterrupted copying walk. + val ref = w.resolve("ref.git") + walk(src, ref, w.resolve("ref.db"), cmd, borrow = false).aborted shouldBe false + reachable(dst) shouldBe reachable(ref) + refs(dst) shouldBe refs(ref) + for (q <- MappingQueries) table(db, q) shouldBe table(w.resolve("ref.db"), q) + } + + test("a resume reuses the alternates file, rewriting it rather than appending") { + val w = Files.createTempDirectory(workRoot, "rewrite-") + val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] + val dst = w.resolve("dst.git") + Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath)).close() + Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath)).close() + Files.readAllLines(alternates(dst)).asScala.toList shouldBe + List(src.getObjectsDirectory.toPath.toAbsolutePath.normalize.toString) + // And dst opened that way really reads src's objects. + val r = Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath)) + try r.getObjectDatabase.has(ObjectId.fromString(src.resolve("HEAD").name)) shouldBe true + finally r.close() + } +} diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 7ac200e4..55797d8b 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -85,6 +85,9 @@ Output: with millions of loose objects. A failed pack never aborts the run: the Parquet dataset is the product, and packing only makes later steps faster. + Except while the cregit repo borrows objects (see + --copy-objects): that pack is required, and a failed one + stops the run, resumably, with a PACK-FAILED marker. --memory-limit SIZE cap the DuckDB heap in the dataset generator (step 10). Omit to accept that script's own default of 8GB. Takes an @@ -117,6 +120,14 @@ Tokenizer: --tokenizer-worker tokenize through a pool of persistent tokenizer processes; same output. Needs --mode pipeline or pipeline-trees (default: off) + --copy-objects + copy every untokenized original blob into the cregit repo during + step 2, as before (env CREGIT_COPY_OBJECTS=1 does the same). + By default step 2 copies none: the cregit repo reads them from + the original repo through objects/info/alternates, and the pack + after step 2 (git repack -a -d, without -l) copies them in once + and deletes that file. Same objects, half the step-2 work. + --gc none and --mode sharded always copy. --jobs N concurrent blame/HTML processes (default: CREGIT_JOBS, otherwise min(4, available CPUs)). Blame is the pipeline's bottleneck. Each file is independent, so the output does not @@ -144,9 +155,64 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" } +# True while dst reads objects from src (blobExec --alternates). `.packing` is +# the file set aside while fsck proves dst complete without it. +borrowing_objects() { + [ -e "$REPO_PATH_CREGIT_BARE/objects/info/alternates" ] \ + || [ -e "$REPO_PATH_CREGIT_BARE/objects/info/alternates.packing" ] +} + +# Copies the borrowed objects into dst and stops the borrowing. Not optional, as +# packing otherwise is: until this succeeds dst is unreadable without src. +# repack -a -d one pack of every reachable object, the borrowed ones too: +# WITHOUT -l, which would leave them out +# fsck run with the alternates file set aside, so it proves that dst +# is complete on its own; the file is deleted only after that +# A failure puts the file back (dst readable again), writes PACK-FAILED and stops +# the run; any resume retries this first. +dissolve_alternates() { + local dst="$REPO_PATH_CREGIT_BARE" alt="$REPO_PATH_CREGIT_BARE/objects/info/alternates" + local repack_args=(-a -d) + [ "$GC_MODE" = aggressive ] && repack_args+=(-f --depth=50 --window=250) + # A run killed during fsck left the file set aside: put it back first. + if [ -e "$alt.packing" ] && [ ! -e "$alt" ]; then + mv "$alt.packing" "$alt" + fi + log "packing $dst with the objects it borrows from $(cat "$alt")" + if git --git-dir="$dst" pack-refs --all \ + && git --git-dir="$dst" repack "${repack_args[@]}" \ + && mv "$alt" "$alt.packing" \ + && git --git-dir="$dst" fsck --connectivity-only --no-dangling; then + rm -f "$alt.packing" "${WORK}/PACK-FAILED" + # What gc does after its repack; neither one is needed for correctness. + git --git-dir="$dst" prune --expire=now \ + || log "warning: git prune failed; unreachable loose objects stay" + git --git-dir="$dst" commit-graph write --reachable \ + || log "warning: commit-graph write failed; later steps walk history more slowly" + log "pack done: $dst holds all its objects and no longer borrows any" + return 0 + fi + if [ -e "$alt.packing" ] && [ ! -e "$alt" ]; then + mv "$alt.packing" "$alt" + fi + local summary="the pack that copies the borrowed original blobs into $dst failed. + $dst still reads them from $REPO_PATH_ORIGINAL_BARE (objects/info/alternates), + so it is readable, but it must not outlive that repo. Free memory or disk." + write_resume_marker "PACK-FAILED" "git repack -a -d" 1 "$summary" + die "$summary + Marker: ${WORK}/PACK-FAILED + Resume at step 3: the runner retries the pack before anything else. + run_pipeline_process.sh: $0 [same flags] 3 + ctp.py (the external driver): python3 ctp.py run [same flags] --from-step 3" +} + # Packs the generated bare repo, as --gc selects. A gc failure only warns: under # `set -e` it would fire the EXIT trap, which deletes $WORK and every finished step. pack_cregit_repo() { + if borrowing_objects; then + dissolve_alternates + return 0 + fi case "$GC_MODE" in none) log "gc skipped (--gc none)" @@ -222,8 +288,10 @@ RETOKENIZE="" BLOB_TIMEOUT="${CREGIT_BLOB_TIMEOUT:-}" STALL_TIMEOUT="${CREGIT_STALL_TIMEOUT:-}" TOKENIZER_WORKER=0 +# 1: step 2 copies the original blobs into dst instead of borrowing them. +COPY_OBJECTS="${CREGIT_COPY_OBJECTS:-0}" -KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES" +KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES PACK-FAILED" keep_markers_present() { local m @@ -384,6 +452,7 @@ while [ $# -gt 0 ]; do --retokenize) need_val "$@"; RETOKENIZE="$2"; shift 2 ;; --blob-timeout) need_val "$@"; BLOB_TIMEOUT="$2"; shift 2 ;; --tokenizer-worker) TOKENIZER_WORKER=1; shift ;; + --copy-objects) COPY_OBJECTS=1; shift ;; --stall-timeout) need_val "$@"; STALL_TIMEOUT="$2"; shift 2 ;; --gc) need_val "$@"; GC_MODE="$2"; shift 2 ;; --memory-limit) need_val "$@"; MEMORY_LIMIT="$2"; shift 2 ;; @@ -877,12 +946,17 @@ else [ "$MASK_WIDENED" = 1 ] && WIDENED_FLAG="--mask-widened" WORKER_FLAG="" [ "$TOKENIZER_WORKER" = 1 ] && WORKER_FLAG="--tokenizer-worker=${CREGIT}/tokenizeByBlobId/tokenWorker.pl" + # Borrowing needs the pack that dissolves it, so --gc none copies instead. + ALTERNATES_FLAG="" + if [ "$COPY_OBJECTS" != 1 ] && [ "$GC_MODE" != none ]; then + ALTERNATES_FLAG="--alternates" + fi # blobExec refuses --retokenize without --memo-dir: the memo caches the same tokens. RETOKENIZE_FLAGS=() if [ -n "$RETOKENIZE" ]; then RETOKENIZE_FLAGS=("--retokenize=$RETOKENIZE" "--memo-dir=$MEMO_DIR") fi - java -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} \ + java -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} $ALTERNATES_FLAG \ "--tokenizer-identity=$TOKENIZER_IDENTITY" \ ${RETOKENIZE_FLAGS[@]+"${RETOKENIZE_FLAGS[@]}"} \ ${BLOB_TIMEOUT:+--blob-timeout=$BLOB_TIMEOUT} \ @@ -915,6 +989,13 @@ pack_cregit_repo fi end_step +# A resume past step 2 can find dst still borrowing: the pack failed or was +# killed. Steps 3+ must not start on that, so the pack runs first. +if [ "$FROM_STEP" -gt 2 ] && [ -d "$REPO_PATH_CREGIT_BARE" ] && borrowing_objects; then + log "resume: $REPO_PATH_CREGIT_BARE still borrows objects; packing it before step $FROM_STEP" + dissolve_alternates +fi + # --------------------------------------------------------------------------- # Step 3 — git log DB (original repo) # --------------------------------------------------------------------------- diff --git a/tests/pack_alternates.sh b/tests/pack_alternates.sh new file mode 100755 index 00000000..9a7b1ee0 --- /dev/null +++ b/tests/pack_alternates.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# Tests step 2's borrowed objects: blobExec --alternates by default, the pack that +# must copy them into the cregit repo, and what happens when that pack fails. +# java is stubbed: as blobExec it builds a real dst that borrows src's objects. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNNER="$ROOT/run_pipeline_process.sh" +PASS=0 +FAIL=0 + +check() { # $1 = description, $2 = condition result (0/1) + if [ "$2" -eq 0 ]; then + echo " ok — $1"; PASS=$((PASS + 1)) + else + echo " FAIL — $1"; FAIL=$((FAIL + 1)) + fi +} + +# As blobExec: dst gets one new commit on src's tree. With --alternates it +# borrows that tree and its blobs; without, it copies them (git fetch). +# STUB_LOCK_PACK=1 then makes objects/pack read-only, so a repack fails. +make_stub_java() { # $1 = bin dir + cat > "$1/java" <<'STUB' +#!/usr/bin/env bash +[ -n "${STUB_ARGV:-}" ] && printf '%s\n' "$*" >> "$STUB_ARGV" +case "$*" in *blobExec*) ;; *) exit 0 ;; esac +pos=(); skip=0; alt=0 +for a in "$@"; do + if [ "$skip" = 1 ]; then skip=0; continue; fi + case "$a" in + -jar) skip=1 ;; + --alternates) alt=1 ;; + -*) ;; + *) pos+=("$a") ;; + esac +done +src=${pos[0]} dst=${pos[1]} +[ -d "$dst" ] || git init -q --bare "$dst" +if [ "$alt" = 1 ]; then + echo "$(cd "$src/objects" && pwd)" > "$dst/objects/info/alternates" +else + git --git-dir="$dst" fetch -q "$src" "+refs/heads/*:refs/remotes/src/*" +fi +tree=$(git --git-dir="$src" rev-parse 'HEAD^{tree}') +c=$(echo rewritten | GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@x GIT_COMMITTER_NAME=t \ + GIT_COMMITTER_EMAIL=t@x GIT_AUTHOR_DATE='@1700000000 +0000' GIT_COMMITTER_DATE='@1700000000 +0000' \ + git --git-dir="$dst" commit-tree "$tree") +git --git-dir="$dst" update-ref refs/heads/master "$c" +git --git-dir="$dst" for-each-ref --format='%(refname)' refs/remotes | while read -r r; do + git --git-dir="$dst" update-ref -d "$r"; done +[ "${STUB_LOCK_PACK:-0}" = 1 ] && chmod a-w "$dst/objects/pack" +exit 0 +STUB + chmod +x "$1/java" +} + +# The runner takes CREGIT from $(pwd): tokenize/ for the identity, plus build +# files and placeholder artifacts, or the staleness guard would build. +repo_fixture() { + local f p + f=$(mktemp -d "${TMPDIR:-/tmp}/packrepo-XXXXXX") + cp "$RUNNER" "$f/run_pipeline_process.sh" + cp -r "$ROOT/tokenize" "$f/tokenize" + rm -rf "$f/tokenize/rustTokenizer/target" "$f/tokenize/srcMLtoken/srcml2token" + for p in blobExec slickGitLog persons remapCommits; do + mkdir -p "$f/$p/src" + cp "$ROOT/$p/build.sbt" "$f/$p/build.sbt" + cp -r "$ROOT/$p/project" "$f/$p/project" + rm -rf "$f/$p/project/target" "$f/$p/project/project" + done + for p in tokenize/srcMLtoken/srcml2token \ + tokenize/rustTokenizer/target/release/rust_tokenizer \ + blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar \ + slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar \ + persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar \ + remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar + do + mkdir -p "$f/$(dirname "$p")" + echo "placeholder v1" > "$f/$p" + chmod +x "$f/$p" + done + printf '#!/bin/sh\necho "%s/libsrcml"\n' "$f" > "$f/tokenize/srcMLtoken/srcml2token" + echo "placeholder v1" > "$f/libsrcml" + printf '%s' "$f" +} + +# A work dir resumable at step 2: a real original bare repo with two commits. +fixture() { + local w t + w=$(mktemp -d "${TMPDIR:-/tmp}/packwork-XXXXXX") + t="$w/tmp-src" + git init -q "$t" + echo "int a;" > "$t/a.c"; echo "readme" > "$t/README" + git -C "$t" add . && git -C "$t" -c user.name=t -c user.email=t@x commit -q -m one + echo "readme 2" > "$t/README" + git -C "$t" -c user.name=t -c user.email=t@x commit -q -am two + git clone -q --bare "$t" "$w/proj-original.git" + rm -rf "$t" + mkdir -p "$w/memo" + printf '%s' "$w" +} + +run_from() { # $1 = work dir, rest = extra runner args + FROM_STEP + local w=$1; shift + ( cd "$REPO" \ + && PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 timeout 300 \ + bash ./run_pipeline_process.sh \ + --repo-url /nonexistent/does-not-exist.git \ + --repo-name proj \ + --work "$w" \ + --skip-html --mask '\.c$' \ + "$@" 2>&1 ) +} + +DST=proj-cregit.git + +# True when dst holds every object it references, without any alternates file. +self_contained() { # $1 = dst + [ ! -e "$1/objects/info/alternates" ] && [ ! -e "$1/objects/info/alternates.packing" ] \ + && git --git-dir="$1" fsck --connectivity-only --no-dangling >/dev/null 2>&1 \ + && git --git-dir="$1" cat-file -e 'master:README' +} + +BIN=$(mktemp -d "${TMPDIR:-/tmp}/packbin-XXXXXX") +make_stub_java "$BIN" +REPO=$(repo_fixture) +trap 'rm -rf "$BIN" "$REPO"' EXIT + +# --------------------------------------------------------------------------- +echo "case 1: by default step 2 borrows, and the pack leaves dst whole on its own" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_ARGV="$ARGV" run_from "$W" 2) +grep -q 'Step 3' <<<"$OUT"; check "step 2 completed and the pipeline moved on" $? +grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; check "blobExec was given --alternates" $? +grep -q 'no longer borrows any' <<<"$OUT"; check "the runner packed the borrowed objects in" $? +self_contained "$W/$DST"; check "dst has no alternates file and fsck passes without it" $? +git --git-dir="$W/$DST" count-objects -v | grep -q '^count: 0' +check "every object is packed, none left loose" $? +[ ! -e "$W/PACK-FAILED" ]; check "no PACK-FAILED marker" $? +git clone -q "$W/$DST" "$W/clone" && [ -f "$W/clone/README" ] && [ ! -e "$W/clone/.git/objects/info/alternates" ] +check "a step-6 style clone reads every file and borrows nothing" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 2: --copy-objects, CREGIT_COPY_OBJECTS=1 and --gc none all copy instead" +for how in flag env gcnone; do + W=$(fixture); ARGV="$W/argv.log" + case $how in + flag) OUT=$(STUB_ARGV="$ARGV" run_from "$W" --copy-objects 2) ;; + env) OUT=$(CREGIT_COPY_OBJECTS=1 STUB_ARGV="$ARGV" run_from "$W" 2) ;; + gcnone) OUT=$(STUB_ARGV="$ARGV" run_from "$W" --gc none 2) ;; + esac + grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; [ $? -ne 0 ] + check "$how: blobExec was NOT given --alternates" $? + [ ! -e "$W/$DST/objects/info/alternates" ] && git --git-dir="$W/$DST" cat-file -e 'master:README' + check "$how: dst holds its objects without borrowing" $? + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "case 3: a failed pack stops the run, resumably, and leaves dst readable" +W=$(fixture) +OUT=$(STUB_LOCK_PACK=1 run_from "$W" 2); RC=$? +[ "$RC" -ne 0 ]; check "the run failed (got exit $RC)" $? +grep -q 'Step 3' <<<"$OUT"; [ $? -ne 0 ]; check "and stopped before step 3" $? +[ -f "$W/PACK-FAILED" ]; check "a PACK-FAILED marker was written" $? +[ -f "$W/$DST/objects/info/alternates" ]; check "the alternates file is still in place" $? +git --git-dir="$W/$DST" cat-file -e 'master:README'; check "so dst still reads the borrowed blobs" $? +chmod u+w "$W/$DST/objects/pack" +OUT=$(run_from "$W" 3) +grep -q 'still borrows objects; packing it before step 3' <<<"$OUT" +check "a resume at step 3 packs first" $? +self_contained "$W/$DST"; check "and dst is then whole on its own" $? +[ ! -e "$W/PACK-FAILED" ]; check "and the marker is gone" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 4: a run killed while fsck ran (file set aside) is put back and redone" +W=$(fixture) +OUT=$(STUB_LOCK_PACK=1 run_from "$W" 2) +chmod u+w "$W/$DST/objects/pack" +mv "$W/$DST/objects/info/alternates" "$W/$DST/objects/info/alternates.packing" +OUT=$(run_from "$W" 4) +grep -q 'packing it before step 4' <<<"$OUT"; check "a resume at step 4 packs first" $? +self_contained "$W/$DST"; check "and dst is whole, with neither file left" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 5: a full run (FROM_STEP=1) keeps the work dir when the pack fails" +W=$(mktemp -d "${TMPDIR:-/tmp}/packfull-XXXXXX") +S=$(fixture) +OUT=$( cd "$REPO" && STUB_LOCK_PACK=1 PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 timeout 300 \ + bash ./run_pipeline_process.sh --repo-url "$S/proj-original.git" --repo-name proj \ + --work "$W/work" --skip-html --mask '\.c$' 2>&1 ); RC=$? +[ "$RC" -ne 0 ] && [ -f "$W/work/PACK-FAILED" ] && [ -d "$W/work/$DST" ] +check "the EXIT trap kept \$WORK: the marker says it is resumable (exit $RC)" $? +chmod -R u+w "$W" "$S"; rm -rf "$W" "$S" + +echo "" +echo "pack_alternates: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] From c6afc4b51aa076561f7ae18ce639f3e6adb4281c Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 17:19:34 +0000 Subject: [PATCH 2/9] perf(blobExec): group commits per SQLite transaction, size the cache Step 2's mapping DB ran one transaction per commit with SQLite's 2 MB page cache and no mmap. On psi-probe the per-commit transaction was 28% of the consumer thread and lookups 79% of the producer; on foxhound the JVM made 7.0e9 page reads against a 2.5 GB DB. Mapping.inGroupedTx lets up to 256 commits (or 2 s of them) share one transaction. A group is all-or-nothing like the transaction it replaces: a commit's rows are one body, a throw in a body rolls the group back, inTx and close commit an open group first, and the walk commits it when it stops, by abort too. So a commit row is still never durable without its trees and blobs. A crash or a watchdog halt loses at most one group, which resume re-walks with tokens from the memo. Mapping.open also sets cache_size to 256 MiB and mmap_size to 4 GiB. The cache is private memory that fills only as pages are read, so at most the DB's size: up to 768 MiB for the census's three runs. The map reads the kernel's page cache in place; those are shared, reclaimable file pages, the ones plain reads cache anyway, so it adds no memory. Defaults on in blobExec, so the runner and ctp.py need no change. Opt out with --commits-per-transaction=1, --sqlite-cache-mb=0 and --sqlite-mmap-mb=0. Only speed changes: rows and order are the same. --- .../src/main/scala/cregit/blobexec/Main.scala | 30 +++++- .../main/scala/cregit/blobexec/Mapping.scala | 92 +++++++++++++++- .../main/scala/cregit/blobexec/Walker.scala | 31 +++++- .../cregit/blobexec/AlternatesSpec.scala | 18 ++-- .../cregit/blobexec/MainOptionsSpec.scala | 25 ++++- .../scala/cregit/blobexec/MappingSpec.scala | 102 ++++++++++++++++++ 6 files changed, 282 insertions(+), 16 deletions(-) diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index fc852ad3..52b6b01f 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -77,6 +77,11 @@ object Main { | --alternates write /objects/info/alternates naming 's objects | and copy no original blob; the caller must then repack | without -l and delete that file. Not with --shard + | --commits-per-transaction= + | commits whose mapping rows share one SQLite transaction + | (default ${Walker.DefaultCommitsPerTransaction}; 1 = one per commit) + | --sqlite-cache-mb= SQLite page cache for the mapping DB (default ${Mapping.DefaultCacheMiB}; 0 = SQLite's 2 MB) + | --sqlite-mmap-mb= memory-map up to this much of the mapping DB (default ${Mapping.DefaultMmapMiB}; 0 = off) | --blob-timeout= budget for one run (default ${BlobExec.DefaultTimeoutSeconds}); then its process | tree is killed and the blob excluded | --stall-timeout= exit ${Walker.StalledExitStatus} after this long with no progress (default ${Walker.DefaultStallTimeoutSeconds}); @@ -107,6 +112,8 @@ object Main { memoDir: Option[Path] = None, tokenizerWorker: Option[Path] = None, alternates: Boolean = false, + commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction, + sqlite: Mapping.Tuning = Mapping.Tuning(), blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, stallExplicit: Boolean = false, @@ -228,10 +235,19 @@ object Main { private val TokenizerWorkerFlag = new ValuedFlag("--tokenizer-worker") private val BlobTimeoutFlag = new ValuedFlag("--blob-timeout") private val StallTimeoutFlag = new ValuedFlag("--stall-timeout") + private val CommitsPerTxFlag = new ValuedFlag("--commits-per-transaction") + private val SqliteCacheFlag = new ValuedFlag("--sqlite-cache-mb") + private val SqliteMmapFlag = new ValuedFlag("--sqlite-mmap-mb") private def flagValue[A](flag: ValuedFlag, spec: String)(parse: String => Either[String, A]): Either[String, A] = parse(spec).left.map(Messages.badFlagValue(flag.name, _)) + private def positive(spec: String): Either[String, Int] = + spec.toIntOption.filter(_ > 0).toRight(s"must be a positive whole number [$spec]") + + private def mebibytes(spec: String): Either[String, Int] = + spec.toIntOption.filter(_ >= 0).toRight(s"must be a whole number of MiB, 0 or more [$spec]") + private def seconds(spec: String): Either[String, Int] = parsePositiveSeconds(spec).toRight(Messages.secondsWanted(spec)) @@ -272,6 +288,12 @@ object Main { Right(o.copy(tokenizerWorker = Some(Paths.get(spec)))) case BlobTimeoutFlag(spec) => flagValue(BlobTimeoutFlag, spec)(seconds).map(secs => o.copy(blobTimeoutSeconds = secs)) + case CommitsPerTxFlag(spec) => + flagValue(CommitsPerTxFlag, spec)(positive).map(n => o.copy(commitsPerTransaction = n)) + case SqliteCacheFlag(spec) => + flagValue(SqliteCacheFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(cacheMiB = n))) + case SqliteMmapFlag(spec) => + flagValue(SqliteMmapFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(mmapMiB = n))) case StallTimeoutFlag(spec) => flagValue(StallTimeoutFlag, spec)(seconds).map(secs => o.copy(stallTimeoutSeconds = secs, stallExplicit = true)) case other => Left(Messages.unknownFlag(other)) @@ -376,13 +398,16 @@ object Main { s"maskWidened=${o.maskWidened} denylistEntries=${BlobDenylist.shipped.size} " + s"tokenizerIdentity=${if (o.tokenizerIdentity.isEmpty) "none" else o.tokenizerIdentity.render} " + s"retokenize=${if (o.retokenize.isEmpty) "none" else o.retokenize.toVector.sorted.mkString(",")} " + - s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)}" + s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)} " + + s"alternates=${o.alternates} commitsPerTransaction=${o.commitsPerTransaction} " + + s"sqliteCacheMiB=${o.sqlite.cacheMiB} sqliteMmapMiB=${o.sqlite.mmapMiB}" } private final case class Refusal(status: Int, message: String) private def openMapping(o: Options, src: FileRepository, dst: FileRepository): Either[Refusal, Mapping] = - Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst))) + Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst), + o.sqlite)) .toEither .left.map(e => refusalFor(e, o).getOrElse(throw e)) @@ -448,6 +473,7 @@ object Main { blobTimeoutSeconds = o.blobTimeoutSeconds, stallTimeoutSeconds = stallSeconds, workerPool = workerPool, + commitsPerTransaction = o.commitsPerTransaction, denylist = BlobDenylist.shipped ).run() finally workerPool.foreach(_.close()) diff --git a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala index d6739ae7..c13e57ee 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala @@ -296,8 +296,10 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends Mapping.RetokenizeResult(before, rowCounts, extensions, invalidated, memo) } - /** Run `body` inside a transaction; commit on success, rollback on throw. */ + /** Run `body` inside a transaction; commit on success, rollback on throw. + * An open group (see [[inGroupedTx]]) is committed first. */ def inTx[A](body: => A): A = { + commitGroup() conn.setAutoCommit(false) try { val result = body @@ -312,7 +314,67 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } } + /** A PRAGMA's current value on this connection, for tests. */ + private[blobexec] def pragmaValue(name: String): Long = { + val st = conn.createStatement() + try { val rs = st.executeQuery(s"PRAGMA $name"); rs.next(); rs.getLong(1) } finally st.close() + } + + // -- grouped transactions --------------------------------------------------- + // A commit's rows are one body; up to `maxBodies` bodies share one transaction. + // Each SQLite transaction rewrites the pages it touched into the WAL, and the + // walk touches the same index pages for every commit, so per-commit + // transactions cost more than the rows. A group is all-or-nothing, like the + // one-commit transaction it replaces: a crash loses whole bodies, never part of + // one, so a commit row is still never durable without its trees and blobs, and + // the lost commits are re-walked on resume (their tokens come from the memo). + + private var groupOpen = false + private var groupBodies = 0 + private var groupStartNanos = 0L + + /** As [[inTx]], but `body` joins the open group, which is committed once it + * holds `maxBodies` bodies or is `maxNanos` old. A throw in `body` rolls back + * the whole group. `maxBodies` <= 1 is plain [[inTx]]. Not thread-safe: the + * caller serializes access, as it does for every other method here. */ + def inGroupedTx[A](maxBodies: Int, maxNanos: Long)(body: => A): A = + if (maxBodies <= 1) inTx(body) + else { + if (!groupOpen) { + conn.setAutoCommit(false) + groupOpen = true + groupBodies = 0 + groupStartNanos = System.nanoTime() + } + val result = + try body + catch { case t: Throwable => abandonGroup(); throw t } + groupBodies += 1 + if (groupBodies >= maxBodies || System.nanoTime() - groupStartNanos >= maxNanos) commitGroup() + result + } + + /** Commit the open group, if any. The walk calls it when it stops, by abort too. */ + def commitGroup(): Unit = + if (groupOpen) { + try conn.commit() + catch { case t: Throwable => abandonGroup(); throw t } + groupOpen = false + groupBodies = 0 + conn.setAutoCommit(true) + } + + private def abandonGroup(): Unit = + if (groupOpen) { + groupOpen = false + groupBodies = 0 + try conn.rollback() catch { case _: Throwable => () } + try conn.setAutoCommit(true) catch { case _: Throwable => () } + } + override def close(): Unit = { + // Every body in the group finished (a failed one rolled the group back). + commitGroup() (List(selBlob, insBlob, selCommit, insCommit, selTree, insTree, selRef, insRef, delRef, selMeta, insMeta) ++ warmSelBlob.toList ++ warmSelTree.toList) .foreach(s => try s.close() catch { case _: Throwable => () }) @@ -456,6 +518,30 @@ object Mapping { } finally st.close() } + /** Read-path sizing for the walk's DB. Both change speed only, never results. + * cacheMiB SQLite's own page cache, private anonymous memory, filled only + * as pages are read: the DB's size at most (default 2 MB). + * mmapMiB map up to this much of the file. Lookups then read the kernel's + * page cache in place instead of copying each page out with a + * syscall. Those are file pages: shared and reclaimable, and the + * same pages that reads cache anyway. 0 turns it off. */ + final case class Tuning(cacheMiB: Int = DefaultCacheMiB, mmapMiB: Int = DefaultMmapMiB) { + require(cacheMiB >= 0 && mmapMiB >= 0, s"SQLite sizes must be >= 0 (cache=$cacheMiB MiB, mmap=$mmapMiB MiB)") + } + val DefaultCacheMiB = 256 + val DefaultMmapMiB = 4096 + /** SQLite's defaults: what every run used before. */ + val UntunedSqlite: Tuning = Tuning(cacheMiB = 0, mmapMiB = 0) + + private def applyTuning(conn: Connection, t: Tuning): Unit = { + val st = conn.createStatement() + try { + // Negative cache_size is KiB; 0 keeps the default. + if (t.cacheMiB > 0) st.execute(s"PRAGMA cache_size = -${t.cacheMiB.toLong * 1024L}") + if (t.mmapMiB > 0) st.execute(s"PRAGMA mmap_size = ${t.mmapMiB.toLong * 1024L * 1024L}") + } finally st.close() + } + /** Opt-in for `--mask-widened`. `newBlobResolves` has no default: `new_blob` ids * live only in dst, and a wiped dst would dangle every reused row. */ final case class MaskWidening(newBlobResolves: String => Boolean, @@ -468,7 +554,8 @@ object Mapping { def open(path: Path, command: String, mask: String, warm: Option[Path] = None, maskWidening: Option[MaskWidening] = None, tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, - retokenize: Option[Retokenize] = None): Mapping = { + retokenize: Option[Retokenize] = None, + tuning: Tuning = Tuning()): Mapping = { // Refused, not composed: each invalidation is verified against a state the // other would change. Run one, then the other as a resume. require(!(maskWidening.isDefined && retokenize.isDefined), @@ -492,6 +579,7 @@ object Mapping { val url = s"jdbc:sqlite:${path.toAbsolutePath}" val conn = DriverManager.getConnection(url) enableForeignKeys(conn) + applyTuning(conn, tuning) val st = conn.createStatement() try Schema.foreach(st.execute) finally st.close() diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala index 19aae1c7..aea1cb94 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala @@ -62,6 +62,9 @@ final class Walker( blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, workerPool: Option[TokenizerWorkerPool] = None, + // Commits whose mapping rows share one SQLite transaction (Mapping.inGroupedTx). + // 1 is one transaction per commit, as before. + commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction, // A parameter only so tests can supply a fixture. denylist: BlobDenylist = BlobDenylist.shipped ) { @@ -72,6 +75,15 @@ final class Walker( // touch it, so the CPU-heavy path stays lock-free. private val dbLock = new AnyRef + /** On the way out of a walk: the group holds finished commits only (a failed + * body rolled it back), so keeping them is right, and a commit error here + * must not hide the one already propagating. */ + private def commitGroupQuietly(): Unit = + dbLock.synchronized { + try mapping.commitGroup() + catch { case scala.util.control.NonFatal(e) => System.err.println(s"warning: mapping commit failed: ${e.getMessage}") } + } + private val blobCommandExecutions = new LongAdder private val blobsTimedOut = new LongAdder private val blobsOversized = new LongAdder @@ -441,7 +453,7 @@ final class Walker( val newCommit = buildCommit(rc, parents, newTreeId, commitInserter) - mapping.inTx { + mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) { // Matching blobs the cmd resolved (Replace or Skip outcomes). resolved.foreach { case ((origSha, path), newId) => mapping.putBlob(origSha, path, newId.name) @@ -464,9 +476,11 @@ final class Walker( } } + mapping.commitGroup() treeInserter.flush() commitInserter.flush() } finally { + commitGroupQuietly() treeInserter.close() commitInserter.close() pool.shutdown() @@ -568,7 +582,7 @@ final class Walker( }.map(ObjectId.fromString) val newCommit = buildCommitFromData(data, parents, newTreeId, commitInserter) dbLock.synchronized { - mapping.inTx { + mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) { resolved.foreach { case ((origSha, path), newId) => mapping.putBlob(origSha, path, newId.name) } @@ -594,9 +608,11 @@ final class Walker( } } + dbLock.synchronized(mapping.commitGroup()) treeInserter.flush() commitInserter.flush() } finally { + commitGroupQuietly() treeInserter.close() commitInserter.close() pool.shutdown() @@ -701,7 +717,7 @@ final class Walker( }.map(ObjectId.fromString) val newCommit = buildCommitFromData(data, parents, newTreeId, commitInserter) dbLock.synchronized { - mapping.inTx { + mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) { resolved.foreach { case ((origSha, path), newId) => mapping.putBlob(origSha, path, newId.name) } @@ -725,8 +741,10 @@ final class Walker( } } + dbLock.synchronized(mapping.commitGroup()) commitInserter.flush() } finally { + commitGroupQuietly() commitInserter.close() pool.shutdown() if (!pool.awaitTermination(1, TimeUnit.MINUTES)) { pool.shutdownNow(); () } @@ -1408,6 +1426,13 @@ final class Walker( } object Walker { + /** Commits per SQLite transaction. A crash or a watchdog halt loses at most this + * many commits' rows (or [[TransactionMaxNanos]] of them), and resume re-walks + * them. */ + val DefaultCommitsPerTransaction = 256 + /** A group is also committed once this old, so a slow stretch keeps its rows. */ + val TransactionMaxNanos: Long = 2L * 1000L * 1000L * 1000L + /** JGit's default `core.streamFileThreshold`, at or above which `getBytes` * throws. Only a fast path: [[Walker#readBlob]] also catches the exception. */ diff --git a/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala index 08733c87..d175b81e 100644 --- a/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala @@ -69,13 +69,14 @@ class AlternatesSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { } private def walk(src: FileRepository, dstPath: Path, db: Path, cmd: String, - borrow: Boolean, abortOnError: Boolean = false): WalkStats = { + borrow: Boolean, abortOnError: Boolean = false, + commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction): WalkStats = { val incremental = Files.isDirectory(dstPath) val dst = Main.openOrInitDst(dstPath, Option.when(borrow)(src.getObjectsDirectory.toPath)) val mapping = Mapping.open(db, cmd, Mask) try new Walker(src, dst, mapping, Mask.r, cmd, abortOnError, 2, pipeline = true, destinationMayContainObjects = incremental, borrowOriginalObjects = borrow, - denylist = BlobDenylist.empty).run() + denylist = BlobDenylist.empty, commitsPerTransaction = commitsPerTransaction).run() finally { mapping.close(); dst.close() } } @@ -155,7 +156,7 @@ class AlternatesSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { Files.readString(clone.resolve("a.c")) shouldBe "INT A = BOOM;\n" } - test("an interrupted borrowing walk resumes to the same result") { + test("an interrupted borrowing walk resumes to the same result, with grouped transactions") { val w = Files.createTempDirectory(workRoot, "resume-") val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] val flag = w.resolve("flag") @@ -163,20 +164,21 @@ class AlternatesSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { val dst = w.resolve("dst.git") val db = w.resolve("dst.db") - // c3's BOOM blob aborts the walk after c1 and c2. - val first = walk(src, dst, db, cmd, borrow = true, abortOnError = true) + // c3's BOOM blob aborts the walk: c1 and c2 are done and must be durable even + // though they share one still-open transaction group. + val first = walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100) first.aborted shouldBe true table(db, "SELECT count(*) FROM commit_map") shouldBe Vector("2") Files.createFile(flag) - val second = walk(src, dst, db, cmd, borrow = true, abortOnError = true) + val second = walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100) second.aborted shouldBe false second.commitsProcessed shouldBe 3 dissolve(dst) - // Reference: one uninterrupted copying walk. + // Reference: one uninterrupted copying walk, one commit per transaction. val ref = w.resolve("ref.git") - walk(src, ref, w.resolve("ref.db"), cmd, borrow = false).aborted shouldBe false + walk(src, ref, w.resolve("ref.db"), cmd, borrow = false, commitsPerTransaction = 1).aborted shouldBe false reachable(dst) shouldBe reachable(ref) refs(dst) shouldBe refs(ref) for (q <- MappingQueries) table(db, q) shouldBe table(w.resolve("ref.db"), q) diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala index d5637fb2..9331afae 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala @@ -218,6 +218,26 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { inside(Main.parse("--shard=1/4" +: positional)) { case Right(o) => o.shard shouldEqual Some((1, 4)) } } + test("the step-2 speed flags are read, and default to on") { + inside(Main.parse(Seq("--alternates", "--commits-per-transaction=1", "--sqlite-cache-mb=0", + "--sqlite-mmap-mb=0") ++ positional)) { case Right(o) => + o.alternates shouldBe true + o.commitsPerTransaction shouldBe 1 + o.sqlite shouldEqual Mapping.UntunedSqlite + } + inside(Main.parse(positional)) { case Right(o) => + o.alternates shouldBe false // the caller must repack, so the caller asks for it + o.commitsPerTransaction shouldBe Walker.DefaultCommitsPerTransaction + o.sqlite shouldEqual Mapping.Tuning(Mapping.DefaultCacheMiB, Mapping.DefaultMmapMiB) + } + } + + test("--alternates is refused with --shard") { + inside(Main.parse(Seq("--alternates", "--shard=0/2") ++ positional)) { case Left(why) => + why should include("--alternates cannot be used with --shard") + } + } + test("parse leaves the defaults when no flag is given") { inside(Main.parse(positional)) { case Right(o) => o shouldEqual Main.Options(positional = positional.toVector) @@ -239,7 +259,10 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { "--shard=1" -> "--shard", "--warm=/nonexistent/db" -> "--warm", "--tokenizer-identity=" -> "--tokenizer-identity", - "--memo-dir=/nonexistent/" -> "--memo-dir" + "--memo-dir=/nonexistent/" -> "--memo-dir", + "--commits-per-transaction=0" -> "--commits-per-transaction", + "--sqlite-cache-mb=-1" -> "--sqlite-cache-mb", + "--sqlite-mmap-mb=lots" -> "--sqlite-mmap-mb" ).foreach { case (flag, name) => withClue(s"$flag: ") { inside(Main.parse(flag +: positional)) { case Left(why) => why should startWith(s"Error: $name") } diff --git a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala index 46af5926..1d4a1c15 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala @@ -441,4 +441,106 @@ class MappingSpec extends AnyFunSuite with Matchers { m.allCommitOrigShas.toSet shouldEqual Set("a", "b", "c") } } + + // -- grouped transactions ---------------------------------------------------- + + /** What a second connection (another process, or the DB after a crash) sees. */ + private def committedCommits(db: java.nio.file.Path): Int = { + val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db") + try { + val rs = c.createStatement().executeQuery("SELECT count(*) FROM commit_map") + rs.next(); rs.getInt(1) + } finally c.close() + } + + private val Forever = Long.MaxValue + + test("a group commits once it holds maxBodies bodies, and not before") { + val db = Files.createTempFile("mapping-group-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(3, Forever)(m.putCommit("c1", "n1")) + m.inGroupedTx(3, Forever)(m.putCommit("c2", "n2")) + committedCommits(db) shouldBe 0 + m.getCommit("c2") shouldBe Some("n2") // the writer reads its own open group + m.inGroupedTx(3, Forever)(m.putCommit("c3", "n3")) + committedCommits(db) shouldBe 3 + } finally m.close() + } + + test("a group also commits once it is maxNanos old") { + val db = Files.createTempFile("mapping-group-age-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(1000, 0L)(m.putCommit("c1", "n1")) + committedCommits(db) shouldBe 1 + } finally m.close() + } + + test("a body that throws rolls back its whole group and nothing committed before") { + val db = Files.createTempFile("mapping-group-throw-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(2, Forever)(m.putCommit("c1", "n1")) + m.inGroupedTx(2, Forever)(m.putCommit("c2", "n2")) // group of 2 committed + m.inGroupedTx(2, Forever)(m.putCommit("c3", "n3")) // open group + an[IllegalStateException] should be thrownBy + m.inGroupedTx(2, Forever) { m.putTree("t4", "n4"); throw new IllegalStateException("boom") } + m.getCommit("c3") shouldBe None + m.getTree("t4") shouldBe None + m.getCommit("c2") shouldBe Some("n2") + // And the connection is usable afterwards. + m.inGroupedTx(2, Forever)(m.putCommit("c5", "n5")) + m.commitGroup() + committedCommits(db) shouldBe 3 + } finally m.close() + } + + test("inTx and close commit an open group first") { + val db = Files.createTempFile("mapping-group-close-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + m.inGroupedTx(10, Forever)(m.putCommit("c1", "n1")) + m.inTx(m.setMeta("k", "v")) + committedCommits(db) shouldBe 1 + m.inGroupedTx(10, Forever)(m.putCommit("c2", "n2")) + m.close() + committedCommits(db) shouldBe 2 + } + + test("a lost open group is all-or-nothing: no commit row without its trees and blobs") { + // A crash is a connection that never commits: simulate it by abandoning the + // process's view (a second connection reads only what was committed). + val db = Files.createTempFile("mapping-group-crash-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(5, Forever) { m.putBlob("b1", "a.c", "nb1"); m.putTree("t1", "nt1"); m.putCommit("c1", "n1") } + committedCommits(db) shouldBe 0 + val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db") + try { + val rs = c.createStatement().executeQuery( + "SELECT (SELECT count(*) FROM blob_map) + (SELECT count(*) FROM tree_map)") + rs.next(); rs.getInt(1) shouldBe 0 + } finally c.close() + } finally m.close() + } + + test("maxBodies of 1 is a transaction per body, as before") { + val db = Files.createTempFile("mapping-group-one-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(1, Forever)(m.putCommit("c1", "n1")) + committedCommits(db) shouldBe 1 + } finally m.close() + } + + test("the SQLite sizes reach the connection") { + val db = Files.createTempFile("mapping-tuning-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*", tuning = Mapping.Tuning(cacheMiB = 64, mmapMiB = 128)) + try { + m.pragmaValue("cache_size") shouldBe -64L * 1024L + m.pragmaValue("mmap_size") shouldBe 128L * 1024L * 1024L + } finally m.close() + val u = Mapping.open(db, "/bin/cat", ".*", tuning = Mapping.UntunedSqlite) + try u.pragmaValue("mmap_size") shouldBe 0L finally u.close() + } } From afcb15a07eb8aafe888ab8490bf9d318d8db10a9 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 17:21:28 +0000 Subject: [PATCH 3/9] perf(blobExec): write dst's loose objects at zlib level 1 Every object step 2 writes is a loose file that lives only until the pack after step 2, which re-deflates it at git's pack.compression. So deflating it at JGit's default level 6 is wasted work: the profile measured -11% consumer CPU on psi-probe with level 1. blobExec sets core.compression=1 on dst's config in memory only. The config file is not written, so git repack and steps 3+ see the same config as before, and pack.compression needs no override. Object ids hash the content, not the deflate stream, so nothing in the output can depend on the level. If JGit reloaded the file mid-walk, the only effect would be the default level again. Default on in blobExec; opt out with --loose-compression=-1. --- .../src/main/scala/cregit/blobexec/Main.scala | 27 +++++++++++- .../blobexec/LooseCompressionSpec.scala | 43 +++++++++++++++++++ .../cregit/blobexec/MainOptionsSpec.scala | 7 ++- 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index 52b6b01f..c7baf12a 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -14,6 +14,11 @@ object Main { /** Not 3 (memo mismatch): a no-op request must be told apart from a refusal. */ private[blobexec] val RetokenizeIneffectiveExitStatus = 7 + /** zlib level 1 for dst's loose objects: they are temporary, because the pack + * after step 2 re-deflates every one at git's pack.compression. Measured -11% + * consumer CPU on psi-probe against JGit's default level. */ + private[blobexec] val DefaultLooseCompression = 1 + private val UsageExitStatus = 1 private val RefusedExitStatus = 3 @@ -82,6 +87,8 @@ object Main { | (default ${Walker.DefaultCommitsPerTransaction}; 1 = one per commit) | --sqlite-cache-mb= SQLite page cache for the mapping DB (default ${Mapping.DefaultCacheMiB}; 0 = SQLite's 2 MB) | --sqlite-mmap-mb= memory-map up to this much of the mapping DB (default ${Mapping.DefaultMmapMiB}; 0 = off) + | --loose-compression= zlib level of the loose objects written to , in memory only + | (default ${DefaultLooseCompression}; -1 = JGit's default, 6). git repacks them anyway | --blob-timeout= budget for one run (default ${BlobExec.DefaultTimeoutSeconds}); then its process | tree is killed and the blob excluded | --stall-timeout= exit ${Walker.StalledExitStatus} after this long with no progress (default ${Walker.DefaultStallTimeoutSeconds}); @@ -114,6 +121,7 @@ object Main { alternates: Boolean = false, commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction, sqlite: Mapping.Tuning = Mapping.Tuning(), + looseCompression: Int = DefaultLooseCompression, blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, stallExplicit: Boolean = false, @@ -238,6 +246,7 @@ object Main { private val CommitsPerTxFlag = new ValuedFlag("--commits-per-transaction") private val SqliteCacheFlag = new ValuedFlag("--sqlite-cache-mb") private val SqliteMmapFlag = new ValuedFlag("--sqlite-mmap-mb") + private val LooseCompressionFlag = new ValuedFlag("--loose-compression") private def flagValue[A](flag: ValuedFlag, spec: String)(parse: String => Either[String, A]): Either[String, A] = parse(spec).left.map(Messages.badFlagValue(flag.name, _)) @@ -248,6 +257,9 @@ object Main { private def mebibytes(spec: String): Either[String, Int] = spec.toIntOption.filter(_ >= 0).toRight(s"must be a whole number of MiB, 0 or more [$spec]") + private def level(spec: String): Either[String, Int] = + spec.toIntOption.filter(n => n >= -1 && n <= 9).toRight(s"must be a zlib level, -1 to 9 [$spec]") + private def seconds(spec: String): Either[String, Int] = parsePositiveSeconds(spec).toRight(Messages.secondsWanted(spec)) @@ -294,6 +306,8 @@ object Main { flagValue(SqliteCacheFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(cacheMiB = n))) case SqliteMmapFlag(spec) => flagValue(SqliteMmapFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(mmapMiB = n))) + case LooseCompressionFlag(spec) => + flagValue(LooseCompressionFlag, spec)(level).map(n => o.copy(looseCompression = n)) case StallTimeoutFlag(spec) => flagValue(StallTimeoutFlag, spec)(seconds).map(secs => o.copy(stallTimeoutSeconds = secs, stallExplicit = true)) case other => Left(Messages.unknownFlag(other)) @@ -370,6 +384,7 @@ object Main { val src = openSrc(o.src) val dst = openOrInitDst(o.dst, borrowFrom = Option.when(o.alternates)(src.getObjectsDirectory.toPath)) + setLooseCompression(dst, o.looseCompression) val mapping = openMapping(o, src, dst) match { case Right(m) => m case Left(refusal) => @@ -400,7 +415,8 @@ object Main { s"retokenize=${if (o.retokenize.isEmpty) "none" else o.retokenize.toVector.sorted.mkString(",")} " + s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)} " + s"alternates=${o.alternates} commitsPerTransaction=${o.commitsPerTransaction} " + - s"sqliteCacheMiB=${o.sqlite.cacheMiB} sqliteMmapMiB=${o.sqlite.mmapMiB}" + s"sqliteCacheMiB=${o.sqlite.cacheMiB} sqliteMmapMiB=${o.sqlite.mmapMiB} " + + s"looseCompression=${o.looseCompression}" } private final case class Refusal(status: Int, message: String) @@ -534,6 +550,15 @@ object Main { } } + /** In memory only: dst's config file is not touched, so nothing that reads dst + * later (git repack, steps 3+) can see it. Object ids do not depend on it. JGit + * reads it when an inserter is created; a reload of the file would only bring + * back the default level, a speed loss, never a different object. */ + private[blobexec] def setLooseCompression(dst: FileRepository, level: Int): Unit = { + val config = dst.getConfig // loads the file first, if it changed + config.setInt("core", null, "compression", level) + } + /** One absolute line, replacing whatever was there: a resume writes the same * file again. Written to a temporary file and renamed, so git never reads half. */ private[blobexec] def writeAlternates(file: Path, objects: Path): Unit = { diff --git a/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala b/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala new file mode 100644 index 00000000..955e3d59 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala @@ -0,0 +1,43 @@ +package cregit.blobexec + +import org.eclipse.jgit.lib.{Constants, ObjectId} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ + +/** `--loose-compression`: the zlib level of dst's loose objects, set in memory. */ +class LooseCompressionSpec extends AnyFunSuite with Matchers { + + /** Insert one object and return its loose file's zlib header (CMF, FLG). */ + private def headerAfterInsert(level: Option[Int]): (Path, Int, ObjectId) = { + val dir = Files.createTempDirectory("loose-") + val path = dir.resolve("dst.git") + val dst = Main.openOrInitDst(path) + try { + level.foreach(Main.setLooseCompression(dst, _)) + val ins = dst.newObjectInserter() + val id = try ins.insert(Constants.OBJ_BLOB, ("x" * 4096).getBytes("UTF-8")) finally ins.close() + val file = path.resolve("objects").resolve(id.name.take(2)).resolve(id.name.drop(2)) + val bytes = Files.readAllBytes(file) + (path, bytes(1) & 0xff, id) + } finally dst.close() + } + + test("level 1 writes zlib's fastest level; no setting writes JGit's default") { + // FLG's top two bits are FLEVEL: 0 = fastest (level 1), 2 = default (6). + val (fastRepo, fastFlg, fastId) = headerAfterInsert(Some(1)) + val (defRepo, defFlg, defId) = headerAfterInsert(None) + (fastFlg >> 6) shouldBe 0 + (defFlg >> 6) shouldBe 2 + fastId shouldEqual defId // the id hashes the content, never the deflate stream + Main.DefaultLooseCompression shouldBe 1 + } + + test("the level stays in memory: dst's config file is not changed") { + val (repo, _, _) = headerAfterInsert(Some(1)) + val config = Files.readAllLines(repo.resolve("config")).asScala.mkString("\n") + config should not include "compression" + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala index 9331afae..4583e4aa 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala @@ -220,8 +220,9 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { test("the step-2 speed flags are read, and default to on") { inside(Main.parse(Seq("--alternates", "--commits-per-transaction=1", "--sqlite-cache-mb=0", - "--sqlite-mmap-mb=0") ++ positional)) { case Right(o) => + "--sqlite-mmap-mb=0", "--loose-compression=-1") ++ positional)) { case Right(o) => o.alternates shouldBe true + o.looseCompression shouldBe -1 o.commitsPerTransaction shouldBe 1 o.sqlite shouldEqual Mapping.UntunedSqlite } @@ -229,6 +230,7 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { o.alternates shouldBe false // the caller must repack, so the caller asks for it o.commitsPerTransaction shouldBe Walker.DefaultCommitsPerTransaction o.sqlite shouldEqual Mapping.Tuning(Mapping.DefaultCacheMiB, Mapping.DefaultMmapMiB) + o.looseCompression shouldBe 1 } } @@ -262,7 +264,8 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { "--memo-dir=/nonexistent/" -> "--memo-dir", "--commits-per-transaction=0" -> "--commits-per-transaction", "--sqlite-cache-mb=-1" -> "--sqlite-cache-mb", - "--sqlite-mmap-mb=lots" -> "--sqlite-mmap-mb" + "--sqlite-mmap-mb=lots" -> "--sqlite-mmap-mb", + "--loose-compression=10" -> "--loose-compression" ).foreach { case (flag, name) => withClue(s"$flag: ") { inside(Main.parse(flag +: positional)) { case Left(why) => why should startWith(s"Error: $name") } From d6b168ffadbdfd337f7b216a60410e73853a1cb3 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 17:22:32 +0000 Subject: [PATCH 4/9] perf(blobExec): hash with the JDK's SHA-1 by default JGit's own Java SHA-1 also runs collision detection (SHA1DC) on every object it hashes. The profile measured -5% with the JDK's intrinsic SHA-1 (org.eclipse.jgit.util.sha1.implementation=jdkNative). Main sets that property at start, unless the operator set it. JGit reads it for every new hasher. Both give the same id for every input that is not a crafted SHA-1 collision; on one of those the Java hasher throws, the JDK one does not. A test checks the ids agree, including git's empty blob. Default on; opt out with java -Dorg.eclipse.jgit.util.sha1.implementation=java. --- .../src/main/scala/cregit/blobexec/Main.scala | 15 +++++- .../blobexec/Sha1ImplementationSpec.scala | 46 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index c7baf12a..3a162165 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -355,7 +355,20 @@ object Main { } yield full } + /** JGit hashes with its own Java SHA-1, which also runs collision detection + * (SHA1DC); the JDK's intrinsic one is faster (-5% on psi-probe). Both give the + * same id for every input that is not a crafted SHA-1 collision; on one of + * those the Java one throws instead. Set before JGit hashes anything, and only + * when the operator did not choose: -Dorg.eclipse.jgit.util.sha1.implementation=java + * keeps the old one. JGit reads the property on every new hasher. */ + private[blobexec] val Sha1ImplementationProperty = "org.eclipse.jgit.util.sha1.implementation" + + private[blobexec] def defaultSha1Implementation(): Unit = + if (System.getProperty(Sha1ImplementationProperty) == null) + System.setProperty(Sha1ImplementationProperty, "jdkNative") + def main(args: Array[String]): Unit = { + defaultSha1Implementation() val options = parse(args.toVector) match { case Right(o) => o case Left(why) => exit(why, UsageExitStatus) @@ -416,7 +429,7 @@ object Main { s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)} " + s"alternates=${o.alternates} commitsPerTransaction=${o.commitsPerTransaction} " + s"sqliteCacheMiB=${o.sqlite.cacheMiB} sqliteMmapMiB=${o.sqlite.mmapMiB} " + - s"looseCompression=${o.looseCompression}" + s"looseCompression=${o.looseCompression} sha1=${System.getProperty(Sha1ImplementationProperty)}" } private final case class Refusal(status: Int, message: String) diff --git a/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala b/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala new file mode 100644 index 00000000..66cba157 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala @@ -0,0 +1,46 @@ +package cregit.blobexec + +import org.eclipse.jgit.lib.{Constants, ObjectInserter} +import org.eclipse.jgit.util.sha1.SHA1 +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +/** blobExec hashes with the JDK's SHA-1 unless told otherwise; ids must not change. */ +class Sha1ImplementationSpec extends AnyFunSuite with Matchers { + + private val P = Main.Sha1ImplementationProperty + + private def withProperty[A](value: Option[String])(body: => A): A = { + val saved = Option(System.getProperty(P)) + value match { case Some(v) => System.setProperty(P, v); case None => System.clearProperty(P) } + try body + finally saved match { case Some(v) => System.setProperty(P, v); case None => System.clearProperty(P) } + } + + private def idOf(bytes: Array[Byte]): String = + new ObjectInserter.Formatter().idFor(Constants.OBJ_BLOB, bytes).name + + private val samples: Seq[Array[Byte]] = + Seq("", "a", "int main(){return 0;}\n", "x" * 100000).map(_.getBytes("UTF-8")) :+ + Array.tabulate[Byte](70000)(i => (i * 31 % 251).toByte) + + test("unset, it becomes jdkNative; an operator's choice is kept") { + withProperty(None) { + Main.defaultSha1Implementation() + System.getProperty(P) shouldBe "jdkNative" + SHA1.newInstance().getClass.getSimpleName shouldBe "SHA1Native" + } + withProperty(Some("java")) { + Main.defaultSha1Implementation() + System.getProperty(P) shouldBe "java" + SHA1.newInstance().getClass.getSimpleName shouldBe "SHA1Java" + } + } + + test("both implementations give the same object ids") { + val java = withProperty(Some("java"))(samples.map(idOf)) + val native = withProperty(Some("jdkNative"))(samples.map(idOf)) + native shouldEqual java + java.head shouldBe "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" // git's empty blob + } +} From 0e1e5f57d894f950542468d8854d7ef7ce91f0e5 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 17:23:40 +0000 Subject: [PATCH 5/9] perf(step2): pack the cregit repo without a bitmap index git writes a reachability bitmap by default when it packs a bare repo. It serves fetches and clones over a transport, and nothing does either with the cregit repo: step 6 clones it locally, which hard-links the pack. On Linux the bitmap cost 1.4 h of the 14.4 h gc. Both the gc (--gc plain/aggressive) and the repack that ends borrowing now run with -c repack.writeBitmaps=false. The pack's objects are the same; only the .bitmap file is gone. The shell test checks both paths. Always on: there is nothing to opt out of. --- run_pipeline_process.sh | 7 +++++-- tests/pack_alternates.sh | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 55797d8b..cddf5ab4 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -180,7 +180,7 @@ dissolve_alternates() { fi log "packing $dst with the objects it borrows from $(cat "$alt")" if git --git-dir="$dst" pack-refs --all \ - && git --git-dir="$dst" repack "${repack_args[@]}" \ + && git -c repack.writeBitmaps=false --git-dir="$dst" repack "${repack_args[@]}" \ && mv "$alt" "$alt.packing" \ && git --git-dir="$dst" fsck --connectivity-only --no-dangling; then rm -f "$alt.packing" "${WORK}/PACK-FAILED" @@ -208,6 +208,9 @@ dissolve_alternates() { # Packs the generated bare repo, as --gc selects. A gc failure only warns: under # `set -e` it would fire the EXIT trap, which deletes $WORK and every finished step. +# No bitmap index, here or in dissolve_alternates: git writes one by default for +# a bare repo, it serves only fetches and clones over a transport, and nothing +# does one (step 6 clones locally). It cost 1.4 h of Linux's 14.4 h gc. pack_cregit_repo() { if borrowing_objects; then dissolve_alternates @@ -224,7 +227,7 @@ pack_cregit_repo() { esac # shellcheck disable=SC2086 - if git --git-dir="$REPO_PATH_CREGIT_BARE" gc $gc_args; then + if git -c repack.writeBitmaps=false --git-dir="$REPO_PATH_CREGIT_BARE" gc $gc_args; then log "gc ($GC_MODE) done" else log "warning: gc ($GC_MODE) failed; the object store is still readable" diff --git a/tests/pack_alternates.sh b/tests/pack_alternates.sh index 9a7b1ee0..ad2b54ff 100755 --- a/tests/pack_alternates.sh +++ b/tests/pack_alternates.sh @@ -138,6 +138,7 @@ self_contained "$W/$DST"; check "dst has no alternates file and fsck passes with git --git-dir="$W/$DST" count-objects -v | grep -q '^count: 0' check "every object is packed, none left loose" $? [ ! -e "$W/PACK-FAILED" ]; check "no PACK-FAILED marker" $? +! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null; check "and no bitmap index" $? git clone -q "$W/$DST" "$W/clone" && [ -f "$W/clone/README" ] && [ ! -e "$W/clone/.git/objects/info/alternates" ] check "a step-6 style clone reads every file and borrows nothing" $? rm -rf "$W" @@ -155,6 +156,11 @@ for how in flag env gcnone; do check "$how: blobExec was NOT given --alternates" $? [ ! -e "$W/$DST/objects/info/alternates" ] && git --git-dir="$W/$DST" cat-file -e 'master:README' check "$how: dst holds its objects without borrowing" $? + if [ "$how" = flag ]; then + compgen -G "$W/$DST/objects/pack/*.pack" >/dev/null \ + && ! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null + check "$how: gc packed dst, without a bitmap index" $? + fi rm -rf "$W" done From 1919a3b0066e129e47ddab3860d885b9ac211b54 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 17:25:11 +0000 Subject: [PATCH 6/9] feat(runner): pass blobExec and JVM flags through from the environment The step-2 speed-ups in blobExec are its defaults, and ctp.py has no pass-through for runner flags. So a run that must opt out of one had no way to: the census calls the runner through ctp.py, and only the environment reaches it. CREGIT_BLOBEXEC_OPTS now adds blobExec flags (for example --commits-per-transaction=1 --loose-compression=-1) and CREGIT_BLOBEXEC_JAVA_OPTS adds JVM flags before -jar (for example -Dorg.eclipse.jgit.util.sha1.implementation=java, or -XX:ActiveProcessorCount=3 to size the tokenizer pool). Both are empty by default, so the command line is unchanged. --- run_pipeline_process.sh | 11 ++++++++++- tests/pack_alternates.sh | 12 ++++++++++++ tests/pipeline_workdir_guard.sh | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index cddf5ab4..7beffd8a 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -128,6 +128,12 @@ Tokenizer: after step 2 (git repack -a -d, without -l) copies them in once and deletes that file. Same objects, half the step-2 work. --gc none and --mode sharded always copy. + blobExec's other step-2 speed-ups are its defaults; opt out of + one through env CREGIT_BLOBEXEC_OPTS (blobExec flags, e.g. + "--commits-per-transaction=1 --sqlite-cache-mb=0 + --sqlite-mmap-mb=0 --loose-compression=-1") and + CREGIT_BLOBEXEC_JAVA_OPTS (JVM flags, e.g. + -Dorg.eclipse.jgit.util.sha1.implementation=java). --jobs N concurrent blame/HTML processes (default: CREGIT_JOBS, otherwise min(4, available CPUs)). Blame is the pipeline's bottleneck. Each file is independent, so the output does not @@ -959,7 +965,10 @@ else if [ -n "$RETOKENIZE" ]; then RETOKENIZE_FLAGS=("--retokenize=$RETOKENIZE" "--memo-dir=$MEMO_DIR") fi - java -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} $ALTERNATES_FLAG \ + # Word-split on purpose: each holds several flags. + # shellcheck disable=SC2086 + java ${CREGIT_BLOBEXEC_JAVA_OPTS:-} -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} $ALTERNATES_FLAG \ + ${CREGIT_BLOBEXEC_OPTS:-} \ "--tokenizer-identity=$TOKENIZER_IDENTITY" \ ${RETOKENIZE_FLAGS[@]+"${RETOKENIZE_FLAGS[@]}"} \ ${BLOB_TIMEOUT:+--blob-timeout=$BLOB_TIMEOUT} \ diff --git a/tests/pack_alternates.sh b/tests/pack_alternates.sh index ad2b54ff..b878a1ff 100755 --- a/tests/pack_alternates.sh +++ b/tests/pack_alternates.sh @@ -164,6 +164,18 @@ for how in flag env gcnone; do rm -rf "$W" done +# --------------------------------------------------------------------------- +echo "case 2b: CREGIT_BLOBEXEC_OPTS and CREGIT_BLOBEXEC_JAVA_OPTS reach blobExec" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(CREGIT_BLOBEXEC_OPTS="--commits-per-transaction=1 --loose-compression=-1" \ + CREGIT_BLOBEXEC_JAVA_OPTS="-Dorg.eclipse.jgit.util.sha1.implementation=java -XX:ActiveProcessorCount=3" \ + STUB_ARGV="$ARGV" run_from "$W" 2) +grep blobExec "$ARGV" | grep -q -- '--commits-per-transaction=1 --loose-compression=-1 ' +check "the blobExec flags, before the positional arguments" $? +grep blobExec "$ARGV" | grep -q -- '^-Dorg.eclipse.jgit.util.sha1.implementation=java -XX:ActiveProcessorCount=3 -jar ' +check "the JVM flags, before -jar" $? +rm -rf "$W" + # --------------------------------------------------------------------------- echo "case 3: a failed pack stops the run, resumably, and leaves dst readable" W=$(fixture) diff --git a/tests/pipeline_workdir_guard.sh b/tests/pipeline_workdir_guard.sh index 11229242..b167b59d 100755 --- a/tests/pipeline_workdir_guard.sh +++ b/tests/pipeline_workdir_guard.sh @@ -247,7 +247,7 @@ echo "case 17: without the flag nothing changes, and the flag is what reaches bl grep -q 'WIDENED_FLAG=""' "$RUNNER"; check "the flag defaults to absent" $? grep -q '\[ "\$MASK_WIDENED" = 1 \] && WIDENED_FLAG="--mask-widened"' "$RUNNER" check "and is only added when asked for" $? -grep -q 'java -jar "\$BFG" \$MODE_FLAG \$WIDENED_FLAG' "$RUNNER" +grep -q 'java ${CREGIT_BLOBEXEC_JAVA_OPTS:-} -jar "\$BFG" \$MODE_FLAG \$WIDENED_FLAG' "$RUNNER" check "so it reaches blobExec's argv" $? echo From d06311408e0118579208fb98a664cb63e6628cfb Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 18:11:56 +0000 Subject: [PATCH 7/9] test(step2): cover aggressive gc, a killed walk and a lost group The borrowing path had no test for --gc aggressive, for a walk killed mid-way and resumed at step 2, or for a crash that loses the open transaction group while dst already holds that group's objects. pack_alternates.sh gains two cases: --gc aggressive borrows and its repack leaves dst whole with no bitmap; a SIGKILLed walk leaves dst borrowing, and a step-2 resume packs it and a step-6 clone reads it. AlternatesSpec rolls the DB back to the last committed group after the walk wrote the later commits' objects, drops the refs, resumes, and checks objects, refs and mapping rows against an uninterrupted walk. --- .../cregit/blobexec/AlternatesSpec.scala | 35 +++++++++++++++++++ tests/pack_alternates.sh | 32 +++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala index d175b81e..314a6d36 100644 --- a/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala @@ -184,6 +184,41 @@ class AlternatesSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { for (q <- MappingQueries) table(db, q) shouldBe table(w.resolve("ref.db"), q) } + test("a crash that loses the open group resumes to the same result") { + val w = Files.createTempDirectory(workRoot, "crash-") + val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] + val flag = w.resolve("flag") + val cmd = command(w, flag) + val dst = w.resolve("dst.git") + val db = w.resolve("dst.db") + val snapshot = w.resolve("snapshot.db") + + // The DB as the last committed group left it: c1 and c2. + walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100).aborted shouldBe true + Files.copy(db, snapshot) + + // The walk goes on and writes c3..c5's objects to dst, then dies before + // their group commits: the DB rolls back to the snapshot, and no ref is + // projected yet. That is what a kill or the watchdog's halt leaves. + Files.createFile(flag) + walk(src, dst, db, cmd, borrow = true, commitsPerTransaction = 100).aborted shouldBe false + for (suffix <- Seq("", "-wal", "-shm")) Files.deleteIfExists(w.resolve("dst.db" + suffix)) + Files.copy(snapshot, db) + git(dst, "for-each-ref", "--format=%(refname)")._2.linesIterator.foreach(r => git(dst, "update-ref", "-d", r)._1 shouldBe 0) + table(db, "SELECT count(*) FROM commit_map") shouldBe Vector("2") + + val resumed = walk(src, dst, db, cmd, borrow = true, commitsPerTransaction = 100) + resumed.aborted shouldBe false + resumed.commitsProcessed shouldBe 3 + dissolve(dst) + + val ref = w.resolve("ref.git") + walk(src, ref, w.resolve("ref.db"), cmd, borrow = false, commitsPerTransaction = 1).aborted shouldBe false + reachable(dst) shouldBe reachable(ref) + refs(dst) shouldBe refs(ref) + for (q <- MappingQueries) table(db, q) shouldBe table(w.resolve("ref.db"), q) + } + test("a resume reuses the alternates file, rewriting it rather than appending") { val w = Files.createTempDirectory(workRoot, "rewrite-") val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] diff --git a/tests/pack_alternates.sh b/tests/pack_alternates.sh index b878a1ff..4f7fa051 100755 --- a/tests/pack_alternates.sh +++ b/tests/pack_alternates.sh @@ -20,6 +20,7 @@ check() { # $1 = description, $2 = condition result (0/1) # As blobExec: dst gets one new commit on src's tree. With --alternates it # borrows that tree and its blobs; without, it copies them (git fetch). # STUB_LOCK_PACK=1 then makes objects/pack read-only, so a repack fails. +# STUB_KILL_WALK=1 drops the ref again and SIGKILLs itself, as a killed walk. make_stub_java() { # $1 = bin dir cat > "$1/java" <<'STUB' #!/usr/bin/env bash @@ -50,6 +51,11 @@ git --git-dir="$dst" update-ref refs/heads/master "$c" git --git-dir="$dst" for-each-ref --format='%(refname)' refs/remotes | while read -r r; do git --git-dir="$dst" update-ref -d "$r"; done [ "${STUB_LOCK_PACK:-0}" = 1 ] && chmod a-w "$dst/objects/pack" +# A walk killed mid-way: dst borrows and holds a new object, but no ref yet. +if [ "${STUB_KILL_WALK:-0}" = 1 ]; then + git --git-dir="$dst" update-ref -d refs/heads/master + kill -9 $$ +fi exit 0 STUB chmod +x "$1/java" @@ -215,6 +221,32 @@ OUT=$( cd "$REPO" && STUB_LOCK_PACK=1 PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexi check "the EXIT trap kept \$WORK: the marker says it is resumable (exit $RC)" $? chmod -R u+w "$W" "$S"; rm -rf "$W" "$S" +# --------------------------------------------------------------------------- +echo "case 6: --gc aggressive borrows too, and its repack leaves dst whole" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_ARGV="$ARGV" run_from "$W" --gc aggressive 2) +grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; check "blobExec was given --alternates" $? +grep -q 'no longer borrows any' <<<"$OUT" && grep -q 'Step 3' <<<"$OUT" +check "the runner packed the borrowed objects in and moved on" $? +self_contained "$W/$DST"; check "dst has no alternates file and fsck passes without it" $? +! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null; check "and no bitmap index" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 7: a walk killed mid-way leaves dst borrowing; a step-2 resume finishes it" +W=$(fixture) +OUT=$(STUB_KILL_WALK=1 run_from "$W" 2); RC=$? +[ "$RC" -ne 0 ] && ! grep -q 'Step 3' <<<"$OUT"; check "the run failed in step 2 (exit $RC)" $? +[ -f "$W/$DST/objects/info/alternates" ] && [ -d "$W/proj-original.git" ] +check "dst still borrows, and src is kept" $? +OUT=$(run_from "$W" 2) +grep -q 'no longer borrows any' <<<"$OUT" && grep -q 'Step 3' <<<"$OUT" +check "the resume ran step 2 again, packed, and moved on" $? +self_contained "$W/$DST"; check "dst is whole on its own" $? +git clone -q "$W/$DST" "$W/clone" && [ -f "$W/clone/README" ] && [ ! -e "$W/clone/.git/objects/info/alternates" ] +check "a step-6 style clone reads every file and borrows nothing" $? +rm -rf "$W" + echo "" echo "pack_alternates: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ] From 9dde4ae09e881ca641dacb0c7b9030bc09253f9e Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 18:28:30 +0000 Subject: [PATCH 8/9] fix(runner): split the blobExec pass-through without globbing, and log it CREGIT_BLOBEXEC_OPTS and CREGIT_BLOBEXEC_JAVA_OPTS were expanded unquoted, so a * in a flag could turn into file names from the runner's directory. And a value exported by mistake changed every run with no line in the pipeline log to show it. Both are now read into arrays with read -ra, which splits on blanks and never globs, and each is logged when set. Unset, the command line is the same as before. --- run_pipeline_process.sh | 11 ++++++++--- tests/pack_alternates.sh | 11 +++++++++++ tests/pipeline_workdir_guard.sh | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 7beffd8a..5fe501c6 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -965,10 +965,15 @@ else if [ -n "$RETOKENIZE" ]; then RETOKENIZE_FLAGS=("--retokenize=$RETOKENIZE" "--memo-dir=$MEMO_DIR") fi - # Word-split on purpose: each holds several flags. + # Each holds several flags: split on blanks, but never glob-expanded. Logged, + # because a stray export would change every run without a trace. + read -r -a BLOBEXEC_OPTS <<< "${CREGIT_BLOBEXEC_OPTS:-}" + read -r -a BLOBEXEC_JAVA_OPTS <<< "${CREGIT_BLOBEXEC_JAVA_OPTS:-}" + [ ${#BLOBEXEC_OPTS[@]} -gt 0 ] && log "blobExec flags from CREGIT_BLOBEXEC_OPTS: ${BLOBEXEC_OPTS[*]}" + [ ${#BLOBEXEC_JAVA_OPTS[@]} -gt 0 ] && log "JVM flags from CREGIT_BLOBEXEC_JAVA_OPTS: ${BLOBEXEC_JAVA_OPTS[*]}" # shellcheck disable=SC2086 - java ${CREGIT_BLOBEXEC_JAVA_OPTS:-} -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} $ALTERNATES_FLAG \ - ${CREGIT_BLOBEXEC_OPTS:-} \ + java ${BLOBEXEC_JAVA_OPTS[@]+"${BLOBEXEC_JAVA_OPTS[@]}"} -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} $ALTERNATES_FLAG \ + ${BLOBEXEC_OPTS[@]+"${BLOBEXEC_OPTS[@]}"} \ "--tokenizer-identity=$TOKENIZER_IDENTITY" \ ${RETOKENIZE_FLAGS[@]+"${RETOKENIZE_FLAGS[@]}"} \ ${BLOB_TIMEOUT:+--blob-timeout=$BLOB_TIMEOUT} \ diff --git a/tests/pack_alternates.sh b/tests/pack_alternates.sh index 4f7fa051..d6e347b3 100755 --- a/tests/pack_alternates.sh +++ b/tests/pack_alternates.sh @@ -180,6 +180,17 @@ grep blobExec "$ARGV" | grep -q -- '--commits-per-transaction=1 --loose-compress check "the blobExec flags, before the positional arguments" $? grep blobExec "$ARGV" | grep -q -- '^-Dorg.eclipse.jgit.util.sha1.implementation=java -XX:ActiveProcessorCount=3 -jar ' check "the JVM flags, before -jar" $? +grep -q 'blobExec flags from CREGIT_BLOBEXEC_OPTS: --commits-per-transaction=1 --loose-compression=-1' <<<"$OUT" \ + && grep -q 'JVM flags from CREGIT_BLOBEXEC_JAVA_OPTS: -Dorg' <<<"$OUT" +check "and both are logged" $? +rm -rf "$W" +# The runner runs in $REPO: a file there is what an unquoted --x=* would match. +W=$(fixture); ARGV="$W/argv.log"; touch "$REPO/--x=bait" +OUT=$(CREGIT_BLOBEXEC_OPTS="--x=*" STUB_ARGV="$ARGV" run_from "$W" 2) +rm -f "$REPO/--x=bait" +grep blobExec "$ARGV" | grep -qF -- ' --x=* '; check "a * in the flags is passed as is, not glob-expanded" $? +OUT=$(STUB_ARGV="$W/argv2.log" run_from "$W" 2) +! grep -q 'from CREGIT_BLOBEXEC' <<<"$OUT"; check "unset, nothing is logged" $? rm -rf "$W" # --------------------------------------------------------------------------- diff --git a/tests/pipeline_workdir_guard.sh b/tests/pipeline_workdir_guard.sh index b167b59d..36dccc83 100755 --- a/tests/pipeline_workdir_guard.sh +++ b/tests/pipeline_workdir_guard.sh @@ -247,7 +247,7 @@ echo "case 17: without the flag nothing changes, and the flag is what reaches bl grep -q 'WIDENED_FLAG=""' "$RUNNER"; check "the flag defaults to absent" $? grep -q '\[ "\$MASK_WIDENED" = 1 \] && WIDENED_FLAG="--mask-widened"' "$RUNNER" check "and is only added when asked for" $? -grep -q 'java ${CREGIT_BLOBEXEC_JAVA_OPTS:-} -jar "\$BFG" \$MODE_FLAG \$WIDENED_FLAG' "$RUNNER" +grep -qF 'java ${BLOBEXEC_JAVA_OPTS[@]+"${BLOBEXEC_JAVA_OPTS[@]}"} -jar "$BFG" $MODE_FLAG $WIDENED_FLAG' "$RUNNER" check "so it reaches blobExec's argv" $? echo From 1a392cdcb6984028a9d3285261647361886f8f58 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 21:44:55 +0000 Subject: [PATCH 9/9] feat(runner): let CREGIT_WRITE_BITMAPS=1 bring the bitmap back The pack of the cregit repo writes no bitmap index, and that was the one step-2 speed-up with no opt-out. CREGIT_WRITE_BITMAPS=1 now drops the -c repack.writeBitmaps=false from both the borrowing repack and git gc, so git writes its default again. ctp.py passes the environment through, so the census can use it without a new flag. The README's shell test loop also lists pack_alternates, as CI does. --- README.md | 2 +- run_pipeline_process.sh | 11 +++++++++-- tests/pack_alternates.sh | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ea89528a..91a7a9d2 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ cd ../.. bash tests/test_token_worker.sh prove tests/t pytest -q -for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do +for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough pack_alternates; do bash "tests/$t.sh" done diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 5fe501c6..5b4c5332 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -134,6 +134,8 @@ Tokenizer: --sqlite-mmap-mb=0 --loose-compression=-1") and CREGIT_BLOBEXEC_JAVA_OPTS (JVM flags, e.g. -Dorg.eclipse.jgit.util.sha1.implementation=java). + The pack of the cregit repo writes no bitmap index (nothing + reads one); env CREGIT_WRITE_BITMAPS=1 writes it again. --jobs N concurrent blame/HTML processes (default: CREGIT_JOBS, otherwise min(4, available CPUs)). Blame is the pipeline's bottleneck. Each file is independent, so the output does not @@ -186,7 +188,7 @@ dissolve_alternates() { fi log "packing $dst with the objects it borrows from $(cat "$alt")" if git --git-dir="$dst" pack-refs --all \ - && git -c repack.writeBitmaps=false --git-dir="$dst" repack "${repack_args[@]}" \ + && git ${NO_BITMAP_CFG[@]+"${NO_BITMAP_CFG[@]}"} --git-dir="$dst" repack "${repack_args[@]}" \ && mv "$alt" "$alt.packing" \ && git --git-dir="$dst" fsck --connectivity-only --no-dangling; then rm -f "$alt.packing" "${WORK}/PACK-FAILED" @@ -217,6 +219,7 @@ dissolve_alternates() { # No bitmap index, here or in dissolve_alternates: git writes one by default for # a bare repo, it serves only fetches and clones over a transport, and nothing # does one (step 6 clones locally). It cost 1.4 h of Linux's 14.4 h gc. +# CREGIT_WRITE_BITMAPS=1 gives git its default back. pack_cregit_repo() { if borrowing_objects; then dissolve_alternates @@ -233,7 +236,7 @@ pack_cregit_repo() { esac # shellcheck disable=SC2086 - if git -c repack.writeBitmaps=false --git-dir="$REPO_PATH_CREGIT_BARE" gc $gc_args; then + if git ${NO_BITMAP_CFG[@]+"${NO_BITMAP_CFG[@]}"} --git-dir="$REPO_PATH_CREGIT_BARE" gc $gc_args; then log "gc ($GC_MODE) done" else log "warning: gc ($GC_MODE) failed; the object store is still readable" @@ -299,6 +302,10 @@ STALL_TIMEOUT="${CREGIT_STALL_TIMEOUT:-}" TOKENIZER_WORKER=0 # 1: step 2 copies the original blobs into dst instead of borrowing them. COPY_OBJECTS="${CREGIT_COPY_OBJECTS:-0}" +# The packs of dst get no bitmap index (see pack_cregit_repo) unless +# CREGIT_WRITE_BITMAPS=1 asks for git's default. +NO_BITMAP_CFG=(-c repack.writeBitmaps=false) +if [ "${CREGIT_WRITE_BITMAPS:-0}" = 1 ]; then NO_BITMAP_CFG=(); fi KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES PACK-FAILED" diff --git a/tests/pack_alternates.sh b/tests/pack_alternates.sh index d6e347b3..7ea50f68 100755 --- a/tests/pack_alternates.sh +++ b/tests/pack_alternates.sh @@ -243,6 +243,20 @@ self_contained "$W/$DST"; check "dst has no alternates file and fsck passes with ! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null; check "and no bitmap index" $? rm -rf "$W" +# --------------------------------------------------------------------------- +echo "case 6b: CREGIT_WRITE_BITMAPS=1 brings the bitmap index back, on both paths" +for how in borrow copy; do + W=$(fixture) + case $how in + borrow) OUT=$(CREGIT_WRITE_BITMAPS=1 run_from "$W" 2) ;; + copy) OUT=$(CREGIT_WRITE_BITMAPS=1 run_from "$W" --copy-objects 2) ;; + esac + grep -q 'Step 3' <<<"$OUT" && compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null + check "$how: the pack has a bitmap index" $? + git --git-dir="$W/$DST" cat-file -e 'master:README'; check "$how: dst reads its files" $? + rm -rf "$W" +done + # --------------------------------------------------------------------------- echo "case 7: a walk killed mid-way leaves dst borrowing; a step-2 resume finishes it" W=$(fixture)