diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d0f4b9ee..b6e89015 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -85,7 +85,7 @@ jobs:
- name: Pipeline tests (shell)
run: |
devenv shell -- bash -c 'set -e
- for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do
+ for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough pack_alternates; do
bash "tests/$t.sh"
done'
diff --git a/README.md b/README.md
index ea89528a..91a7a9d2 100644
--- a/README.md
+++ b/README.md
@@ -115,7 +115,7 @@ cd ../..
bash tests/test_token_worker.sh
prove tests/t
pytest -q
-for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do
+for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough pack_alternates; do
bash "tests/$t.sh"
done
diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala
index 8ca84b7e..3a162165 100644
--- a/blobExec/src/main/scala/cregit/blobexec/Main.scala
+++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala
@@ -14,6 +14,11 @@ object Main {
/** Not 3 (memo mismatch): a no-op request must be told apart from a refusal. */
private[blobexec] val RetokenizeIneffectiveExitStatus = 7
+ /** zlib level 1 for dst's loose objects: they are temporary, because the pack
+ * after step 2 re-deflates every one at git's pack.compression. Measured -11%
+ * consumer CPU on psi-probe against JGit's default level. */
+ private[blobexec] val DefaultLooseCompression = 1
+
private val UsageExitStatus = 1
private val RefusedExitStatus = 3
@@ -74,6 +79,16 @@ object Main {
| --tokenizer-identity and --memo-dir; a no-op is refused (status ${RetokenizeIneffectiveExitStatus})
| --memo-dir=
the tokenizer memo ($$BFG_MEMO_DIR) that --retokenize purges
| --tokenizer-worker= tokenize through a pool of persistent workers; needs --pipeline(-trees)
+ | --alternates write /objects/info/alternates naming 's objects
+ | and copy no original blob; the caller must then repack
+ | without -l and delete that file. Not with --shard
+ | --commits-per-transaction=
+ | commits whose mapping rows share one SQLite transaction
+ | (default ${Walker.DefaultCommitsPerTransaction}; 1 = one per commit)
+ | --sqlite-cache-mb= SQLite page cache for the mapping DB (default ${Mapping.DefaultCacheMiB}; 0 = SQLite's 2 MB)
+ | --sqlite-mmap-mb= memory-map up to this much of the mapping DB (default ${Mapping.DefaultMmapMiB}; 0 = off)
+ | --loose-compression= zlib level of the loose objects written to , in memory only
+ | (default ${DefaultLooseCompression}; -1 = JGit's default, 6). git repacks them anyway
| --blob-timeout= budget for one run (default ${BlobExec.DefaultTimeoutSeconds}); then its process
| tree is killed and the blob excluded
| --stall-timeout= exit ${Walker.StalledExitStatus} after this long with no progress (default ${Walker.DefaultStallTimeoutSeconds});
@@ -103,6 +118,10 @@ object Main {
retokenize: Set[String] = Set.empty,
memoDir: Option[Path] = None,
tokenizerWorker: Option[Path] = None,
+ alternates: Boolean = false,
+ commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction,
+ sqlite: Mapping.Tuning = Mapping.Tuning(),
+ looseCompression: Int = DefaultLooseCompression,
blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds,
stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds,
stallExplicit: Boolean = false,
@@ -136,6 +155,8 @@ object Main {
"into, or the invalidation would leave the real memo's stale entries in place."
val PipelinesExclusive = withUsage("Error: --pipeline and --pipeline-trees are mutually exclusive")
+ val AlternatesUnderShard = withUsage(
+ "Error: --alternates cannot be used with --shard: shard_merge.py copies each shard's own objects")
val ShardWithPipeline = withUsage(
"Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees")
val MaskWidenedUnderShard = withUsage(
@@ -222,10 +243,23 @@ object Main {
private val TokenizerWorkerFlag = new ValuedFlag("--tokenizer-worker")
private val BlobTimeoutFlag = new ValuedFlag("--blob-timeout")
private val StallTimeoutFlag = new ValuedFlag("--stall-timeout")
+ private val CommitsPerTxFlag = new ValuedFlag("--commits-per-transaction")
+ private val SqliteCacheFlag = new ValuedFlag("--sqlite-cache-mb")
+ private val SqliteMmapFlag = new ValuedFlag("--sqlite-mmap-mb")
+ private val LooseCompressionFlag = new ValuedFlag("--loose-compression")
private def flagValue[A](flag: ValuedFlag, spec: String)(parse: String => Either[String, A]): Either[String, A] =
parse(spec).left.map(Messages.badFlagValue(flag.name, _))
+ private def positive(spec: String): Either[String, Int] =
+ spec.toIntOption.filter(_ > 0).toRight(s"must be a positive whole number [$spec]")
+
+ private def mebibytes(spec: String): Either[String, Int] =
+ spec.toIntOption.filter(_ >= 0).toRight(s"must be a whole number of MiB, 0 or more [$spec]")
+
+ private def level(spec: String): Either[String, Int] =
+ spec.toIntOption.filter(n => n >= -1 && n <= 9).toRight(s"must be a zlib level, -1 to 9 [$spec]")
+
private def seconds(spec: String): Either[String, Int] =
parsePositiveSeconds(spec).toRight(Messages.secondsWanted(spec))
@@ -249,6 +283,7 @@ object Main {
case "--pipeline" => Right(o.copy(pipeline = true))
case "--pipeline-trees" => Right(o.copy(pipelineTrees = true))
case "--mask-widened" => Right(o.copy(maskWidened = true))
+ case "--alternates" => Right(o.copy(alternates = true))
case TokenizerIdentityFlag(spec) =>
flagValue(TokenizerIdentityFlag, spec)(TokenizerIdentity.parse).map(id => o.copy(tokenizerIdentity = id))
case RetokenizeFlag(spec) =>
@@ -265,6 +300,14 @@ object Main {
Right(o.copy(tokenizerWorker = Some(Paths.get(spec))))
case BlobTimeoutFlag(spec) =>
flagValue(BlobTimeoutFlag, spec)(seconds).map(secs => o.copy(blobTimeoutSeconds = secs))
+ case CommitsPerTxFlag(spec) =>
+ flagValue(CommitsPerTxFlag, spec)(positive).map(n => o.copy(commitsPerTransaction = n))
+ case SqliteCacheFlag(spec) =>
+ flagValue(SqliteCacheFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(cacheMiB = n)))
+ case SqliteMmapFlag(spec) =>
+ flagValue(SqliteMmapFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(mmapMiB = n)))
+ case LooseCompressionFlag(spec) =>
+ flagValue(LooseCompressionFlag, spec)(level).map(n => o.copy(looseCompression = n))
case StallTimeoutFlag(spec) =>
flagValue(StallTimeoutFlag, spec)(seconds).map(secs => o.copy(stallTimeoutSeconds = secs, stallExplicit = true))
case other => Left(Messages.unknownFlag(other))
@@ -285,6 +328,7 @@ object Main {
(retokenizing && o.shard.isDefined) -> (() => Messages.RetokenizeUnderShard),
(!retokenizing && o.memoDir.isDefined) -> (() => Messages.MemoDirWithoutRetokenize),
(o.shard.isDefined && o.anyPipeline) -> (() => Messages.ShardWithPipeline),
+ (o.shard.isDefined && o.alternates) -> (() => Messages.AlternatesUnderShard),
o.tokenizerWorker.exists(p => !Files.isRegularFile(p) || !Files.isExecutable(p)) ->
(() => Messages.workerNotExecutable(o.tokenizerWorker.get)),
(o.tokenizerWorker.isDefined && !o.anyPipeline) -> (() => Messages.WorkerNeedsPipeline)
@@ -311,7 +355,20 @@ object Main {
} yield full
}
+ /** JGit hashes with its own Java SHA-1, which also runs collision detection
+ * (SHA1DC); the JDK's intrinsic one is faster (-5% on psi-probe). Both give the
+ * same id for every input that is not a crafted SHA-1 collision; on one of
+ * those the Java one throws instead. Set before JGit hashes anything, and only
+ * when the operator did not choose: -Dorg.eclipse.jgit.util.sha1.implementation=java
+ * keeps the old one. JGit reads the property on every new hasher. */
+ private[blobexec] val Sha1ImplementationProperty = "org.eclipse.jgit.util.sha1.implementation"
+
+ private[blobexec] def defaultSha1Implementation(): Unit =
+ if (System.getProperty(Sha1ImplementationProperty) == null)
+ System.setProperty(Sha1ImplementationProperty, "jdkNative")
+
def main(args: Array[String]): Unit = {
+ defaultSha1Implementation()
val options = parse(args.toVector) match {
case Right(o) => o
case Left(why) => exit(why, UsageExitStatus)
@@ -339,7 +396,8 @@ object Main {
println(startLine(o, stallSeconds, incremental))
val src = openSrc(o.src)
- val dst = openOrInitDst(o.dst)
+ val dst = openOrInitDst(o.dst, borrowFrom = Option.when(o.alternates)(src.getObjectsDirectory.toPath))
+ setLooseCompression(dst, o.looseCompression)
val mapping = openMapping(o, src, dst) match {
case Right(m) => m
case Left(refusal) =>
@@ -368,13 +426,17 @@ object Main {
s"maskWidened=${o.maskWidened} denylistEntries=${BlobDenylist.shipped.size} " +
s"tokenizerIdentity=${if (o.tokenizerIdentity.isEmpty) "none" else o.tokenizerIdentity.render} " +
s"retokenize=${if (o.retokenize.isEmpty) "none" else o.retokenize.toVector.sorted.mkString(",")} " +
- s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)}"
+ s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)} " +
+ s"alternates=${o.alternates} commitsPerTransaction=${o.commitsPerTransaction} " +
+ s"sqliteCacheMiB=${o.sqlite.cacheMiB} sqliteMmapMiB=${o.sqlite.mmapMiB} " +
+ s"looseCompression=${o.looseCompression} sha1=${System.getProperty(Sha1ImplementationProperty)}"
}
private final case class Refusal(status: Int, message: String)
private def openMapping(o: Options, src: FileRepository, dst: FileRepository): Either[Refusal, Mapping] =
- Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst)))
+ Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst),
+ o.sqlite))
.toEither
.left.map(e => refusalFor(e, o).getOrElse(throw e))
@@ -436,9 +498,11 @@ object Main {
src, dst, mapping, o.mask.r, o.command, o.abortOnError, parallelism,
o.pipeline, o.pipelineTrees, o.shard,
destinationMayContainObjects = incremental,
+ borrowOriginalObjects = o.alternates,
blobTimeoutSeconds = o.blobTimeoutSeconds,
stallTimeoutSeconds = stallSeconds,
workerPool = workerPool,
+ commitsPerTransaction = o.commitsPerTransaction,
denylist = BlobDenylist.shipped
).run()
finally workerPool.foreach(_.close())
@@ -456,6 +520,7 @@ object Main {
s"originalBlobDestinationLookups=${stats.originalBlobDestinationLookups} " +
s"originalBlobBytesCopied=${stats.originalBlobBytesCopied} " +
s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " +
+ s"originalBlobsBorrowed=${stats.originalBlobsBorrowed} " +
s"refsProjected=${stats.refsProjected} " +
s"blobsTimedOut=${stats.blobsTimedOut} " +
s"blobsOversized=${stats.blobsOversized} " +
@@ -480,10 +545,40 @@ object Main {
FileRepositoryBuilder.create(gitDir).asInstanceOf[FileRepository]
}
- private def openOrInitDst(path: Path): FileRepository = {
+ /** `borrowFrom` names src's objects directory: it is written to dst's
+ * objects/info/alternates BEFORE dst is opened for the walk, because JGit reads
+ * that file once, and a walk that skipped a copy must find the object there. */
+ private[blobexec] def openOrInitDst(path: Path, borrowFrom: Option[Path] = None): FileRepository = {
val exists = Files.isDirectory(path)
val repo = FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository]
if (!exists) repo.create(true)
- repo
+ borrowFrom match {
+ case None => repo
+ case Some(objects) =>
+ val info = repo.getObjectsDirectory.toPath.resolve("info")
+ repo.close()
+ Files.createDirectories(info)
+ writeAlternates(info.resolve("alternates"), objects)
+ FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository]
+ }
+ }
+
+ /** In memory only: dst's config file is not touched, so nothing that reads dst
+ * later (git repack, steps 3+) can see it. Object ids do not depend on it. JGit
+ * reads it when an inserter is created; a reload of the file would only bring
+ * back the default level, a speed loss, never a different object. */
+ private[blobexec] def setLooseCompression(dst: FileRepository, level: Int): Unit = {
+ val config = dst.getConfig // loads the file first, if it changed
+ config.setInt("core", null, "compression", level)
+ }
+
+ /** One absolute line, replacing whatever was there: a resume writes the same
+ * file again. Written to a temporary file and renamed, so git never reads half. */
+ private[blobexec] def writeAlternates(file: Path, objects: Path): Unit = {
+ val line = objects.toAbsolutePath.normalize.toString + "\n"
+ val tmp = file.resolveSibling(file.getFileName.toString + ".tmp")
+ Files.write(tmp, line.getBytes(java.nio.charset.StandardCharsets.UTF_8))
+ Files.move(tmp, file, java.nio.file.StandardCopyOption.REPLACE_EXISTING,
+ java.nio.file.StandardCopyOption.ATOMIC_MOVE)
}
}
diff --git a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala
index d6739ae7..c13e57ee 100644
--- a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala
+++ b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala
@@ -296,8 +296,10 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends
Mapping.RetokenizeResult(before, rowCounts, extensions, invalidated, memo)
}
- /** Run `body` inside a transaction; commit on success, rollback on throw. */
+ /** Run `body` inside a transaction; commit on success, rollback on throw.
+ * An open group (see [[inGroupedTx]]) is committed first. */
def inTx[A](body: => A): A = {
+ commitGroup()
conn.setAutoCommit(false)
try {
val result = body
@@ -312,7 +314,67 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends
}
}
+ /** A PRAGMA's current value on this connection, for tests. */
+ private[blobexec] def pragmaValue(name: String): Long = {
+ val st = conn.createStatement()
+ try { val rs = st.executeQuery(s"PRAGMA $name"); rs.next(); rs.getLong(1) } finally st.close()
+ }
+
+ // -- grouped transactions ---------------------------------------------------
+ // A commit's rows are one body; up to `maxBodies` bodies share one transaction.
+ // Each SQLite transaction rewrites the pages it touched into the WAL, and the
+ // walk touches the same index pages for every commit, so per-commit
+ // transactions cost more than the rows. A group is all-or-nothing, like the
+ // one-commit transaction it replaces: a crash loses whole bodies, never part of
+ // one, so a commit row is still never durable without its trees and blobs, and
+ // the lost commits are re-walked on resume (their tokens come from the memo).
+
+ private var groupOpen = false
+ private var groupBodies = 0
+ private var groupStartNanos = 0L
+
+ /** As [[inTx]], but `body` joins the open group, which is committed once it
+ * holds `maxBodies` bodies or is `maxNanos` old. A throw in `body` rolls back
+ * the whole group. `maxBodies` <= 1 is plain [[inTx]]. Not thread-safe: the
+ * caller serializes access, as it does for every other method here. */
+ def inGroupedTx[A](maxBodies: Int, maxNanos: Long)(body: => A): A =
+ if (maxBodies <= 1) inTx(body)
+ else {
+ if (!groupOpen) {
+ conn.setAutoCommit(false)
+ groupOpen = true
+ groupBodies = 0
+ groupStartNanos = System.nanoTime()
+ }
+ val result =
+ try body
+ catch { case t: Throwable => abandonGroup(); throw t }
+ groupBodies += 1
+ if (groupBodies >= maxBodies || System.nanoTime() - groupStartNanos >= maxNanos) commitGroup()
+ result
+ }
+
+ /** Commit the open group, if any. The walk calls it when it stops, by abort too. */
+ def commitGroup(): Unit =
+ if (groupOpen) {
+ try conn.commit()
+ catch { case t: Throwable => abandonGroup(); throw t }
+ groupOpen = false
+ groupBodies = 0
+ conn.setAutoCommit(true)
+ }
+
+ private def abandonGroup(): Unit =
+ if (groupOpen) {
+ groupOpen = false
+ groupBodies = 0
+ try conn.rollback() catch { case _: Throwable => () }
+ try conn.setAutoCommit(true) catch { case _: Throwable => () }
+ }
+
override def close(): Unit = {
+ // Every body in the group finished (a failed one rolled the group back).
+ commitGroup()
(List(selBlob, insBlob, selCommit, insCommit, selTree, insTree,
selRef, insRef, delRef, selMeta, insMeta) ++ warmSelBlob.toList ++ warmSelTree.toList)
.foreach(s => try s.close() catch { case _: Throwable => () })
@@ -456,6 +518,30 @@ object Mapping {
} finally st.close()
}
+ /** Read-path sizing for the walk's DB. Both change speed only, never results.
+ * cacheMiB SQLite's own page cache, private anonymous memory, filled only
+ * as pages are read: the DB's size at most (default 2 MB).
+ * mmapMiB map up to this much of the file. Lookups then read the kernel's
+ * page cache in place instead of copying each page out with a
+ * syscall. Those are file pages: shared and reclaimable, and the
+ * same pages that reads cache anyway. 0 turns it off. */
+ final case class Tuning(cacheMiB: Int = DefaultCacheMiB, mmapMiB: Int = DefaultMmapMiB) {
+ require(cacheMiB >= 0 && mmapMiB >= 0, s"SQLite sizes must be >= 0 (cache=$cacheMiB MiB, mmap=$mmapMiB MiB)")
+ }
+ val DefaultCacheMiB = 256
+ val DefaultMmapMiB = 4096
+ /** SQLite's defaults: what every run used before. */
+ val UntunedSqlite: Tuning = Tuning(cacheMiB = 0, mmapMiB = 0)
+
+ private def applyTuning(conn: Connection, t: Tuning): Unit = {
+ val st = conn.createStatement()
+ try {
+ // Negative cache_size is KiB; 0 keeps the default.
+ if (t.cacheMiB > 0) st.execute(s"PRAGMA cache_size = -${t.cacheMiB.toLong * 1024L}")
+ if (t.mmapMiB > 0) st.execute(s"PRAGMA mmap_size = ${t.mmapMiB.toLong * 1024L * 1024L}")
+ } finally st.close()
+ }
+
/** Opt-in for `--mask-widened`. `newBlobResolves` has no default: `new_blob` ids
* live only in dst, and a wiped dst would dangle every reused row. */
final case class MaskWidening(newBlobResolves: String => Boolean,
@@ -468,7 +554,8 @@ object Mapping {
def open(path: Path, command: String, mask: String, warm: Option[Path] = None,
maskWidening: Option[MaskWidening] = None,
tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty,
- retokenize: Option[Retokenize] = None): Mapping = {
+ retokenize: Option[Retokenize] = None,
+ tuning: Tuning = Tuning()): Mapping = {
// Refused, not composed: each invalidation is verified against a state the
// other would change. Run one, then the other as a resume.
require(!(maskWidening.isDefined && retokenize.isDefined),
@@ -492,6 +579,7 @@ object Mapping {
val url = s"jdbc:sqlite:${path.toAbsolutePath}"
val conn = DriverManager.getConnection(url)
enableForeignKeys(conn)
+ applyTuning(conn, tuning)
val st = conn.createStatement()
try Schema.foreach(st.execute) finally st.close()
diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala
index 5c1ece55..aea1cb94 100644
--- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala
+++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala
@@ -35,7 +35,9 @@ final case class WalkStats(
originalBlobCacheHits: Long,
originalBlobDestinationLookups: Long,
originalBlobBytesCopied: Long,
- originalBlobBytesAvoided: Long
+ originalBlobBytesAvoided: Long,
+ /** Original blobs left in src, which dst reads through objects/info/alternates. */
+ originalBlobsBorrowed: Long = 0L
)
/** Rebuild `src` history into `dst`, persisting mappings to `mapping`.
@@ -53,9 +55,16 @@ final class Walker(
shard: Option[(Int, Int)] = None,
deduplicateOriginalBlobs: Boolean = true,
destinationMayContainObjects: Boolean = true,
+ // dst's objects/info/alternates names src's objects: an original blob is
+ // already readable from dst, so it is never copied. The caller must repack dst
+ // without -l before dropping the alternates file (run_pipeline_process.sh does).
+ borrowOriginalObjects: Boolean = false,
blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds,
stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds,
workerPool: Option[TokenizerWorkerPool] = None,
+ // Commits whose mapping rows share one SQLite transaction (Mapping.inGroupedTx).
+ // 1 is one transaction per commit, as before.
+ commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction,
// A parameter only so tests can supply a fixture.
denylist: BlobDenylist = BlobDenylist.shipped
) {
@@ -66,6 +75,15 @@ final class Walker(
// touch it, so the CPU-heavy path stays lock-free.
private val dbLock = new AnyRef
+ /** On the way out of a walk: the group holds finished commits only (a failed
+ * body rolled it back), so keeping them is right, and a commit error here
+ * must not hide the one already propagating. */
+ private def commitGroupQuietly(): Unit =
+ dbLock.synchronized {
+ try mapping.commitGroup()
+ catch { case scala.util.control.NonFatal(e) => System.err.println(s"warning: mapping commit failed: ${e.getMessage}") }
+ }
+
private val blobCommandExecutions = new LongAdder
private val blobsTimedOut = new LongAdder
private val blobsOversized = new LongAdder
@@ -79,6 +97,7 @@ final class Walker(
private val originalBlobDestinationLookups = new LongAdder
private val originalBlobBytesCopied = new LongAdder
private val originalBlobBytesAvoided = new LongAdder
+ private val originalBlobsBorrowed = new LongAdder
// -- stall watchdog ------------------------------------------------------
// `Await.result` is unbounded: a duration cannot tell a wedged run from a large
@@ -215,7 +234,8 @@ final class Walker(
originalBlobCacheHits = originalBlobCacheHits.sum(),
originalBlobDestinationLookups = originalBlobDestinationLookups.sum(),
originalBlobBytesCopied = originalBlobBytesCopied.sum(),
- originalBlobBytesAvoided = originalBlobBytesAvoided.sum()
+ originalBlobBytesAvoided = originalBlobBytesAvoided.sum(),
+ originalBlobsBorrowed = originalBlobsBorrowed.sum()
)
} finally revWalk.close()
}
@@ -433,7 +453,7 @@ final class Walker(
val newCommit = buildCommit(rc, parents, newTreeId, commitInserter)
- mapping.inTx {
+ mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) {
// Matching blobs the cmd resolved (Replace or Skip outcomes).
resolved.foreach { case ((origSha, path), newId) =>
mapping.putBlob(origSha, path, newId.name)
@@ -456,9 +476,11 @@ final class Walker(
}
}
+ mapping.commitGroup()
treeInserter.flush()
commitInserter.flush()
} finally {
+ commitGroupQuietly()
treeInserter.close()
commitInserter.close()
pool.shutdown()
@@ -560,7 +582,7 @@ final class Walker(
}.map(ObjectId.fromString)
val newCommit = buildCommitFromData(data, parents, newTreeId, commitInserter)
dbLock.synchronized {
- mapping.inTx {
+ mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) {
resolved.foreach { case ((origSha, path), newId) =>
mapping.putBlob(origSha, path, newId.name)
}
@@ -586,9 +608,11 @@ final class Walker(
}
}
+ dbLock.synchronized(mapping.commitGroup())
treeInserter.flush()
commitInserter.flush()
} finally {
+ commitGroupQuietly()
treeInserter.close()
commitInserter.close()
pool.shutdown()
@@ -693,7 +717,7 @@ final class Walker(
}.map(ObjectId.fromString)
val newCommit = buildCommitFromData(data, parents, newTreeId, commitInserter)
dbLock.synchronized {
- mapping.inTx {
+ mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) {
resolved.foreach { case ((origSha, path), newId) =>
mapping.putBlob(origSha, path, newId.name)
}
@@ -717,8 +741,10 @@ final class Walker(
}
}
+ dbLock.synchronized(mapping.commitGroup())
commitInserter.flush()
} finally {
+ commitGroupQuietly()
commitInserter.close()
pool.shutdown()
if (!pool.awaitTermination(1, TimeUnit.MINUTES)) { pool.shutdownNow(); () }
@@ -1139,6 +1165,13 @@ final class Walker(
// Copying a 435 GB repository's unmasked blobs is work, not a stall.
progress(s"original blob ${id.name}")
+ // Borrowed: src holds the blob and dst reads src's objects, so there is
+ // nothing to write. The pass-through copy was half the consumer's time.
+ if (borrowOriginalObjects) {
+ originalBlobsBorrowed.increment()
+ return
+ }
+
if (!deduplicateOriginalBlobs) {
val (_, size) = insertBlob(inserter)
originalBlobCopies.increment()
@@ -1393,6 +1426,13 @@ final class Walker(
}
object Walker {
+ /** Commits per SQLite transaction. A crash or a watchdog halt loses at most this
+ * many commits' rows (or [[TransactionMaxNanos]] of them), and resume re-walks
+ * them. */
+ val DefaultCommitsPerTransaction = 256
+ /** A group is also committed once this old, so a slow stretch keeps its rows. */
+ val TransactionMaxNanos: Long = 2L * 1000L * 1000L * 1000L
+
/** JGit's default `core.streamFileThreshold`, at or above which `getBytes`
* throws. Only a fast path: [[Walker#readBlob]] also catches the exception. */
diff --git a/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala
new file mode 100644
index 00000000..314a6d36
--- /dev/null
+++ b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala
@@ -0,0 +1,235 @@
+package cregit.blobexec
+
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.internal.storage.file.FileRepository
+import org.eclipse.jgit.lib.ObjectId
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
+
+import java.nio.file.{Files, Path}
+import scala.jdk.CollectionConverters._
+import scala.sys.process._
+
+/** `--alternates`: dst borrows src's objects instead of copying the original
+ * blobs, and `git repack -a -d` (no -l) then makes dst whole. The output must be
+ * the same objects, refs and mapping rows as a run that copies. */
+class AlternatesSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll {
+
+ private val workRoot: Path = Files.createTempDirectory("alternates-")
+
+ override def afterAll(): Unit = deleteRecursive(workRoot)
+
+ private def deleteRecursive(p: Path): Unit =
+ if (Files.exists(p)) {
+ if (Files.isDirectory(p)) Files.list(p).iterator().asScala.toList.foreach(deleteRecursive)
+ Files.delete(p)
+ }
+
+ private val Mask = """\.[ch]$"""
+
+ private def commit(git: Git, files: Map[String, String], message: String): Unit = {
+ val wt = git.getRepository.getWorkTree.toPath
+ files.foreach { case (rel, body) =>
+ val p = wt.resolve(rel)
+ Files.createDirectories(p.getParent)
+ Files.writeString(p, body)
+ }
+ files.keys.foreach(f => git.add().addFilepattern(f).call())
+ git.commit().setMessage(message)
+ .setAuthor("Tester", "t@example.org").setCommitter("Tester", "t@example.org").call()
+ ()
+ }
+
+ /** Five commits on master plus a side branch. a.c/b.h are tokenized; the rest
+ * pass through, among them a blob that only an old commit holds. */
+ private def sourceRepo(dir: Path): Git = {
+ Files.createDirectories(dir)
+ val git = Git.init().setDirectory(dir.toFile).setBare(false).call()
+ commit(git, Map("a.c" -> "int a;\n", "README.md" -> "readme 1\n", "docs/x.txt" -> "x\n"), "c1")
+ commit(git, Map("b.h" -> "int b;\n", "README.md" -> "readme 2\n"), "c2")
+ commit(git, Map("a.c" -> "int a = BOOM;\n", "docs/y.txt" -> "y\n"), "c3")
+ git.branchCreate().setName("side").call()
+ commit(git, Map("lib/z.c" -> "int z;\n", "README.md" -> "readme 3\n"), "c4")
+ commit(git, Map("docs/x.txt" -> "x2\n"), "c5")
+ git
+ }
+
+ /** Uppercases its input; fails on a BOOM blob unless `flag` exists. */
+ private def command(dir: Path, flag: Path): String = {
+ val f = dir.resolve("cmd.sh")
+ Files.writeString(f,
+ s"""#!/bin/sh
+ |c=$$(cat)
+ |case "$$c" in *BOOM*) [ -e '$flag' ] || exit 1 ;; esac
+ |printf '%s\\n' "$$c" | tr a-z A-Z
+ |""".stripMargin)
+ f.toFile.setExecutable(true)
+ f.toAbsolutePath.toString
+ }
+
+ private def walk(src: FileRepository, dstPath: Path, db: Path, cmd: String,
+ borrow: Boolean, abortOnError: Boolean = false,
+ commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction): WalkStats = {
+ val incremental = Files.isDirectory(dstPath)
+ val dst = Main.openOrInitDst(dstPath, Option.when(borrow)(src.getObjectsDirectory.toPath))
+ val mapping = Mapping.open(db, cmd, Mask)
+ try new Walker(src, dst, mapping, Mask.r, cmd, abortOnError, 2, pipeline = true,
+ destinationMayContainObjects = incremental, borrowOriginalObjects = borrow,
+ denylist = BlobDenylist.empty, commitsPerTransaction = commitsPerTransaction).run()
+ finally { mapping.close(); dst.close() }
+ }
+
+ private def git(gitDir: Path, args: String*): (Int, String) = {
+ val out = new StringBuilder
+ val rc = Process(Seq("git", "--git-dir", gitDir.toString) ++ args)
+ .!(ProcessLogger(l => out.append(l).append('\n'), _ => ()))
+ (rc, out.toString)
+ }
+
+ private def reachable(gitDir: Path): Vector[String] = {
+ val (rc, out) = git(gitDir, "rev-list", "--objects", "--all")
+ rc shouldBe 0
+ out.linesIterator.map(_.takeWhile(_ != ' ')).toVector.sorted
+ }
+
+ private def refs(gitDir: Path): String = git(gitDir, "for-each-ref", "--format=%(objectname) %(refname)")._2
+
+ private def table(db: Path, sql: String): Vector[String] = {
+ val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db")
+ try {
+ val rs = c.createStatement().executeQuery(sql)
+ val b = Vector.newBuilder[String]
+ while (rs.next()) b += (1 to rs.getMetaData.getColumnCount).map(rs.getString).mkString(" ")
+ b.result()
+ } finally c.close()
+ }
+
+ // processed_at is a wall-clock stamp, so it is left out.
+ private val MappingQueries = Seq(
+ "SELECT orig_commit, new_commit FROM commit_map ORDER BY 1",
+ "SELECT orig_blob, path, new_blob FROM blob_map ORDER BY 1, 2",
+ "SELECT orig_tree, new_tree FROM tree_map ORDER BY 1")
+
+ private def alternates(dst: Path): Path = dst.resolve("objects/info/alternates")
+
+ /** What run_pipeline_process.sh's dissolve_alternates does, minus logging. */
+ private def dissolve(dst: Path): Unit = {
+ git(dst, "repack", "-a", "-d")._1 shouldBe 0
+ Files.delete(alternates(dst))
+ git(dst, "fsck", "--connectivity-only", "--no-dangling")._1 shouldBe 0
+ }
+
+ test("a borrowing walk copies no original blob and, once repacked, equals a copying walk") {
+ val w = Files.createTempDirectory(workRoot, "equal-")
+ val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository]
+ val flag = Files.createFile(w.resolve("flag"))
+ val cmd = command(w, flag)
+
+ val copied = walk(src, w.resolve("copy.git"), w.resolve("copy.db"), cmd, borrow = false)
+ val borrowed = walk(src, w.resolve("borrow.git"), w.resolve("borrow.db"), cmd, borrow = true)
+
+ copied.originalBlobCopies should be > 0L
+ copied.originalBlobsBorrowed shouldBe 0L
+ borrowed.originalBlobCopies shouldBe 0L
+ borrowed.originalBlobsBorrowed shouldBe copied.originalBlobCopyRequests
+ borrowed.commitsProcessed shouldBe 5
+
+ val b = w.resolve("borrow.git")
+ Files.readString(alternates(b)).trim shouldBe src.getObjectsDirectory.toPath.toAbsolutePath.normalize.toString
+
+ // Without the file, dst is missing the borrowed blobs: the pack is not optional.
+ Files.move(alternates(b), b.resolve("objects/info/alternates.aside"))
+ git(b, "fsck", "--connectivity-only", "--no-dangling")._1 should not be 0
+ Files.move(b.resolve("objects/info/alternates.aside"), alternates(b))
+
+ dissolve(b)
+ reachable(b) shouldBe reachable(w.resolve("copy.git"))
+ refs(b) shouldBe refs(w.resolve("copy.git"))
+ for (q <- MappingQueries)
+ table(w.resolve("borrow.db"), q) shouldBe table(w.resolve("copy.db"), q)
+
+ // A non-bare clone, as step 6 makes, reads every file.
+ val clone = w.resolve("clone")
+ Process(Seq("git", "clone", "-q", b.toString, clone.toString)).! shouldBe 0
+ Files.readString(clone.resolve("README.md")) shouldBe "readme 3\n"
+ Files.readString(clone.resolve("a.c")) shouldBe "INT A = BOOM;\n"
+ }
+
+ test("an interrupted borrowing walk resumes to the same result, with grouped transactions") {
+ val w = Files.createTempDirectory(workRoot, "resume-")
+ val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository]
+ val flag = w.resolve("flag")
+ val cmd = command(w, flag)
+ val dst = w.resolve("dst.git")
+ val db = w.resolve("dst.db")
+
+ // c3's BOOM blob aborts the walk: c1 and c2 are done and must be durable even
+ // though they share one still-open transaction group.
+ val first = walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100)
+ first.aborted shouldBe true
+ table(db, "SELECT count(*) FROM commit_map") shouldBe Vector("2")
+
+ Files.createFile(flag)
+ val second = walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100)
+ second.aborted shouldBe false
+ second.commitsProcessed shouldBe 3
+ dissolve(dst)
+
+ // Reference: one uninterrupted copying walk, one commit per transaction.
+ val ref = w.resolve("ref.git")
+ walk(src, ref, w.resolve("ref.db"), cmd, borrow = false, commitsPerTransaction = 1).aborted shouldBe false
+ reachable(dst) shouldBe reachable(ref)
+ refs(dst) shouldBe refs(ref)
+ for (q <- MappingQueries) table(db, q) shouldBe table(w.resolve("ref.db"), q)
+ }
+
+ test("a crash that loses the open group resumes to the same result") {
+ val w = Files.createTempDirectory(workRoot, "crash-")
+ val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository]
+ val flag = w.resolve("flag")
+ val cmd = command(w, flag)
+ val dst = w.resolve("dst.git")
+ val db = w.resolve("dst.db")
+ val snapshot = w.resolve("snapshot.db")
+
+ // The DB as the last committed group left it: c1 and c2.
+ walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100).aborted shouldBe true
+ Files.copy(db, snapshot)
+
+ // The walk goes on and writes c3..c5's objects to dst, then dies before
+ // their group commits: the DB rolls back to the snapshot, and no ref is
+ // projected yet. That is what a kill or the watchdog's halt leaves.
+ Files.createFile(flag)
+ walk(src, dst, db, cmd, borrow = true, commitsPerTransaction = 100).aborted shouldBe false
+ for (suffix <- Seq("", "-wal", "-shm")) Files.deleteIfExists(w.resolve("dst.db" + suffix))
+ Files.copy(snapshot, db)
+ git(dst, "for-each-ref", "--format=%(refname)")._2.linesIterator.foreach(r => git(dst, "update-ref", "-d", r)._1 shouldBe 0)
+ table(db, "SELECT count(*) FROM commit_map") shouldBe Vector("2")
+
+ val resumed = walk(src, dst, db, cmd, borrow = true, commitsPerTransaction = 100)
+ resumed.aborted shouldBe false
+ resumed.commitsProcessed shouldBe 3
+ dissolve(dst)
+
+ val ref = w.resolve("ref.git")
+ walk(src, ref, w.resolve("ref.db"), cmd, borrow = false, commitsPerTransaction = 1).aborted shouldBe false
+ reachable(dst) shouldBe reachable(ref)
+ refs(dst) shouldBe refs(ref)
+ for (q <- MappingQueries) table(db, q) shouldBe table(w.resolve("ref.db"), q)
+ }
+
+ test("a resume reuses the alternates file, rewriting it rather than appending") {
+ val w = Files.createTempDirectory(workRoot, "rewrite-")
+ val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository]
+ val dst = w.resolve("dst.git")
+ Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath)).close()
+ Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath)).close()
+ Files.readAllLines(alternates(dst)).asScala.toList shouldBe
+ List(src.getObjectsDirectory.toPath.toAbsolutePath.normalize.toString)
+ // And dst opened that way really reads src's objects.
+ val r = Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath))
+ try r.getObjectDatabase.has(ObjectId.fromString(src.resolve("HEAD").name)) shouldBe true
+ finally r.close()
+ }
+}
diff --git a/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala b/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala
new file mode 100644
index 00000000..955e3d59
--- /dev/null
+++ b/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala
@@ -0,0 +1,43 @@
+package cregit.blobexec
+
+import org.eclipse.jgit.lib.{Constants, ObjectId}
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
+
+import java.nio.file.{Files, Path}
+import scala.jdk.CollectionConverters._
+
+/** `--loose-compression`: the zlib level of dst's loose objects, set in memory. */
+class LooseCompressionSpec extends AnyFunSuite with Matchers {
+
+ /** Insert one object and return its loose file's zlib header (CMF, FLG). */
+ private def headerAfterInsert(level: Option[Int]): (Path, Int, ObjectId) = {
+ val dir = Files.createTempDirectory("loose-")
+ val path = dir.resolve("dst.git")
+ val dst = Main.openOrInitDst(path)
+ try {
+ level.foreach(Main.setLooseCompression(dst, _))
+ val ins = dst.newObjectInserter()
+ val id = try ins.insert(Constants.OBJ_BLOB, ("x" * 4096).getBytes("UTF-8")) finally ins.close()
+ val file = path.resolve("objects").resolve(id.name.take(2)).resolve(id.name.drop(2))
+ val bytes = Files.readAllBytes(file)
+ (path, bytes(1) & 0xff, id)
+ } finally dst.close()
+ }
+
+ test("level 1 writes zlib's fastest level; no setting writes JGit's default") {
+ // FLG's top two bits are FLEVEL: 0 = fastest (level 1), 2 = default (6).
+ val (fastRepo, fastFlg, fastId) = headerAfterInsert(Some(1))
+ val (defRepo, defFlg, defId) = headerAfterInsert(None)
+ (fastFlg >> 6) shouldBe 0
+ (defFlg >> 6) shouldBe 2
+ fastId shouldEqual defId // the id hashes the content, never the deflate stream
+ Main.DefaultLooseCompression shouldBe 1
+ }
+
+ test("the level stays in memory: dst's config file is not changed") {
+ val (repo, _, _) = headerAfterInsert(Some(1))
+ val config = Files.readAllLines(repo.resolve("config")).asScala.mkString("\n")
+ config should not include "compression"
+ }
+}
diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala
index d5637fb2..4583e4aa 100644
--- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala
+++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala
@@ -218,6 +218,28 @@ class MainOptionsSpec extends AnyFunSuite with Matchers {
inside(Main.parse("--shard=1/4" +: positional)) { case Right(o) => o.shard shouldEqual Some((1, 4)) }
}
+ test("the step-2 speed flags are read, and default to on") {
+ inside(Main.parse(Seq("--alternates", "--commits-per-transaction=1", "--sqlite-cache-mb=0",
+ "--sqlite-mmap-mb=0", "--loose-compression=-1") ++ positional)) { case Right(o) =>
+ o.alternates shouldBe true
+ o.looseCompression shouldBe -1
+ o.commitsPerTransaction shouldBe 1
+ o.sqlite shouldEqual Mapping.UntunedSqlite
+ }
+ inside(Main.parse(positional)) { case Right(o) =>
+ o.alternates shouldBe false // the caller must repack, so the caller asks for it
+ o.commitsPerTransaction shouldBe Walker.DefaultCommitsPerTransaction
+ o.sqlite shouldEqual Mapping.Tuning(Mapping.DefaultCacheMiB, Mapping.DefaultMmapMiB)
+ o.looseCompression shouldBe 1
+ }
+ }
+
+ test("--alternates is refused with --shard") {
+ inside(Main.parse(Seq("--alternates", "--shard=0/2") ++ positional)) { case Left(why) =>
+ why should include("--alternates cannot be used with --shard")
+ }
+ }
+
test("parse leaves the defaults when no flag is given") {
inside(Main.parse(positional)) { case Right(o) =>
o shouldEqual Main.Options(positional = positional.toVector)
@@ -239,7 +261,11 @@ class MainOptionsSpec extends AnyFunSuite with Matchers {
"--shard=1" -> "--shard",
"--warm=/nonexistent/db" -> "--warm",
"--tokenizer-identity=" -> "--tokenizer-identity",
- "--memo-dir=/nonexistent/" -> "--memo-dir"
+ "--memo-dir=/nonexistent/" -> "--memo-dir",
+ "--commits-per-transaction=0" -> "--commits-per-transaction",
+ "--sqlite-cache-mb=-1" -> "--sqlite-cache-mb",
+ "--sqlite-mmap-mb=lots" -> "--sqlite-mmap-mb",
+ "--loose-compression=10" -> "--loose-compression"
).foreach { case (flag, name) =>
withClue(s"$flag: ") {
inside(Main.parse(flag +: positional)) { case Left(why) => why should startWith(s"Error: $name") }
diff --git a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala
index 46af5926..1d4a1c15 100644
--- a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala
+++ b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala
@@ -441,4 +441,106 @@ class MappingSpec extends AnyFunSuite with Matchers {
m.allCommitOrigShas.toSet shouldEqual Set("a", "b", "c")
}
}
+
+ // -- grouped transactions ----------------------------------------------------
+
+ /** What a second connection (another process, or the DB after a crash) sees. */
+ private def committedCommits(db: java.nio.file.Path): Int = {
+ val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db")
+ try {
+ val rs = c.createStatement().executeQuery("SELECT count(*) FROM commit_map")
+ rs.next(); rs.getInt(1)
+ } finally c.close()
+ }
+
+ private val Forever = Long.MaxValue
+
+ test("a group commits once it holds maxBodies bodies, and not before") {
+ val db = Files.createTempFile("mapping-group-", ".db")
+ val m = Mapping.open(db, "/bin/cat", ".*")
+ try {
+ m.inGroupedTx(3, Forever)(m.putCommit("c1", "n1"))
+ m.inGroupedTx(3, Forever)(m.putCommit("c2", "n2"))
+ committedCommits(db) shouldBe 0
+ m.getCommit("c2") shouldBe Some("n2") // the writer reads its own open group
+ m.inGroupedTx(3, Forever)(m.putCommit("c3", "n3"))
+ committedCommits(db) shouldBe 3
+ } finally m.close()
+ }
+
+ test("a group also commits once it is maxNanos old") {
+ val db = Files.createTempFile("mapping-group-age-", ".db")
+ val m = Mapping.open(db, "/bin/cat", ".*")
+ try {
+ m.inGroupedTx(1000, 0L)(m.putCommit("c1", "n1"))
+ committedCommits(db) shouldBe 1
+ } finally m.close()
+ }
+
+ test("a body that throws rolls back its whole group and nothing committed before") {
+ val db = Files.createTempFile("mapping-group-throw-", ".db")
+ val m = Mapping.open(db, "/bin/cat", ".*")
+ try {
+ m.inGroupedTx(2, Forever)(m.putCommit("c1", "n1"))
+ m.inGroupedTx(2, Forever)(m.putCommit("c2", "n2")) // group of 2 committed
+ m.inGroupedTx(2, Forever)(m.putCommit("c3", "n3")) // open group
+ an[IllegalStateException] should be thrownBy
+ m.inGroupedTx(2, Forever) { m.putTree("t4", "n4"); throw new IllegalStateException("boom") }
+ m.getCommit("c3") shouldBe None
+ m.getTree("t4") shouldBe None
+ m.getCommit("c2") shouldBe Some("n2")
+ // And the connection is usable afterwards.
+ m.inGroupedTx(2, Forever)(m.putCommit("c5", "n5"))
+ m.commitGroup()
+ committedCommits(db) shouldBe 3
+ } finally m.close()
+ }
+
+ test("inTx and close commit an open group first") {
+ val db = Files.createTempFile("mapping-group-close-", ".db")
+ val m = Mapping.open(db, "/bin/cat", ".*")
+ m.inGroupedTx(10, Forever)(m.putCommit("c1", "n1"))
+ m.inTx(m.setMeta("k", "v"))
+ committedCommits(db) shouldBe 1
+ m.inGroupedTx(10, Forever)(m.putCommit("c2", "n2"))
+ m.close()
+ committedCommits(db) shouldBe 2
+ }
+
+ test("a lost open group is all-or-nothing: no commit row without its trees and blobs") {
+ // A crash is a connection that never commits: simulate it by abandoning the
+ // process's view (a second connection reads only what was committed).
+ val db = Files.createTempFile("mapping-group-crash-", ".db")
+ val m = Mapping.open(db, "/bin/cat", ".*")
+ try {
+ m.inGroupedTx(5, Forever) { m.putBlob("b1", "a.c", "nb1"); m.putTree("t1", "nt1"); m.putCommit("c1", "n1") }
+ committedCommits(db) shouldBe 0
+ val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db")
+ try {
+ val rs = c.createStatement().executeQuery(
+ "SELECT (SELECT count(*) FROM blob_map) + (SELECT count(*) FROM tree_map)")
+ rs.next(); rs.getInt(1) shouldBe 0
+ } finally c.close()
+ } finally m.close()
+ }
+
+ test("maxBodies of 1 is a transaction per body, as before") {
+ val db = Files.createTempFile("mapping-group-one-", ".db")
+ val m = Mapping.open(db, "/bin/cat", ".*")
+ try {
+ m.inGroupedTx(1, Forever)(m.putCommit("c1", "n1"))
+ committedCommits(db) shouldBe 1
+ } finally m.close()
+ }
+
+ test("the SQLite sizes reach the connection") {
+ val db = Files.createTempFile("mapping-tuning-", ".db")
+ val m = Mapping.open(db, "/bin/cat", ".*", tuning = Mapping.Tuning(cacheMiB = 64, mmapMiB = 128))
+ try {
+ m.pragmaValue("cache_size") shouldBe -64L * 1024L
+ m.pragmaValue("mmap_size") shouldBe 128L * 1024L * 1024L
+ } finally m.close()
+ val u = Mapping.open(db, "/bin/cat", ".*", tuning = Mapping.UntunedSqlite)
+ try u.pragmaValue("mmap_size") shouldBe 0L finally u.close()
+ }
}
diff --git a/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala b/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala
new file mode 100644
index 00000000..66cba157
--- /dev/null
+++ b/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala
@@ -0,0 +1,46 @@
+package cregit.blobexec
+
+import org.eclipse.jgit.lib.{Constants, ObjectInserter}
+import org.eclipse.jgit.util.sha1.SHA1
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
+
+/** blobExec hashes with the JDK's SHA-1 unless told otherwise; ids must not change. */
+class Sha1ImplementationSpec extends AnyFunSuite with Matchers {
+
+ private val P = Main.Sha1ImplementationProperty
+
+ private def withProperty[A](value: Option[String])(body: => A): A = {
+ val saved = Option(System.getProperty(P))
+ value match { case Some(v) => System.setProperty(P, v); case None => System.clearProperty(P) }
+ try body
+ finally saved match { case Some(v) => System.setProperty(P, v); case None => System.clearProperty(P) }
+ }
+
+ private def idOf(bytes: Array[Byte]): String =
+ new ObjectInserter.Formatter().idFor(Constants.OBJ_BLOB, bytes).name
+
+ private val samples: Seq[Array[Byte]] =
+ Seq("", "a", "int main(){return 0;}\n", "x" * 100000).map(_.getBytes("UTF-8")) :+
+ Array.tabulate[Byte](70000)(i => (i * 31 % 251).toByte)
+
+ test("unset, it becomes jdkNative; an operator's choice is kept") {
+ withProperty(None) {
+ Main.defaultSha1Implementation()
+ System.getProperty(P) shouldBe "jdkNative"
+ SHA1.newInstance().getClass.getSimpleName shouldBe "SHA1Native"
+ }
+ withProperty(Some("java")) {
+ Main.defaultSha1Implementation()
+ System.getProperty(P) shouldBe "java"
+ SHA1.newInstance().getClass.getSimpleName shouldBe "SHA1Java"
+ }
+ }
+
+ test("both implementations give the same object ids") {
+ val java = withProperty(Some("java"))(samples.map(idOf))
+ val native = withProperty(Some("jdkNative"))(samples.map(idOf))
+ native shouldEqual java
+ java.head shouldBe "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" // git's empty blob
+ }
+}
diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh
index 7ac200e4..5b4c5332 100755
--- a/run_pipeline_process.sh
+++ b/run_pipeline_process.sh
@@ -85,6 +85,9 @@ Output:
with millions of loose objects.
A failed pack never aborts the run: the Parquet dataset is
the product, and packing only makes later steps faster.
+ Except while the cregit repo borrows objects (see
+ --copy-objects): that pack is required, and a failed one
+ stops the run, resumably, with a PACK-FAILED marker.
--memory-limit SIZE
cap the DuckDB heap in the dataset generator (step 10).
Omit to accept that script's own default of 8GB. Takes an
@@ -117,6 +120,22 @@ Tokenizer:
--tokenizer-worker
tokenize through a pool of persistent tokenizer processes; same
output. Needs --mode pipeline or pipeline-trees (default: off)
+ --copy-objects
+ copy every untokenized original blob into the cregit repo during
+ step 2, as before (env CREGIT_COPY_OBJECTS=1 does the same).
+ By default step 2 copies none: the cregit repo reads them from
+ the original repo through objects/info/alternates, and the pack
+ after step 2 (git repack -a -d, without -l) copies them in once
+ and deletes that file. Same objects, half the step-2 work.
+ --gc none and --mode sharded always copy.
+ blobExec's other step-2 speed-ups are its defaults; opt out of
+ one through env CREGIT_BLOBEXEC_OPTS (blobExec flags, e.g.
+ "--commits-per-transaction=1 --sqlite-cache-mb=0
+ --sqlite-mmap-mb=0 --loose-compression=-1") and
+ CREGIT_BLOBEXEC_JAVA_OPTS (JVM flags, e.g.
+ -Dorg.eclipse.jgit.util.sha1.implementation=java).
+ The pack of the cregit repo writes no bitmap index (nothing
+ reads one); env CREGIT_WRITE_BITMAPS=1 writes it again.
--jobs N concurrent blame/HTML processes (default: CREGIT_JOBS,
otherwise min(4, available CPUs)). Blame is the pipeline's
bottleneck. Each file is independent, so the output does not
@@ -144,9 +163,68 @@ log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
}
+# True while dst reads objects from src (blobExec --alternates). `.packing` is
+# the file set aside while fsck proves dst complete without it.
+borrowing_objects() {
+ [ -e "$REPO_PATH_CREGIT_BARE/objects/info/alternates" ] \
+ || [ -e "$REPO_PATH_CREGIT_BARE/objects/info/alternates.packing" ]
+}
+
+# Copies the borrowed objects into dst and stops the borrowing. Not optional, as
+# packing otherwise is: until this succeeds dst is unreadable without src.
+# repack -a -d one pack of every reachable object, the borrowed ones too:
+# WITHOUT -l, which would leave them out
+# fsck run with the alternates file set aside, so it proves that dst
+# is complete on its own; the file is deleted only after that
+# A failure puts the file back (dst readable again), writes PACK-FAILED and stops
+# the run; any resume retries this first.
+dissolve_alternates() {
+ local dst="$REPO_PATH_CREGIT_BARE" alt="$REPO_PATH_CREGIT_BARE/objects/info/alternates"
+ local repack_args=(-a -d)
+ [ "$GC_MODE" = aggressive ] && repack_args+=(-f --depth=50 --window=250)
+ # A run killed during fsck left the file set aside: put it back first.
+ if [ -e "$alt.packing" ] && [ ! -e "$alt" ]; then
+ mv "$alt.packing" "$alt"
+ fi
+ log "packing $dst with the objects it borrows from $(cat "$alt")"
+ if git --git-dir="$dst" pack-refs --all \
+ && git ${NO_BITMAP_CFG[@]+"${NO_BITMAP_CFG[@]}"} --git-dir="$dst" repack "${repack_args[@]}" \
+ && mv "$alt" "$alt.packing" \
+ && git --git-dir="$dst" fsck --connectivity-only --no-dangling; then
+ rm -f "$alt.packing" "${WORK}/PACK-FAILED"
+ # What gc does after its repack; neither one is needed for correctness.
+ git --git-dir="$dst" prune --expire=now \
+ || log "warning: git prune failed; unreachable loose objects stay"
+ git --git-dir="$dst" commit-graph write --reachable \
+ || log "warning: commit-graph write failed; later steps walk history more slowly"
+ log "pack done: $dst holds all its objects and no longer borrows any"
+ return 0
+ fi
+ if [ -e "$alt.packing" ] && [ ! -e "$alt" ]; then
+ mv "$alt.packing" "$alt"
+ fi
+ local summary="the pack that copies the borrowed original blobs into $dst failed.
+ $dst still reads them from $REPO_PATH_ORIGINAL_BARE (objects/info/alternates),
+ so it is readable, but it must not outlive that repo. Free memory or disk."
+ write_resume_marker "PACK-FAILED" "git repack -a -d" 1 "$summary"
+ die "$summary
+ Marker: ${WORK}/PACK-FAILED
+ Resume at step 3: the runner retries the pack before anything else.
+ run_pipeline_process.sh: $0 [same flags] 3
+ ctp.py (the external driver): python3 ctp.py run [same flags] --from-step 3"
+}
+
# Packs the generated bare repo, as --gc selects. A gc failure only warns: under
# `set -e` it would fire the EXIT trap, which deletes $WORK and every finished step.
+# No bitmap index, here or in dissolve_alternates: git writes one by default for
+# a bare repo, it serves only fetches and clones over a transport, and nothing
+# does one (step 6 clones locally). It cost 1.4 h of Linux's 14.4 h gc.
+# CREGIT_WRITE_BITMAPS=1 gives git its default back.
pack_cregit_repo() {
+ if borrowing_objects; then
+ dissolve_alternates
+ return 0
+ fi
case "$GC_MODE" in
none)
log "gc skipped (--gc none)"
@@ -158,7 +236,7 @@ pack_cregit_repo() {
esac
# shellcheck disable=SC2086
- if git --git-dir="$REPO_PATH_CREGIT_BARE" gc $gc_args; then
+ if git ${NO_BITMAP_CFG[@]+"${NO_BITMAP_CFG[@]}"} --git-dir="$REPO_PATH_CREGIT_BARE" gc $gc_args; then
log "gc ($GC_MODE) done"
else
log "warning: gc ($GC_MODE) failed; the object store is still readable"
@@ -222,8 +300,14 @@ RETOKENIZE=""
BLOB_TIMEOUT="${CREGIT_BLOB_TIMEOUT:-}"
STALL_TIMEOUT="${CREGIT_STALL_TIMEOUT:-}"
TOKENIZER_WORKER=0
+# 1: step 2 copies the original blobs into dst instead of borrowing them.
+COPY_OBJECTS="${CREGIT_COPY_OBJECTS:-0}"
+# The packs of dst get no bitmap index (see pack_cregit_repo) unless
+# CREGIT_WRITE_BITMAPS=1 asks for git's default.
+NO_BITMAP_CFG=(-c repack.writeBitmaps=false)
+if [ "${CREGIT_WRITE_BITMAPS:-0}" = 1 ]; then NO_BITMAP_CFG=(); fi
-KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES"
+KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES PACK-FAILED"
keep_markers_present() {
local m
@@ -384,6 +468,7 @@ while [ $# -gt 0 ]; do
--retokenize) need_val "$@"; RETOKENIZE="$2"; shift 2 ;;
--blob-timeout) need_val "$@"; BLOB_TIMEOUT="$2"; shift 2 ;;
--tokenizer-worker) TOKENIZER_WORKER=1; shift ;;
+ --copy-objects) COPY_OBJECTS=1; shift ;;
--stall-timeout) need_val "$@"; STALL_TIMEOUT="$2"; shift 2 ;;
--gc) need_val "$@"; GC_MODE="$2"; shift 2 ;;
--memory-limit) need_val "$@"; MEMORY_LIMIT="$2"; shift 2 ;;
@@ -877,12 +962,25 @@ else
[ "$MASK_WIDENED" = 1 ] && WIDENED_FLAG="--mask-widened"
WORKER_FLAG=""
[ "$TOKENIZER_WORKER" = 1 ] && WORKER_FLAG="--tokenizer-worker=${CREGIT}/tokenizeByBlobId/tokenWorker.pl"
+ # Borrowing needs the pack that dissolves it, so --gc none copies instead.
+ ALTERNATES_FLAG=""
+ if [ "$COPY_OBJECTS" != 1 ] && [ "$GC_MODE" != none ]; then
+ ALTERNATES_FLAG="--alternates"
+ fi
# blobExec refuses --retokenize without --memo-dir: the memo caches the same tokens.
RETOKENIZE_FLAGS=()
if [ -n "$RETOKENIZE" ]; then
RETOKENIZE_FLAGS=("--retokenize=$RETOKENIZE" "--memo-dir=$MEMO_DIR")
fi
- java -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} \
+ # Each holds several flags: split on blanks, but never glob-expanded. Logged,
+ # because a stray export would change every run without a trace.
+ read -r -a BLOBEXEC_OPTS <<< "${CREGIT_BLOBEXEC_OPTS:-}"
+ read -r -a BLOBEXEC_JAVA_OPTS <<< "${CREGIT_BLOBEXEC_JAVA_OPTS:-}"
+ [ ${#BLOBEXEC_OPTS[@]} -gt 0 ] && log "blobExec flags from CREGIT_BLOBEXEC_OPTS: ${BLOBEXEC_OPTS[*]}"
+ [ ${#BLOBEXEC_JAVA_OPTS[@]} -gt 0 ] && log "JVM flags from CREGIT_BLOBEXEC_JAVA_OPTS: ${BLOBEXEC_JAVA_OPTS[*]}"
+ # shellcheck disable=SC2086
+ java ${BLOBEXEC_JAVA_OPTS[@]+"${BLOBEXEC_JAVA_OPTS[@]}"} -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} $ALTERNATES_FLAG \
+ ${BLOBEXEC_OPTS[@]+"${BLOBEXEC_OPTS[@]}"} \
"--tokenizer-identity=$TOKENIZER_IDENTITY" \
${RETOKENIZE_FLAGS[@]+"${RETOKENIZE_FLAGS[@]}"} \
${BLOB_TIMEOUT:+--blob-timeout=$BLOB_TIMEOUT} \
@@ -915,6 +1013,13 @@ pack_cregit_repo
fi
end_step
+# A resume past step 2 can find dst still borrowing: the pack failed or was
+# killed. Steps 3+ must not start on that, so the pack runs first.
+if [ "$FROM_STEP" -gt 2 ] && [ -d "$REPO_PATH_CREGIT_BARE" ] && borrowing_objects; then
+ log "resume: $REPO_PATH_CREGIT_BARE still borrows objects; packing it before step $FROM_STEP"
+ dissolve_alternates
+fi
+
# ---------------------------------------------------------------------------
# Step 3 — git log DB (original repo)
# ---------------------------------------------------------------------------
diff --git a/tests/pack_alternates.sh b/tests/pack_alternates.sh
new file mode 100755
index 00000000..7ea50f68
--- /dev/null
+++ b/tests/pack_alternates.sh
@@ -0,0 +1,277 @@
+#!/usr/bin/env bash
+# Tests step 2's borrowed objects: blobExec --alternates by default, the pack that
+# must copy them into the cregit repo, and what happens when that pack fails.
+# java is stubbed: as blobExec it builds a real dst that borrows src's objects.
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+RUNNER="$ROOT/run_pipeline_process.sh"
+PASS=0
+FAIL=0
+
+check() { # $1 = description, $2 = condition result (0/1)
+ if [ "$2" -eq 0 ]; then
+ echo " ok — $1"; PASS=$((PASS + 1))
+ else
+ echo " FAIL — $1"; FAIL=$((FAIL + 1))
+ fi
+}
+
+# As blobExec: dst gets one new commit on src's tree. With --alternates it
+# borrows that tree and its blobs; without, it copies them (git fetch).
+# STUB_LOCK_PACK=1 then makes objects/pack read-only, so a repack fails.
+# STUB_KILL_WALK=1 drops the ref again and SIGKILLs itself, as a killed walk.
+make_stub_java() { # $1 = bin dir
+ cat > "$1/java" <<'STUB'
+#!/usr/bin/env bash
+[ -n "${STUB_ARGV:-}" ] && printf '%s\n' "$*" >> "$STUB_ARGV"
+case "$*" in *blobExec*) ;; *) exit 0 ;; esac
+pos=(); skip=0; alt=0
+for a in "$@"; do
+ if [ "$skip" = 1 ]; then skip=0; continue; fi
+ case "$a" in
+ -jar) skip=1 ;;
+ --alternates) alt=1 ;;
+ -*) ;;
+ *) pos+=("$a") ;;
+ esac
+done
+src=${pos[0]} dst=${pos[1]}
+[ -d "$dst" ] || git init -q --bare "$dst"
+if [ "$alt" = 1 ]; then
+ echo "$(cd "$src/objects" && pwd)" > "$dst/objects/info/alternates"
+else
+ git --git-dir="$dst" fetch -q "$src" "+refs/heads/*:refs/remotes/src/*"
+fi
+tree=$(git --git-dir="$src" rev-parse 'HEAD^{tree}')
+c=$(echo rewritten | GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@x GIT_COMMITTER_NAME=t \
+ GIT_COMMITTER_EMAIL=t@x GIT_AUTHOR_DATE='@1700000000 +0000' GIT_COMMITTER_DATE='@1700000000 +0000' \
+ git --git-dir="$dst" commit-tree "$tree")
+git --git-dir="$dst" update-ref refs/heads/master "$c"
+git --git-dir="$dst" for-each-ref --format='%(refname)' refs/remotes | while read -r r; do
+ git --git-dir="$dst" update-ref -d "$r"; done
+[ "${STUB_LOCK_PACK:-0}" = 1 ] && chmod a-w "$dst/objects/pack"
+# A walk killed mid-way: dst borrows and holds a new object, but no ref yet.
+if [ "${STUB_KILL_WALK:-0}" = 1 ]; then
+ git --git-dir="$dst" update-ref -d refs/heads/master
+ kill -9 $$
+fi
+exit 0
+STUB
+ chmod +x "$1/java"
+}
+
+# The runner takes CREGIT from $(pwd): tokenize/ for the identity, plus build
+# files and placeholder artifacts, or the staleness guard would build.
+repo_fixture() {
+ local f p
+ f=$(mktemp -d "${TMPDIR:-/tmp}/packrepo-XXXXXX")
+ cp "$RUNNER" "$f/run_pipeline_process.sh"
+ cp -r "$ROOT/tokenize" "$f/tokenize"
+ rm -rf "$f/tokenize/rustTokenizer/target" "$f/tokenize/srcMLtoken/srcml2token"
+ for p in blobExec slickGitLog persons remapCommits; do
+ mkdir -p "$f/$p/src"
+ cp "$ROOT/$p/build.sbt" "$f/$p/build.sbt"
+ cp -r "$ROOT/$p/project" "$f/$p/project"
+ rm -rf "$f/$p/project/target" "$f/$p/project/project"
+ done
+ for p in tokenize/srcMLtoken/srcml2token \
+ tokenize/rustTokenizer/target/release/rust_tokenizer \
+ blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar \
+ slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar \
+ persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar \
+ remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar
+ do
+ mkdir -p "$f/$(dirname "$p")"
+ echo "placeholder v1" > "$f/$p"
+ chmod +x "$f/$p"
+ done
+ printf '#!/bin/sh\necho "%s/libsrcml"\n' "$f" > "$f/tokenize/srcMLtoken/srcml2token"
+ echo "placeholder v1" > "$f/libsrcml"
+ printf '%s' "$f"
+}
+
+# A work dir resumable at step 2: a real original bare repo with two commits.
+fixture() {
+ local w t
+ w=$(mktemp -d "${TMPDIR:-/tmp}/packwork-XXXXXX")
+ t="$w/tmp-src"
+ git init -q "$t"
+ echo "int a;" > "$t/a.c"; echo "readme" > "$t/README"
+ git -C "$t" add . && git -C "$t" -c user.name=t -c user.email=t@x commit -q -m one
+ echo "readme 2" > "$t/README"
+ git -C "$t" -c user.name=t -c user.email=t@x commit -q -am two
+ git clone -q --bare "$t" "$w/proj-original.git"
+ rm -rf "$t"
+ mkdir -p "$w/memo"
+ printf '%s' "$w"
+}
+
+run_from() { # $1 = work dir, rest = extra runner args + FROM_STEP
+ local w=$1; shift
+ ( cd "$REPO" \
+ && PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 timeout 300 \
+ bash ./run_pipeline_process.sh \
+ --repo-url /nonexistent/does-not-exist.git \
+ --repo-name proj \
+ --work "$w" \
+ --skip-html --mask '\.c$' \
+ "$@" 2>&1 )
+}
+
+DST=proj-cregit.git
+
+# True when dst holds every object it references, without any alternates file.
+self_contained() { # $1 = dst
+ [ ! -e "$1/objects/info/alternates" ] && [ ! -e "$1/objects/info/alternates.packing" ] \
+ && git --git-dir="$1" fsck --connectivity-only --no-dangling >/dev/null 2>&1 \
+ && git --git-dir="$1" cat-file -e 'master:README'
+}
+
+BIN=$(mktemp -d "${TMPDIR:-/tmp}/packbin-XXXXXX")
+make_stub_java "$BIN"
+REPO=$(repo_fixture)
+trap 'rm -rf "$BIN" "$REPO"' EXIT
+
+# ---------------------------------------------------------------------------
+echo "case 1: by default step 2 borrows, and the pack leaves dst whole on its own"
+W=$(fixture); ARGV="$W/argv.log"
+OUT=$(STUB_ARGV="$ARGV" run_from "$W" 2)
+grep -q 'Step 3' <<<"$OUT"; check "step 2 completed and the pipeline moved on" $?
+grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; check "blobExec was given --alternates" $?
+grep -q 'no longer borrows any' <<<"$OUT"; check "the runner packed the borrowed objects in" $?
+self_contained "$W/$DST"; check "dst has no alternates file and fsck passes without it" $?
+git --git-dir="$W/$DST" count-objects -v | grep -q '^count: 0'
+check "every object is packed, none left loose" $?
+[ ! -e "$W/PACK-FAILED" ]; check "no PACK-FAILED marker" $?
+! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null; check "and no bitmap index" $?
+git clone -q "$W/$DST" "$W/clone" && [ -f "$W/clone/README" ] && [ ! -e "$W/clone/.git/objects/info/alternates" ]
+check "a step-6 style clone reads every file and borrows nothing" $?
+rm -rf "$W"
+
+# ---------------------------------------------------------------------------
+echo "case 2: --copy-objects, CREGIT_COPY_OBJECTS=1 and --gc none all copy instead"
+for how in flag env gcnone; do
+ W=$(fixture); ARGV="$W/argv.log"
+ case $how in
+ flag) OUT=$(STUB_ARGV="$ARGV" run_from "$W" --copy-objects 2) ;;
+ env) OUT=$(CREGIT_COPY_OBJECTS=1 STUB_ARGV="$ARGV" run_from "$W" 2) ;;
+ gcnone) OUT=$(STUB_ARGV="$ARGV" run_from "$W" --gc none 2) ;;
+ esac
+ grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; [ $? -ne 0 ]
+ check "$how: blobExec was NOT given --alternates" $?
+ [ ! -e "$W/$DST/objects/info/alternates" ] && git --git-dir="$W/$DST" cat-file -e 'master:README'
+ check "$how: dst holds its objects without borrowing" $?
+ if [ "$how" = flag ]; then
+ compgen -G "$W/$DST/objects/pack/*.pack" >/dev/null \
+ && ! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null
+ check "$how: gc packed dst, without a bitmap index" $?
+ fi
+ rm -rf "$W"
+done
+
+# ---------------------------------------------------------------------------
+echo "case 2b: CREGIT_BLOBEXEC_OPTS and CREGIT_BLOBEXEC_JAVA_OPTS reach blobExec"
+W=$(fixture); ARGV="$W/argv.log"
+OUT=$(CREGIT_BLOBEXEC_OPTS="--commits-per-transaction=1 --loose-compression=-1" \
+ CREGIT_BLOBEXEC_JAVA_OPTS="-Dorg.eclipse.jgit.util.sha1.implementation=java -XX:ActiveProcessorCount=3" \
+ STUB_ARGV="$ARGV" run_from "$W" 2)
+grep blobExec "$ARGV" | grep -q -- '--commits-per-transaction=1 --loose-compression=-1 '
+check "the blobExec flags, before the positional arguments" $?
+grep blobExec "$ARGV" | grep -q -- '^-Dorg.eclipse.jgit.util.sha1.implementation=java -XX:ActiveProcessorCount=3 -jar '
+check "the JVM flags, before -jar" $?
+grep -q 'blobExec flags from CREGIT_BLOBEXEC_OPTS: --commits-per-transaction=1 --loose-compression=-1' <<<"$OUT" \
+ && grep -q 'JVM flags from CREGIT_BLOBEXEC_JAVA_OPTS: -Dorg' <<<"$OUT"
+check "and both are logged" $?
+rm -rf "$W"
+# The runner runs in $REPO: a file there is what an unquoted --x=* would match.
+W=$(fixture); ARGV="$W/argv.log"; touch "$REPO/--x=bait"
+OUT=$(CREGIT_BLOBEXEC_OPTS="--x=*" STUB_ARGV="$ARGV" run_from "$W" 2)
+rm -f "$REPO/--x=bait"
+grep blobExec "$ARGV" | grep -qF -- ' --x=* '; check "a * in the flags is passed as is, not glob-expanded" $?
+OUT=$(STUB_ARGV="$W/argv2.log" run_from "$W" 2)
+! grep -q 'from CREGIT_BLOBEXEC' <<<"$OUT"; check "unset, nothing is logged" $?
+rm -rf "$W"
+
+# ---------------------------------------------------------------------------
+echo "case 3: a failed pack stops the run, resumably, and leaves dst readable"
+W=$(fixture)
+OUT=$(STUB_LOCK_PACK=1 run_from "$W" 2); RC=$?
+[ "$RC" -ne 0 ]; check "the run failed (got exit $RC)" $?
+grep -q 'Step 3' <<<"$OUT"; [ $? -ne 0 ]; check "and stopped before step 3" $?
+[ -f "$W/PACK-FAILED" ]; check "a PACK-FAILED marker was written" $?
+[ -f "$W/$DST/objects/info/alternates" ]; check "the alternates file is still in place" $?
+git --git-dir="$W/$DST" cat-file -e 'master:README'; check "so dst still reads the borrowed blobs" $?
+chmod u+w "$W/$DST/objects/pack"
+OUT=$(run_from "$W" 3)
+grep -q 'still borrows objects; packing it before step 3' <<<"$OUT"
+check "a resume at step 3 packs first" $?
+self_contained "$W/$DST"; check "and dst is then whole on its own" $?
+[ ! -e "$W/PACK-FAILED" ]; check "and the marker is gone" $?
+rm -rf "$W"
+
+# ---------------------------------------------------------------------------
+echo "case 4: a run killed while fsck ran (file set aside) is put back and redone"
+W=$(fixture)
+OUT=$(STUB_LOCK_PACK=1 run_from "$W" 2)
+chmod u+w "$W/$DST/objects/pack"
+mv "$W/$DST/objects/info/alternates" "$W/$DST/objects/info/alternates.packing"
+OUT=$(run_from "$W" 4)
+grep -q 'packing it before step 4' <<<"$OUT"; check "a resume at step 4 packs first" $?
+self_contained "$W/$DST"; check "and dst is whole, with neither file left" $?
+rm -rf "$W"
+
+# ---------------------------------------------------------------------------
+echo "case 5: a full run (FROM_STEP=1) keeps the work dir when the pack fails"
+W=$(mktemp -d "${TMPDIR:-/tmp}/packfull-XXXXXX")
+S=$(fixture)
+OUT=$( cd "$REPO" && STUB_LOCK_PACK=1 PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 timeout 300 \
+ bash ./run_pipeline_process.sh --repo-url "$S/proj-original.git" --repo-name proj \
+ --work "$W/work" --skip-html --mask '\.c$' 2>&1 ); RC=$?
+[ "$RC" -ne 0 ] && [ -f "$W/work/PACK-FAILED" ] && [ -d "$W/work/$DST" ]
+check "the EXIT trap kept \$WORK: the marker says it is resumable (exit $RC)" $?
+chmod -R u+w "$W" "$S"; rm -rf "$W" "$S"
+
+# ---------------------------------------------------------------------------
+echo "case 6: --gc aggressive borrows too, and its repack leaves dst whole"
+W=$(fixture); ARGV="$W/argv.log"
+OUT=$(STUB_ARGV="$ARGV" run_from "$W" --gc aggressive 2)
+grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; check "blobExec was given --alternates" $?
+grep -q 'no longer borrows any' <<<"$OUT" && grep -q 'Step 3' <<<"$OUT"
+check "the runner packed the borrowed objects in and moved on" $?
+self_contained "$W/$DST"; check "dst has no alternates file and fsck passes without it" $?
+! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null; check "and no bitmap index" $?
+rm -rf "$W"
+
+# ---------------------------------------------------------------------------
+echo "case 6b: CREGIT_WRITE_BITMAPS=1 brings the bitmap index back, on both paths"
+for how in borrow copy; do
+ W=$(fixture)
+ case $how in
+ borrow) OUT=$(CREGIT_WRITE_BITMAPS=1 run_from "$W" 2) ;;
+ copy) OUT=$(CREGIT_WRITE_BITMAPS=1 run_from "$W" --copy-objects 2) ;;
+ esac
+ grep -q 'Step 3' <<<"$OUT" && compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null
+ check "$how: the pack has a bitmap index" $?
+ git --git-dir="$W/$DST" cat-file -e 'master:README'; check "$how: dst reads its files" $?
+ rm -rf "$W"
+done
+
+# ---------------------------------------------------------------------------
+echo "case 7: a walk killed mid-way leaves dst borrowing; a step-2 resume finishes it"
+W=$(fixture)
+OUT=$(STUB_KILL_WALK=1 run_from "$W" 2); RC=$?
+[ "$RC" -ne 0 ] && ! grep -q 'Step 3' <<<"$OUT"; check "the run failed in step 2 (exit $RC)" $?
+[ -f "$W/$DST/objects/info/alternates" ] && [ -d "$W/proj-original.git" ]
+check "dst still borrows, and src is kept" $?
+OUT=$(run_from "$W" 2)
+grep -q 'no longer borrows any' <<<"$OUT" && grep -q 'Step 3' <<<"$OUT"
+check "the resume ran step 2 again, packed, and moved on" $?
+self_contained "$W/$DST"; check "dst is whole on its own" $?
+git clone -q "$W/$DST" "$W/clone" && [ -f "$W/clone/README" ] && [ ! -e "$W/clone/.git/objects/info/alternates" ]
+check "a step-6 style clone reads every file and borrows nothing" $?
+rm -rf "$W"
+
+echo ""
+echo "pack_alternates: $PASS passed, $FAIL failed"
+[ "$FAIL" -eq 0 ]
diff --git a/tests/pipeline_workdir_guard.sh b/tests/pipeline_workdir_guard.sh
index 11229242..36dccc83 100755
--- a/tests/pipeline_workdir_guard.sh
+++ b/tests/pipeline_workdir_guard.sh
@@ -247,7 +247,7 @@ echo "case 17: without the flag nothing changes, and the flag is what reaches bl
grep -q 'WIDENED_FLAG=""' "$RUNNER"; check "the flag defaults to absent" $?
grep -q '\[ "\$MASK_WIDENED" = 1 \] && WIDENED_FLAG="--mask-widened"' "$RUNNER"
check "and is only added when asked for" $?
-grep -q 'java -jar "\$BFG" \$MODE_FLAG \$WIDENED_FLAG' "$RUNNER"
+grep -qF 'java ${BLOBEXEC_JAVA_OPTS[@]+"${BLOBEXEC_JAVA_OPTS[@]}"} -jar "$BFG" $MODE_FLAG $WIDENED_FLAG' "$RUNNER"
check "so it reaches blobExec's argv" $?
echo