diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0f4b9ee..b6e89015 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: - name: Pipeline tests (shell) run: | devenv shell -- bash -c 'set -e - for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do + for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough pack_alternates; do bash "tests/$t.sh" done' diff --git a/README.md b/README.md index ea89528a..91a7a9d2 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ cd ../.. bash tests/test_token_worker.sh prove tests/t pytest -q -for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do +for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough pack_alternates; do bash "tests/$t.sh" done diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index 8ca84b7e..3a162165 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -14,6 +14,11 @@ object Main { /** Not 3 (memo mismatch): a no-op request must be told apart from a refusal. */ private[blobexec] val RetokenizeIneffectiveExitStatus = 7 + /** zlib level 1 for dst's loose objects: they are temporary, because the pack + * after step 2 re-deflates every one at git's pack.compression. Measured -11% + * consumer CPU on psi-probe against JGit's default level. */ + private[blobexec] val DefaultLooseCompression = 1 + private val UsageExitStatus = 1 private val RefusedExitStatus = 3 @@ -74,6 +79,16 @@ object Main { | --tokenizer-identity and --memo-dir; a no-op is refused (status ${RetokenizeIneffectiveExitStatus}) | --memo-dir= the tokenizer memo ($$BFG_MEMO_DIR) that --retokenize purges | --tokenizer-worker= tokenize through a pool of persistent workers; needs --pipeline(-trees) + | --alternates write /objects/info/alternates naming 's objects + | and copy no original blob; the caller must then repack + | without -l and delete that file. Not with --shard + | --commits-per-transaction= + | commits whose mapping rows share one SQLite transaction + | (default ${Walker.DefaultCommitsPerTransaction}; 1 = one per commit) + | --sqlite-cache-mb= SQLite page cache for the mapping DB (default ${Mapping.DefaultCacheMiB}; 0 = SQLite's 2 MB) + | --sqlite-mmap-mb= memory-map up to this much of the mapping DB (default ${Mapping.DefaultMmapMiB}; 0 = off) + | --loose-compression= zlib level of the loose objects written to , in memory only + | (default ${DefaultLooseCompression}; -1 = JGit's default, 6). git repacks them anyway | --blob-timeout= budget for one run (default ${BlobExec.DefaultTimeoutSeconds}); then its process | tree is killed and the blob excluded | --stall-timeout= exit ${Walker.StalledExitStatus} after this long with no progress (default ${Walker.DefaultStallTimeoutSeconds}); @@ -103,6 +118,10 @@ object Main { retokenize: Set[String] = Set.empty, memoDir: Option[Path] = None, tokenizerWorker: Option[Path] = None, + alternates: Boolean = false, + commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction, + sqlite: Mapping.Tuning = Mapping.Tuning(), + looseCompression: Int = DefaultLooseCompression, blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, stallExplicit: Boolean = false, @@ -136,6 +155,8 @@ object Main { "into, or the invalidation would leave the real memo's stale entries in place." val PipelinesExclusive = withUsage("Error: --pipeline and --pipeline-trees are mutually exclusive") + val AlternatesUnderShard = withUsage( + "Error: --alternates cannot be used with --shard: shard_merge.py copies each shard's own objects") val ShardWithPipeline = withUsage( "Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees") val MaskWidenedUnderShard = withUsage( @@ -222,10 +243,23 @@ object Main { private val TokenizerWorkerFlag = new ValuedFlag("--tokenizer-worker") private val BlobTimeoutFlag = new ValuedFlag("--blob-timeout") private val StallTimeoutFlag = new ValuedFlag("--stall-timeout") + private val CommitsPerTxFlag = new ValuedFlag("--commits-per-transaction") + private val SqliteCacheFlag = new ValuedFlag("--sqlite-cache-mb") + private val SqliteMmapFlag = new ValuedFlag("--sqlite-mmap-mb") + private val LooseCompressionFlag = new ValuedFlag("--loose-compression") private def flagValue[A](flag: ValuedFlag, spec: String)(parse: String => Either[String, A]): Either[String, A] = parse(spec).left.map(Messages.badFlagValue(flag.name, _)) + private def positive(spec: String): Either[String, Int] = + spec.toIntOption.filter(_ > 0).toRight(s"must be a positive whole number [$spec]") + + private def mebibytes(spec: String): Either[String, Int] = + spec.toIntOption.filter(_ >= 0).toRight(s"must be a whole number of MiB, 0 or more [$spec]") + + private def level(spec: String): Either[String, Int] = + spec.toIntOption.filter(n => n >= -1 && n <= 9).toRight(s"must be a zlib level, -1 to 9 [$spec]") + private def seconds(spec: String): Either[String, Int] = parsePositiveSeconds(spec).toRight(Messages.secondsWanted(spec)) @@ -249,6 +283,7 @@ object Main { case "--pipeline" => Right(o.copy(pipeline = true)) case "--pipeline-trees" => Right(o.copy(pipelineTrees = true)) case "--mask-widened" => Right(o.copy(maskWidened = true)) + case "--alternates" => Right(o.copy(alternates = true)) case TokenizerIdentityFlag(spec) => flagValue(TokenizerIdentityFlag, spec)(TokenizerIdentity.parse).map(id => o.copy(tokenizerIdentity = id)) case RetokenizeFlag(spec) => @@ -265,6 +300,14 @@ object Main { Right(o.copy(tokenizerWorker = Some(Paths.get(spec)))) case BlobTimeoutFlag(spec) => flagValue(BlobTimeoutFlag, spec)(seconds).map(secs => o.copy(blobTimeoutSeconds = secs)) + case CommitsPerTxFlag(spec) => + flagValue(CommitsPerTxFlag, spec)(positive).map(n => o.copy(commitsPerTransaction = n)) + case SqliteCacheFlag(spec) => + flagValue(SqliteCacheFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(cacheMiB = n))) + case SqliteMmapFlag(spec) => + flagValue(SqliteMmapFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(mmapMiB = n))) + case LooseCompressionFlag(spec) => + flagValue(LooseCompressionFlag, spec)(level).map(n => o.copy(looseCompression = n)) case StallTimeoutFlag(spec) => flagValue(StallTimeoutFlag, spec)(seconds).map(secs => o.copy(stallTimeoutSeconds = secs, stallExplicit = true)) case other => Left(Messages.unknownFlag(other)) @@ -285,6 +328,7 @@ object Main { (retokenizing && o.shard.isDefined) -> (() => Messages.RetokenizeUnderShard), (!retokenizing && o.memoDir.isDefined) -> (() => Messages.MemoDirWithoutRetokenize), (o.shard.isDefined && o.anyPipeline) -> (() => Messages.ShardWithPipeline), + (o.shard.isDefined && o.alternates) -> (() => Messages.AlternatesUnderShard), o.tokenizerWorker.exists(p => !Files.isRegularFile(p) || !Files.isExecutable(p)) -> (() => Messages.workerNotExecutable(o.tokenizerWorker.get)), (o.tokenizerWorker.isDefined && !o.anyPipeline) -> (() => Messages.WorkerNeedsPipeline) @@ -311,7 +355,20 @@ object Main { } yield full } + /** JGit hashes with its own Java SHA-1, which also runs collision detection + * (SHA1DC); the JDK's intrinsic one is faster (-5% on psi-probe). Both give the + * same id for every input that is not a crafted SHA-1 collision; on one of + * those the Java one throws instead. Set before JGit hashes anything, and only + * when the operator did not choose: -Dorg.eclipse.jgit.util.sha1.implementation=java + * keeps the old one. JGit reads the property on every new hasher. */ + private[blobexec] val Sha1ImplementationProperty = "org.eclipse.jgit.util.sha1.implementation" + + private[blobexec] def defaultSha1Implementation(): Unit = + if (System.getProperty(Sha1ImplementationProperty) == null) + System.setProperty(Sha1ImplementationProperty, "jdkNative") + def main(args: Array[String]): Unit = { + defaultSha1Implementation() val options = parse(args.toVector) match { case Right(o) => o case Left(why) => exit(why, UsageExitStatus) @@ -339,7 +396,8 @@ object Main { println(startLine(o, stallSeconds, incremental)) val src = openSrc(o.src) - val dst = openOrInitDst(o.dst) + val dst = openOrInitDst(o.dst, borrowFrom = Option.when(o.alternates)(src.getObjectsDirectory.toPath)) + setLooseCompression(dst, o.looseCompression) val mapping = openMapping(o, src, dst) match { case Right(m) => m case Left(refusal) => @@ -368,13 +426,17 @@ object Main { s"maskWidened=${o.maskWidened} denylistEntries=${BlobDenylist.shipped.size} " + s"tokenizerIdentity=${if (o.tokenizerIdentity.isEmpty) "none" else o.tokenizerIdentity.render} " + s"retokenize=${if (o.retokenize.isEmpty) "none" else o.retokenize.toVector.sorted.mkString(",")} " + - s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)}" + s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)} " + + s"alternates=${o.alternates} commitsPerTransaction=${o.commitsPerTransaction} " + + s"sqliteCacheMiB=${o.sqlite.cacheMiB} sqliteMmapMiB=${o.sqlite.mmapMiB} " + + s"looseCompression=${o.looseCompression} sha1=${System.getProperty(Sha1ImplementationProperty)}" } private final case class Refusal(status: Int, message: String) private def openMapping(o: Options, src: FileRepository, dst: FileRepository): Either[Refusal, Mapping] = - Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst))) + Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst), + o.sqlite)) .toEither .left.map(e => refusalFor(e, o).getOrElse(throw e)) @@ -436,9 +498,11 @@ object Main { src, dst, mapping, o.mask.r, o.command, o.abortOnError, parallelism, o.pipeline, o.pipelineTrees, o.shard, destinationMayContainObjects = incremental, + borrowOriginalObjects = o.alternates, blobTimeoutSeconds = o.blobTimeoutSeconds, stallTimeoutSeconds = stallSeconds, workerPool = workerPool, + commitsPerTransaction = o.commitsPerTransaction, denylist = BlobDenylist.shipped ).run() finally workerPool.foreach(_.close()) @@ -456,6 +520,7 @@ object Main { s"originalBlobDestinationLookups=${stats.originalBlobDestinationLookups} " + s"originalBlobBytesCopied=${stats.originalBlobBytesCopied} " + s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " + + s"originalBlobsBorrowed=${stats.originalBlobsBorrowed} " + s"refsProjected=${stats.refsProjected} " + s"blobsTimedOut=${stats.blobsTimedOut} " + s"blobsOversized=${stats.blobsOversized} " + @@ -480,10 +545,40 @@ object Main { FileRepositoryBuilder.create(gitDir).asInstanceOf[FileRepository] } - private def openOrInitDst(path: Path): FileRepository = { + /** `borrowFrom` names src's objects directory: it is written to dst's + * objects/info/alternates BEFORE dst is opened for the walk, because JGit reads + * that file once, and a walk that skipped a copy must find the object there. */ + private[blobexec] def openOrInitDst(path: Path, borrowFrom: Option[Path] = None): FileRepository = { val exists = Files.isDirectory(path) val repo = FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository] if (!exists) repo.create(true) - repo + borrowFrom match { + case None => repo + case Some(objects) => + val info = repo.getObjectsDirectory.toPath.resolve("info") + repo.close() + Files.createDirectories(info) + writeAlternates(info.resolve("alternates"), objects) + FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository] + } + } + + /** In memory only: dst's config file is not touched, so nothing that reads dst + * later (git repack, steps 3+) can see it. Object ids do not depend on it. JGit + * reads it when an inserter is created; a reload of the file would only bring + * back the default level, a speed loss, never a different object. */ + private[blobexec] def setLooseCompression(dst: FileRepository, level: Int): Unit = { + val config = dst.getConfig // loads the file first, if it changed + config.setInt("core", null, "compression", level) + } + + /** One absolute line, replacing whatever was there: a resume writes the same + * file again. Written to a temporary file and renamed, so git never reads half. */ + private[blobexec] def writeAlternates(file: Path, objects: Path): Unit = { + val line = objects.toAbsolutePath.normalize.toString + "\n" + val tmp = file.resolveSibling(file.getFileName.toString + ".tmp") + Files.write(tmp, line.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + Files.move(tmp, file, java.nio.file.StandardCopyOption.REPLACE_EXISTING, + java.nio.file.StandardCopyOption.ATOMIC_MOVE) } } diff --git a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala index d6739ae7..c13e57ee 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala @@ -296,8 +296,10 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends Mapping.RetokenizeResult(before, rowCounts, extensions, invalidated, memo) } - /** Run `body` inside a transaction; commit on success, rollback on throw. */ + /** Run `body` inside a transaction; commit on success, rollback on throw. + * An open group (see [[inGroupedTx]]) is committed first. */ def inTx[A](body: => A): A = { + commitGroup() conn.setAutoCommit(false) try { val result = body @@ -312,7 +314,67 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } } + /** A PRAGMA's current value on this connection, for tests. */ + private[blobexec] def pragmaValue(name: String): Long = { + val st = conn.createStatement() + try { val rs = st.executeQuery(s"PRAGMA $name"); rs.next(); rs.getLong(1) } finally st.close() + } + + // -- grouped transactions --------------------------------------------------- + // A commit's rows are one body; up to `maxBodies` bodies share one transaction. + // Each SQLite transaction rewrites the pages it touched into the WAL, and the + // walk touches the same index pages for every commit, so per-commit + // transactions cost more than the rows. A group is all-or-nothing, like the + // one-commit transaction it replaces: a crash loses whole bodies, never part of + // one, so a commit row is still never durable without its trees and blobs, and + // the lost commits are re-walked on resume (their tokens come from the memo). + + private var groupOpen = false + private var groupBodies = 0 + private var groupStartNanos = 0L + + /** As [[inTx]], but `body` joins the open group, which is committed once it + * holds `maxBodies` bodies or is `maxNanos` old. A throw in `body` rolls back + * the whole group. `maxBodies` <= 1 is plain [[inTx]]. Not thread-safe: the + * caller serializes access, as it does for every other method here. */ + def inGroupedTx[A](maxBodies: Int, maxNanos: Long)(body: => A): A = + if (maxBodies <= 1) inTx(body) + else { + if (!groupOpen) { + conn.setAutoCommit(false) + groupOpen = true + groupBodies = 0 + groupStartNanos = System.nanoTime() + } + val result = + try body + catch { case t: Throwable => abandonGroup(); throw t } + groupBodies += 1 + if (groupBodies >= maxBodies || System.nanoTime() - groupStartNanos >= maxNanos) commitGroup() + result + } + + /** Commit the open group, if any. The walk calls it when it stops, by abort too. */ + def commitGroup(): Unit = + if (groupOpen) { + try conn.commit() + catch { case t: Throwable => abandonGroup(); throw t } + groupOpen = false + groupBodies = 0 + conn.setAutoCommit(true) + } + + private def abandonGroup(): Unit = + if (groupOpen) { + groupOpen = false + groupBodies = 0 + try conn.rollback() catch { case _: Throwable => () } + try conn.setAutoCommit(true) catch { case _: Throwable => () } + } + override def close(): Unit = { + // Every body in the group finished (a failed one rolled the group back). + commitGroup() (List(selBlob, insBlob, selCommit, insCommit, selTree, insTree, selRef, insRef, delRef, selMeta, insMeta) ++ warmSelBlob.toList ++ warmSelTree.toList) .foreach(s => try s.close() catch { case _: Throwable => () }) @@ -456,6 +518,30 @@ object Mapping { } finally st.close() } + /** Read-path sizing for the walk's DB. Both change speed only, never results. + * cacheMiB SQLite's own page cache, private anonymous memory, filled only + * as pages are read: the DB's size at most (default 2 MB). + * mmapMiB map up to this much of the file. Lookups then read the kernel's + * page cache in place instead of copying each page out with a + * syscall. Those are file pages: shared and reclaimable, and the + * same pages that reads cache anyway. 0 turns it off. */ + final case class Tuning(cacheMiB: Int = DefaultCacheMiB, mmapMiB: Int = DefaultMmapMiB) { + require(cacheMiB >= 0 && mmapMiB >= 0, s"SQLite sizes must be >= 0 (cache=$cacheMiB MiB, mmap=$mmapMiB MiB)") + } + val DefaultCacheMiB = 256 + val DefaultMmapMiB = 4096 + /** SQLite's defaults: what every run used before. */ + val UntunedSqlite: Tuning = Tuning(cacheMiB = 0, mmapMiB = 0) + + private def applyTuning(conn: Connection, t: Tuning): Unit = { + val st = conn.createStatement() + try { + // Negative cache_size is KiB; 0 keeps the default. + if (t.cacheMiB > 0) st.execute(s"PRAGMA cache_size = -${t.cacheMiB.toLong * 1024L}") + if (t.mmapMiB > 0) st.execute(s"PRAGMA mmap_size = ${t.mmapMiB.toLong * 1024L * 1024L}") + } finally st.close() + } + /** Opt-in for `--mask-widened`. `newBlobResolves` has no default: `new_blob` ids * live only in dst, and a wiped dst would dangle every reused row. */ final case class MaskWidening(newBlobResolves: String => Boolean, @@ -468,7 +554,8 @@ object Mapping { def open(path: Path, command: String, mask: String, warm: Option[Path] = None, maskWidening: Option[MaskWidening] = None, tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, - retokenize: Option[Retokenize] = None): Mapping = { + retokenize: Option[Retokenize] = None, + tuning: Tuning = Tuning()): Mapping = { // Refused, not composed: each invalidation is verified against a state the // other would change. Run one, then the other as a resume. require(!(maskWidening.isDefined && retokenize.isDefined), @@ -492,6 +579,7 @@ object Mapping { val url = s"jdbc:sqlite:${path.toAbsolutePath}" val conn = DriverManager.getConnection(url) enableForeignKeys(conn) + applyTuning(conn, tuning) val st = conn.createStatement() try Schema.foreach(st.execute) finally st.close() diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala index 5c1ece55..aea1cb94 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala @@ -35,7 +35,9 @@ final case class WalkStats( originalBlobCacheHits: Long, originalBlobDestinationLookups: Long, originalBlobBytesCopied: Long, - originalBlobBytesAvoided: Long + originalBlobBytesAvoided: Long, + /** Original blobs left in src, which dst reads through objects/info/alternates. */ + originalBlobsBorrowed: Long = 0L ) /** Rebuild `src` history into `dst`, persisting mappings to `mapping`. @@ -53,9 +55,16 @@ final class Walker( shard: Option[(Int, Int)] = None, deduplicateOriginalBlobs: Boolean = true, destinationMayContainObjects: Boolean = true, + // dst's objects/info/alternates names src's objects: an original blob is + // already readable from dst, so it is never copied. The caller must repack dst + // without -l before dropping the alternates file (run_pipeline_process.sh does). + borrowOriginalObjects: Boolean = false, blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, workerPool: Option[TokenizerWorkerPool] = None, + // Commits whose mapping rows share one SQLite transaction (Mapping.inGroupedTx). + // 1 is one transaction per commit, as before. + commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction, // A parameter only so tests can supply a fixture. denylist: BlobDenylist = BlobDenylist.shipped ) { @@ -66,6 +75,15 @@ final class Walker( // touch it, so the CPU-heavy path stays lock-free. private val dbLock = new AnyRef + /** On the way out of a walk: the group holds finished commits only (a failed + * body rolled it back), so keeping them is right, and a commit error here + * must not hide the one already propagating. */ + private def commitGroupQuietly(): Unit = + dbLock.synchronized { + try mapping.commitGroup() + catch { case scala.util.control.NonFatal(e) => System.err.println(s"warning: mapping commit failed: ${e.getMessage}") } + } + private val blobCommandExecutions = new LongAdder private val blobsTimedOut = new LongAdder private val blobsOversized = new LongAdder @@ -79,6 +97,7 @@ final class Walker( private val originalBlobDestinationLookups = new LongAdder private val originalBlobBytesCopied = new LongAdder private val originalBlobBytesAvoided = new LongAdder + private val originalBlobsBorrowed = new LongAdder // -- stall watchdog ------------------------------------------------------ // `Await.result` is unbounded: a duration cannot tell a wedged run from a large @@ -215,7 +234,8 @@ final class Walker( originalBlobCacheHits = originalBlobCacheHits.sum(), originalBlobDestinationLookups = originalBlobDestinationLookups.sum(), originalBlobBytesCopied = originalBlobBytesCopied.sum(), - originalBlobBytesAvoided = originalBlobBytesAvoided.sum() + originalBlobBytesAvoided = originalBlobBytesAvoided.sum(), + originalBlobsBorrowed = originalBlobsBorrowed.sum() ) } finally revWalk.close() } @@ -433,7 +453,7 @@ final class Walker( val newCommit = buildCommit(rc, parents, newTreeId, commitInserter) - mapping.inTx { + mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) { // Matching blobs the cmd resolved (Replace or Skip outcomes). resolved.foreach { case ((origSha, path), newId) => mapping.putBlob(origSha, path, newId.name) @@ -456,9 +476,11 @@ final class Walker( } } + mapping.commitGroup() treeInserter.flush() commitInserter.flush() } finally { + commitGroupQuietly() treeInserter.close() commitInserter.close() pool.shutdown() @@ -560,7 +582,7 @@ final class Walker( }.map(ObjectId.fromString) val newCommit = buildCommitFromData(data, parents, newTreeId, commitInserter) dbLock.synchronized { - mapping.inTx { + mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) { resolved.foreach { case ((origSha, path), newId) => mapping.putBlob(origSha, path, newId.name) } @@ -586,9 +608,11 @@ final class Walker( } } + dbLock.synchronized(mapping.commitGroup()) treeInserter.flush() commitInserter.flush() } finally { + commitGroupQuietly() treeInserter.close() commitInserter.close() pool.shutdown() @@ -693,7 +717,7 @@ final class Walker( }.map(ObjectId.fromString) val newCommit = buildCommitFromData(data, parents, newTreeId, commitInserter) dbLock.synchronized { - mapping.inTx { + mapping.inGroupedTx(commitsPerTransaction, TransactionMaxNanos) { resolved.foreach { case ((origSha, path), newId) => mapping.putBlob(origSha, path, newId.name) } @@ -717,8 +741,10 @@ final class Walker( } } + dbLock.synchronized(mapping.commitGroup()) commitInserter.flush() } finally { + commitGroupQuietly() commitInserter.close() pool.shutdown() if (!pool.awaitTermination(1, TimeUnit.MINUTES)) { pool.shutdownNow(); () } @@ -1139,6 +1165,13 @@ final class Walker( // Copying a 435 GB repository's unmasked blobs is work, not a stall. progress(s"original blob ${id.name}") + // Borrowed: src holds the blob and dst reads src's objects, so there is + // nothing to write. The pass-through copy was half the consumer's time. + if (borrowOriginalObjects) { + originalBlobsBorrowed.increment() + return + } + if (!deduplicateOriginalBlobs) { val (_, size) = insertBlob(inserter) originalBlobCopies.increment() @@ -1393,6 +1426,13 @@ final class Walker( } object Walker { + /** Commits per SQLite transaction. A crash or a watchdog halt loses at most this + * many commits' rows (or [[TransactionMaxNanos]] of them), and resume re-walks + * them. */ + val DefaultCommitsPerTransaction = 256 + /** A group is also committed once this old, so a slow stretch keeps its rows. */ + val TransactionMaxNanos: Long = 2L * 1000L * 1000L * 1000L + /** JGit's default `core.streamFileThreshold`, at or above which `getBytes` * throws. Only a fast path: [[Walker#readBlob]] also catches the exception. */ diff --git a/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala new file mode 100644 index 00000000..314a6d36 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/AlternatesSpec.scala @@ -0,0 +1,235 @@ +package cregit.blobexec + +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.internal.storage.file.FileRepository +import org.eclipse.jgit.lib.ObjectId +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ +import scala.sys.process._ + +/** `--alternates`: dst borrows src's objects instead of copying the original + * blobs, and `git repack -a -d` (no -l) then makes dst whole. The output must be + * the same objects, refs and mapping rows as a run that copies. */ +class AlternatesSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { + + private val workRoot: Path = Files.createTempDirectory("alternates-") + + override def afterAll(): Unit = deleteRecursive(workRoot) + + private def deleteRecursive(p: Path): Unit = + if (Files.exists(p)) { + if (Files.isDirectory(p)) Files.list(p).iterator().asScala.toList.foreach(deleteRecursive) + Files.delete(p) + } + + private val Mask = """\.[ch]$""" + + private def commit(git: Git, files: Map[String, String], message: String): Unit = { + val wt = git.getRepository.getWorkTree.toPath + files.foreach { case (rel, body) => + val p = wt.resolve(rel) + Files.createDirectories(p.getParent) + Files.writeString(p, body) + } + files.keys.foreach(f => git.add().addFilepattern(f).call()) + git.commit().setMessage(message) + .setAuthor("Tester", "t@example.org").setCommitter("Tester", "t@example.org").call() + () + } + + /** Five commits on master plus a side branch. a.c/b.h are tokenized; the rest + * pass through, among them a blob that only an old commit holds. */ + private def sourceRepo(dir: Path): Git = { + Files.createDirectories(dir) + val git = Git.init().setDirectory(dir.toFile).setBare(false).call() + commit(git, Map("a.c" -> "int a;\n", "README.md" -> "readme 1\n", "docs/x.txt" -> "x\n"), "c1") + commit(git, Map("b.h" -> "int b;\n", "README.md" -> "readme 2\n"), "c2") + commit(git, Map("a.c" -> "int a = BOOM;\n", "docs/y.txt" -> "y\n"), "c3") + git.branchCreate().setName("side").call() + commit(git, Map("lib/z.c" -> "int z;\n", "README.md" -> "readme 3\n"), "c4") + commit(git, Map("docs/x.txt" -> "x2\n"), "c5") + git + } + + /** Uppercases its input; fails on a BOOM blob unless `flag` exists. */ + private def command(dir: Path, flag: Path): String = { + val f = dir.resolve("cmd.sh") + Files.writeString(f, + s"""#!/bin/sh + |c=$$(cat) + |case "$$c" in *BOOM*) [ -e '$flag' ] || exit 1 ;; esac + |printf '%s\\n' "$$c" | tr a-z A-Z + |""".stripMargin) + f.toFile.setExecutable(true) + f.toAbsolutePath.toString + } + + private def walk(src: FileRepository, dstPath: Path, db: Path, cmd: String, + borrow: Boolean, abortOnError: Boolean = false, + commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction): WalkStats = { + val incremental = Files.isDirectory(dstPath) + val dst = Main.openOrInitDst(dstPath, Option.when(borrow)(src.getObjectsDirectory.toPath)) + val mapping = Mapping.open(db, cmd, Mask) + try new Walker(src, dst, mapping, Mask.r, cmd, abortOnError, 2, pipeline = true, + destinationMayContainObjects = incremental, borrowOriginalObjects = borrow, + denylist = BlobDenylist.empty, commitsPerTransaction = commitsPerTransaction).run() + finally { mapping.close(); dst.close() } + } + + private def git(gitDir: Path, args: String*): (Int, String) = { + val out = new StringBuilder + val rc = Process(Seq("git", "--git-dir", gitDir.toString) ++ args) + .!(ProcessLogger(l => out.append(l).append('\n'), _ => ())) + (rc, out.toString) + } + + private def reachable(gitDir: Path): Vector[String] = { + val (rc, out) = git(gitDir, "rev-list", "--objects", "--all") + rc shouldBe 0 + out.linesIterator.map(_.takeWhile(_ != ' ')).toVector.sorted + } + + private def refs(gitDir: Path): String = git(gitDir, "for-each-ref", "--format=%(objectname) %(refname)")._2 + + private def table(db: Path, sql: String): Vector[String] = { + val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db") + try { + val rs = c.createStatement().executeQuery(sql) + val b = Vector.newBuilder[String] + while (rs.next()) b += (1 to rs.getMetaData.getColumnCount).map(rs.getString).mkString(" ") + b.result() + } finally c.close() + } + + // processed_at is a wall-clock stamp, so it is left out. + private val MappingQueries = Seq( + "SELECT orig_commit, new_commit FROM commit_map ORDER BY 1", + "SELECT orig_blob, path, new_blob FROM blob_map ORDER BY 1, 2", + "SELECT orig_tree, new_tree FROM tree_map ORDER BY 1") + + private def alternates(dst: Path): Path = dst.resolve("objects/info/alternates") + + /** What run_pipeline_process.sh's dissolve_alternates does, minus logging. */ + private def dissolve(dst: Path): Unit = { + git(dst, "repack", "-a", "-d")._1 shouldBe 0 + Files.delete(alternates(dst)) + git(dst, "fsck", "--connectivity-only", "--no-dangling")._1 shouldBe 0 + } + + test("a borrowing walk copies no original blob and, once repacked, equals a copying walk") { + val w = Files.createTempDirectory(workRoot, "equal-") + val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] + val flag = Files.createFile(w.resolve("flag")) + val cmd = command(w, flag) + + val copied = walk(src, w.resolve("copy.git"), w.resolve("copy.db"), cmd, borrow = false) + val borrowed = walk(src, w.resolve("borrow.git"), w.resolve("borrow.db"), cmd, borrow = true) + + copied.originalBlobCopies should be > 0L + copied.originalBlobsBorrowed shouldBe 0L + borrowed.originalBlobCopies shouldBe 0L + borrowed.originalBlobsBorrowed shouldBe copied.originalBlobCopyRequests + borrowed.commitsProcessed shouldBe 5 + + val b = w.resolve("borrow.git") + Files.readString(alternates(b)).trim shouldBe src.getObjectsDirectory.toPath.toAbsolutePath.normalize.toString + + // Without the file, dst is missing the borrowed blobs: the pack is not optional. + Files.move(alternates(b), b.resolve("objects/info/alternates.aside")) + git(b, "fsck", "--connectivity-only", "--no-dangling")._1 should not be 0 + Files.move(b.resolve("objects/info/alternates.aside"), alternates(b)) + + dissolve(b) + reachable(b) shouldBe reachable(w.resolve("copy.git")) + refs(b) shouldBe refs(w.resolve("copy.git")) + for (q <- MappingQueries) + table(w.resolve("borrow.db"), q) shouldBe table(w.resolve("copy.db"), q) + + // A non-bare clone, as step 6 makes, reads every file. + val clone = w.resolve("clone") + Process(Seq("git", "clone", "-q", b.toString, clone.toString)).! shouldBe 0 + Files.readString(clone.resolve("README.md")) shouldBe "readme 3\n" + Files.readString(clone.resolve("a.c")) shouldBe "INT A = BOOM;\n" + } + + test("an interrupted borrowing walk resumes to the same result, with grouped transactions") { + val w = Files.createTempDirectory(workRoot, "resume-") + val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] + val flag = w.resolve("flag") + val cmd = command(w, flag) + val dst = w.resolve("dst.git") + val db = w.resolve("dst.db") + + // c3's BOOM blob aborts the walk: c1 and c2 are done and must be durable even + // though they share one still-open transaction group. + val first = walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100) + first.aborted shouldBe true + table(db, "SELECT count(*) FROM commit_map") shouldBe Vector("2") + + Files.createFile(flag) + val second = walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100) + second.aborted shouldBe false + second.commitsProcessed shouldBe 3 + dissolve(dst) + + // Reference: one uninterrupted copying walk, one commit per transaction. + val ref = w.resolve("ref.git") + walk(src, ref, w.resolve("ref.db"), cmd, borrow = false, commitsPerTransaction = 1).aborted shouldBe false + reachable(dst) shouldBe reachable(ref) + refs(dst) shouldBe refs(ref) + for (q <- MappingQueries) table(db, q) shouldBe table(w.resolve("ref.db"), q) + } + + test("a crash that loses the open group resumes to the same result") { + val w = Files.createTempDirectory(workRoot, "crash-") + val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] + val flag = w.resolve("flag") + val cmd = command(w, flag) + val dst = w.resolve("dst.git") + val db = w.resolve("dst.db") + val snapshot = w.resolve("snapshot.db") + + // The DB as the last committed group left it: c1 and c2. + walk(src, dst, db, cmd, borrow = true, abortOnError = true, commitsPerTransaction = 100).aborted shouldBe true + Files.copy(db, snapshot) + + // The walk goes on and writes c3..c5's objects to dst, then dies before + // their group commits: the DB rolls back to the snapshot, and no ref is + // projected yet. That is what a kill or the watchdog's halt leaves. + Files.createFile(flag) + walk(src, dst, db, cmd, borrow = true, commitsPerTransaction = 100).aborted shouldBe false + for (suffix <- Seq("", "-wal", "-shm")) Files.deleteIfExists(w.resolve("dst.db" + suffix)) + Files.copy(snapshot, db) + git(dst, "for-each-ref", "--format=%(refname)")._2.linesIterator.foreach(r => git(dst, "update-ref", "-d", r)._1 shouldBe 0) + table(db, "SELECT count(*) FROM commit_map") shouldBe Vector("2") + + val resumed = walk(src, dst, db, cmd, borrow = true, commitsPerTransaction = 100) + resumed.aborted shouldBe false + resumed.commitsProcessed shouldBe 3 + dissolve(dst) + + val ref = w.resolve("ref.git") + walk(src, ref, w.resolve("ref.db"), cmd, borrow = false, commitsPerTransaction = 1).aborted shouldBe false + reachable(dst) shouldBe reachable(ref) + refs(dst) shouldBe refs(ref) + for (q <- MappingQueries) table(db, q) shouldBe table(w.resolve("ref.db"), q) + } + + test("a resume reuses the alternates file, rewriting it rather than appending") { + val w = Files.createTempDirectory(workRoot, "rewrite-") + val src = sourceRepo(w.resolve("src")).getRepository.asInstanceOf[FileRepository] + val dst = w.resolve("dst.git") + Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath)).close() + Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath)).close() + Files.readAllLines(alternates(dst)).asScala.toList shouldBe + List(src.getObjectsDirectory.toPath.toAbsolutePath.normalize.toString) + // And dst opened that way really reads src's objects. + val r = Main.openOrInitDst(dst, Some(src.getObjectsDirectory.toPath)) + try r.getObjectDatabase.has(ObjectId.fromString(src.resolve("HEAD").name)) shouldBe true + finally r.close() + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala b/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala new file mode 100644 index 00000000..955e3d59 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/LooseCompressionSpec.scala @@ -0,0 +1,43 @@ +package cregit.blobexec + +import org.eclipse.jgit.lib.{Constants, ObjectId} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ + +/** `--loose-compression`: the zlib level of dst's loose objects, set in memory. */ +class LooseCompressionSpec extends AnyFunSuite with Matchers { + + /** Insert one object and return its loose file's zlib header (CMF, FLG). */ + private def headerAfterInsert(level: Option[Int]): (Path, Int, ObjectId) = { + val dir = Files.createTempDirectory("loose-") + val path = dir.resolve("dst.git") + val dst = Main.openOrInitDst(path) + try { + level.foreach(Main.setLooseCompression(dst, _)) + val ins = dst.newObjectInserter() + val id = try ins.insert(Constants.OBJ_BLOB, ("x" * 4096).getBytes("UTF-8")) finally ins.close() + val file = path.resolve("objects").resolve(id.name.take(2)).resolve(id.name.drop(2)) + val bytes = Files.readAllBytes(file) + (path, bytes(1) & 0xff, id) + } finally dst.close() + } + + test("level 1 writes zlib's fastest level; no setting writes JGit's default") { + // FLG's top two bits are FLEVEL: 0 = fastest (level 1), 2 = default (6). + val (fastRepo, fastFlg, fastId) = headerAfterInsert(Some(1)) + val (defRepo, defFlg, defId) = headerAfterInsert(None) + (fastFlg >> 6) shouldBe 0 + (defFlg >> 6) shouldBe 2 + fastId shouldEqual defId // the id hashes the content, never the deflate stream + Main.DefaultLooseCompression shouldBe 1 + } + + test("the level stays in memory: dst's config file is not changed") { + val (repo, _, _) = headerAfterInsert(Some(1)) + val config = Files.readAllLines(repo.resolve("config")).asScala.mkString("\n") + config should not include "compression" + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala index d5637fb2..4583e4aa 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala @@ -218,6 +218,28 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { inside(Main.parse("--shard=1/4" +: positional)) { case Right(o) => o.shard shouldEqual Some((1, 4)) } } + test("the step-2 speed flags are read, and default to on") { + inside(Main.parse(Seq("--alternates", "--commits-per-transaction=1", "--sqlite-cache-mb=0", + "--sqlite-mmap-mb=0", "--loose-compression=-1") ++ positional)) { case Right(o) => + o.alternates shouldBe true + o.looseCompression shouldBe -1 + o.commitsPerTransaction shouldBe 1 + o.sqlite shouldEqual Mapping.UntunedSqlite + } + inside(Main.parse(positional)) { case Right(o) => + o.alternates shouldBe false // the caller must repack, so the caller asks for it + o.commitsPerTransaction shouldBe Walker.DefaultCommitsPerTransaction + o.sqlite shouldEqual Mapping.Tuning(Mapping.DefaultCacheMiB, Mapping.DefaultMmapMiB) + o.looseCompression shouldBe 1 + } + } + + test("--alternates is refused with --shard") { + inside(Main.parse(Seq("--alternates", "--shard=0/2") ++ positional)) { case Left(why) => + why should include("--alternates cannot be used with --shard") + } + } + test("parse leaves the defaults when no flag is given") { inside(Main.parse(positional)) { case Right(o) => o shouldEqual Main.Options(positional = positional.toVector) @@ -239,7 +261,11 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { "--shard=1" -> "--shard", "--warm=/nonexistent/db" -> "--warm", "--tokenizer-identity=" -> "--tokenizer-identity", - "--memo-dir=/nonexistent/" -> "--memo-dir" + "--memo-dir=/nonexistent/" -> "--memo-dir", + "--commits-per-transaction=0" -> "--commits-per-transaction", + "--sqlite-cache-mb=-1" -> "--sqlite-cache-mb", + "--sqlite-mmap-mb=lots" -> "--sqlite-mmap-mb", + "--loose-compression=10" -> "--loose-compression" ).foreach { case (flag, name) => withClue(s"$flag: ") { inside(Main.parse(flag +: positional)) { case Left(why) => why should startWith(s"Error: $name") } diff --git a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala index 46af5926..1d4a1c15 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala @@ -441,4 +441,106 @@ class MappingSpec extends AnyFunSuite with Matchers { m.allCommitOrigShas.toSet shouldEqual Set("a", "b", "c") } } + + // -- grouped transactions ---------------------------------------------------- + + /** What a second connection (another process, or the DB after a crash) sees. */ + private def committedCommits(db: java.nio.file.Path): Int = { + val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db") + try { + val rs = c.createStatement().executeQuery("SELECT count(*) FROM commit_map") + rs.next(); rs.getInt(1) + } finally c.close() + } + + private val Forever = Long.MaxValue + + test("a group commits once it holds maxBodies bodies, and not before") { + val db = Files.createTempFile("mapping-group-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(3, Forever)(m.putCommit("c1", "n1")) + m.inGroupedTx(3, Forever)(m.putCommit("c2", "n2")) + committedCommits(db) shouldBe 0 + m.getCommit("c2") shouldBe Some("n2") // the writer reads its own open group + m.inGroupedTx(3, Forever)(m.putCommit("c3", "n3")) + committedCommits(db) shouldBe 3 + } finally m.close() + } + + test("a group also commits once it is maxNanos old") { + val db = Files.createTempFile("mapping-group-age-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(1000, 0L)(m.putCommit("c1", "n1")) + committedCommits(db) shouldBe 1 + } finally m.close() + } + + test("a body that throws rolls back its whole group and nothing committed before") { + val db = Files.createTempFile("mapping-group-throw-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(2, Forever)(m.putCommit("c1", "n1")) + m.inGroupedTx(2, Forever)(m.putCommit("c2", "n2")) // group of 2 committed + m.inGroupedTx(2, Forever)(m.putCommit("c3", "n3")) // open group + an[IllegalStateException] should be thrownBy + m.inGroupedTx(2, Forever) { m.putTree("t4", "n4"); throw new IllegalStateException("boom") } + m.getCommit("c3") shouldBe None + m.getTree("t4") shouldBe None + m.getCommit("c2") shouldBe Some("n2") + // And the connection is usable afterwards. + m.inGroupedTx(2, Forever)(m.putCommit("c5", "n5")) + m.commitGroup() + committedCommits(db) shouldBe 3 + } finally m.close() + } + + test("inTx and close commit an open group first") { + val db = Files.createTempFile("mapping-group-close-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + m.inGroupedTx(10, Forever)(m.putCommit("c1", "n1")) + m.inTx(m.setMeta("k", "v")) + committedCommits(db) shouldBe 1 + m.inGroupedTx(10, Forever)(m.putCommit("c2", "n2")) + m.close() + committedCommits(db) shouldBe 2 + } + + test("a lost open group is all-or-nothing: no commit row without its trees and blobs") { + // A crash is a connection that never commits: simulate it by abandoning the + // process's view (a second connection reads only what was committed). + val db = Files.createTempFile("mapping-group-crash-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(5, Forever) { m.putBlob("b1", "a.c", "nb1"); m.putTree("t1", "nt1"); m.putCommit("c1", "n1") } + committedCommits(db) shouldBe 0 + val c = java.sql.DriverManager.getConnection(s"jdbc:sqlite:$db") + try { + val rs = c.createStatement().executeQuery( + "SELECT (SELECT count(*) FROM blob_map) + (SELECT count(*) FROM tree_map)") + rs.next(); rs.getInt(1) shouldBe 0 + } finally c.close() + } finally m.close() + } + + test("maxBodies of 1 is a transaction per body, as before") { + val db = Files.createTempFile("mapping-group-one-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*") + try { + m.inGroupedTx(1, Forever)(m.putCommit("c1", "n1")) + committedCommits(db) shouldBe 1 + } finally m.close() + } + + test("the SQLite sizes reach the connection") { + val db = Files.createTempFile("mapping-tuning-", ".db") + val m = Mapping.open(db, "/bin/cat", ".*", tuning = Mapping.Tuning(cacheMiB = 64, mmapMiB = 128)) + try { + m.pragmaValue("cache_size") shouldBe -64L * 1024L + m.pragmaValue("mmap_size") shouldBe 128L * 1024L * 1024L + } finally m.close() + val u = Mapping.open(db, "/bin/cat", ".*", tuning = Mapping.UntunedSqlite) + try u.pragmaValue("mmap_size") shouldBe 0L finally u.close() + } } diff --git a/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala b/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala new file mode 100644 index 00000000..66cba157 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/Sha1ImplementationSpec.scala @@ -0,0 +1,46 @@ +package cregit.blobexec + +import org.eclipse.jgit.lib.{Constants, ObjectInserter} +import org.eclipse.jgit.util.sha1.SHA1 +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +/** blobExec hashes with the JDK's SHA-1 unless told otherwise; ids must not change. */ +class Sha1ImplementationSpec extends AnyFunSuite with Matchers { + + private val P = Main.Sha1ImplementationProperty + + private def withProperty[A](value: Option[String])(body: => A): A = { + val saved = Option(System.getProperty(P)) + value match { case Some(v) => System.setProperty(P, v); case None => System.clearProperty(P) } + try body + finally saved match { case Some(v) => System.setProperty(P, v); case None => System.clearProperty(P) } + } + + private def idOf(bytes: Array[Byte]): String = + new ObjectInserter.Formatter().idFor(Constants.OBJ_BLOB, bytes).name + + private val samples: Seq[Array[Byte]] = + Seq("", "a", "int main(){return 0;}\n", "x" * 100000).map(_.getBytes("UTF-8")) :+ + Array.tabulate[Byte](70000)(i => (i * 31 % 251).toByte) + + test("unset, it becomes jdkNative; an operator's choice is kept") { + withProperty(None) { + Main.defaultSha1Implementation() + System.getProperty(P) shouldBe "jdkNative" + SHA1.newInstance().getClass.getSimpleName shouldBe "SHA1Native" + } + withProperty(Some("java")) { + Main.defaultSha1Implementation() + System.getProperty(P) shouldBe "java" + SHA1.newInstance().getClass.getSimpleName shouldBe "SHA1Java" + } + } + + test("both implementations give the same object ids") { + val java = withProperty(Some("java"))(samples.map(idOf)) + val native = withProperty(Some("jdkNative"))(samples.map(idOf)) + native shouldEqual java + java.head shouldBe "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" // git's empty blob + } +} diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 7ac200e4..5b4c5332 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -85,6 +85,9 @@ Output: with millions of loose objects. A failed pack never aborts the run: the Parquet dataset is the product, and packing only makes later steps faster. + Except while the cregit repo borrows objects (see + --copy-objects): that pack is required, and a failed one + stops the run, resumably, with a PACK-FAILED marker. --memory-limit SIZE cap the DuckDB heap in the dataset generator (step 10). Omit to accept that script's own default of 8GB. Takes an @@ -117,6 +120,22 @@ Tokenizer: --tokenizer-worker tokenize through a pool of persistent tokenizer processes; same output. Needs --mode pipeline or pipeline-trees (default: off) + --copy-objects + copy every untokenized original blob into the cregit repo during + step 2, as before (env CREGIT_COPY_OBJECTS=1 does the same). + By default step 2 copies none: the cregit repo reads them from + the original repo through objects/info/alternates, and the pack + after step 2 (git repack -a -d, without -l) copies them in once + and deletes that file. Same objects, half the step-2 work. + --gc none and --mode sharded always copy. + blobExec's other step-2 speed-ups are its defaults; opt out of + one through env CREGIT_BLOBEXEC_OPTS (blobExec flags, e.g. + "--commits-per-transaction=1 --sqlite-cache-mb=0 + --sqlite-mmap-mb=0 --loose-compression=-1") and + CREGIT_BLOBEXEC_JAVA_OPTS (JVM flags, e.g. + -Dorg.eclipse.jgit.util.sha1.implementation=java). + The pack of the cregit repo writes no bitmap index (nothing + reads one); env CREGIT_WRITE_BITMAPS=1 writes it again. --jobs N concurrent blame/HTML processes (default: CREGIT_JOBS, otherwise min(4, available CPUs)). Blame is the pipeline's bottleneck. Each file is independent, so the output does not @@ -144,9 +163,68 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" } +# True while dst reads objects from src (blobExec --alternates). `.packing` is +# the file set aside while fsck proves dst complete without it. +borrowing_objects() { + [ -e "$REPO_PATH_CREGIT_BARE/objects/info/alternates" ] \ + || [ -e "$REPO_PATH_CREGIT_BARE/objects/info/alternates.packing" ] +} + +# Copies the borrowed objects into dst and stops the borrowing. Not optional, as +# packing otherwise is: until this succeeds dst is unreadable without src. +# repack -a -d one pack of every reachable object, the borrowed ones too: +# WITHOUT -l, which would leave them out +# fsck run with the alternates file set aside, so it proves that dst +# is complete on its own; the file is deleted only after that +# A failure puts the file back (dst readable again), writes PACK-FAILED and stops +# the run; any resume retries this first. +dissolve_alternates() { + local dst="$REPO_PATH_CREGIT_BARE" alt="$REPO_PATH_CREGIT_BARE/objects/info/alternates" + local repack_args=(-a -d) + [ "$GC_MODE" = aggressive ] && repack_args+=(-f --depth=50 --window=250) + # A run killed during fsck left the file set aside: put it back first. + if [ -e "$alt.packing" ] && [ ! -e "$alt" ]; then + mv "$alt.packing" "$alt" + fi + log "packing $dst with the objects it borrows from $(cat "$alt")" + if git --git-dir="$dst" pack-refs --all \ + && git ${NO_BITMAP_CFG[@]+"${NO_BITMAP_CFG[@]}"} --git-dir="$dst" repack "${repack_args[@]}" \ + && mv "$alt" "$alt.packing" \ + && git --git-dir="$dst" fsck --connectivity-only --no-dangling; then + rm -f "$alt.packing" "${WORK}/PACK-FAILED" + # What gc does after its repack; neither one is needed for correctness. + git --git-dir="$dst" prune --expire=now \ + || log "warning: git prune failed; unreachable loose objects stay" + git --git-dir="$dst" commit-graph write --reachable \ + || log "warning: commit-graph write failed; later steps walk history more slowly" + log "pack done: $dst holds all its objects and no longer borrows any" + return 0 + fi + if [ -e "$alt.packing" ] && [ ! -e "$alt" ]; then + mv "$alt.packing" "$alt" + fi + local summary="the pack that copies the borrowed original blobs into $dst failed. + $dst still reads them from $REPO_PATH_ORIGINAL_BARE (objects/info/alternates), + so it is readable, but it must not outlive that repo. Free memory or disk." + write_resume_marker "PACK-FAILED" "git repack -a -d" 1 "$summary" + die "$summary + Marker: ${WORK}/PACK-FAILED + Resume at step 3: the runner retries the pack before anything else. + run_pipeline_process.sh: $0 [same flags] 3 + ctp.py (the external driver): python3 ctp.py run [same flags] --from-step 3" +} + # Packs the generated bare repo, as --gc selects. A gc failure only warns: under # `set -e` it would fire the EXIT trap, which deletes $WORK and every finished step. +# No bitmap index, here or in dissolve_alternates: git writes one by default for +# a bare repo, it serves only fetches and clones over a transport, and nothing +# does one (step 6 clones locally). It cost 1.4 h of Linux's 14.4 h gc. +# CREGIT_WRITE_BITMAPS=1 gives git its default back. pack_cregit_repo() { + if borrowing_objects; then + dissolve_alternates + return 0 + fi case "$GC_MODE" in none) log "gc skipped (--gc none)" @@ -158,7 +236,7 @@ pack_cregit_repo() { esac # shellcheck disable=SC2086 - if git --git-dir="$REPO_PATH_CREGIT_BARE" gc $gc_args; then + if git ${NO_BITMAP_CFG[@]+"${NO_BITMAP_CFG[@]}"} --git-dir="$REPO_PATH_CREGIT_BARE" gc $gc_args; then log "gc ($GC_MODE) done" else log "warning: gc ($GC_MODE) failed; the object store is still readable" @@ -222,8 +300,14 @@ RETOKENIZE="" BLOB_TIMEOUT="${CREGIT_BLOB_TIMEOUT:-}" STALL_TIMEOUT="${CREGIT_STALL_TIMEOUT:-}" TOKENIZER_WORKER=0 +# 1: step 2 copies the original blobs into dst instead of borrowing them. +COPY_OBJECTS="${CREGIT_COPY_OBJECTS:-0}" +# The packs of dst get no bitmap index (see pack_cregit_repo) unless +# CREGIT_WRITE_BITMAPS=1 asks for git's default. +NO_BITMAP_CFG=(-c repack.writeBitmaps=false) +if [ "${CREGIT_WRITE_BITMAPS:-0}" = 1 ]; then NO_BITMAP_CFG=(); fi -KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES" +KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES PACK-FAILED" keep_markers_present() { local m @@ -384,6 +468,7 @@ while [ $# -gt 0 ]; do --retokenize) need_val "$@"; RETOKENIZE="$2"; shift 2 ;; --blob-timeout) need_val "$@"; BLOB_TIMEOUT="$2"; shift 2 ;; --tokenizer-worker) TOKENIZER_WORKER=1; shift ;; + --copy-objects) COPY_OBJECTS=1; shift ;; --stall-timeout) need_val "$@"; STALL_TIMEOUT="$2"; shift 2 ;; --gc) need_val "$@"; GC_MODE="$2"; shift 2 ;; --memory-limit) need_val "$@"; MEMORY_LIMIT="$2"; shift 2 ;; @@ -877,12 +962,25 @@ else [ "$MASK_WIDENED" = 1 ] && WIDENED_FLAG="--mask-widened" WORKER_FLAG="" [ "$TOKENIZER_WORKER" = 1 ] && WORKER_FLAG="--tokenizer-worker=${CREGIT}/tokenizeByBlobId/tokenWorker.pl" + # Borrowing needs the pack that dissolves it, so --gc none copies instead. + ALTERNATES_FLAG="" + if [ "$COPY_OBJECTS" != 1 ] && [ "$GC_MODE" != none ]; then + ALTERNATES_FLAG="--alternates" + fi # blobExec refuses --retokenize without --memo-dir: the memo caches the same tokens. RETOKENIZE_FLAGS=() if [ -n "$RETOKENIZE" ]; then RETOKENIZE_FLAGS=("--retokenize=$RETOKENIZE" "--memo-dir=$MEMO_DIR") fi - java -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} \ + # Each holds several flags: split on blanks, but never glob-expanded. Logged, + # because a stray export would change every run without a trace. + read -r -a BLOBEXEC_OPTS <<< "${CREGIT_BLOBEXEC_OPTS:-}" + read -r -a BLOBEXEC_JAVA_OPTS <<< "${CREGIT_BLOBEXEC_JAVA_OPTS:-}" + [ ${#BLOBEXEC_OPTS[@]} -gt 0 ] && log "blobExec flags from CREGIT_BLOBEXEC_OPTS: ${BLOBEXEC_OPTS[*]}" + [ ${#BLOBEXEC_JAVA_OPTS[@]} -gt 0 ] && log "JVM flags from CREGIT_BLOBEXEC_JAVA_OPTS: ${BLOBEXEC_JAVA_OPTS[*]}" + # shellcheck disable=SC2086 + java ${BLOBEXEC_JAVA_OPTS[@]+"${BLOBEXEC_JAVA_OPTS[@]}"} -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} $ALTERNATES_FLAG \ + ${BLOBEXEC_OPTS[@]+"${BLOBEXEC_OPTS[@]}"} \ "--tokenizer-identity=$TOKENIZER_IDENTITY" \ ${RETOKENIZE_FLAGS[@]+"${RETOKENIZE_FLAGS[@]}"} \ ${BLOB_TIMEOUT:+--blob-timeout=$BLOB_TIMEOUT} \ @@ -915,6 +1013,13 @@ pack_cregit_repo fi end_step +# A resume past step 2 can find dst still borrowing: the pack failed or was +# killed. Steps 3+ must not start on that, so the pack runs first. +if [ "$FROM_STEP" -gt 2 ] && [ -d "$REPO_PATH_CREGIT_BARE" ] && borrowing_objects; then + log "resume: $REPO_PATH_CREGIT_BARE still borrows objects; packing it before step $FROM_STEP" + dissolve_alternates +fi + # --------------------------------------------------------------------------- # Step 3 — git log DB (original repo) # --------------------------------------------------------------------------- diff --git a/tests/pack_alternates.sh b/tests/pack_alternates.sh new file mode 100755 index 00000000..7ea50f68 --- /dev/null +++ b/tests/pack_alternates.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +# Tests step 2's borrowed objects: blobExec --alternates by default, the pack that +# must copy them into the cregit repo, and what happens when that pack fails. +# java is stubbed: as blobExec it builds a real dst that borrows src's objects. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNNER="$ROOT/run_pipeline_process.sh" +PASS=0 +FAIL=0 + +check() { # $1 = description, $2 = condition result (0/1) + if [ "$2" -eq 0 ]; then + echo " ok — $1"; PASS=$((PASS + 1)) + else + echo " FAIL — $1"; FAIL=$((FAIL + 1)) + fi +} + +# As blobExec: dst gets one new commit on src's tree. With --alternates it +# borrows that tree and its blobs; without, it copies them (git fetch). +# STUB_LOCK_PACK=1 then makes objects/pack read-only, so a repack fails. +# STUB_KILL_WALK=1 drops the ref again and SIGKILLs itself, as a killed walk. +make_stub_java() { # $1 = bin dir + cat > "$1/java" <<'STUB' +#!/usr/bin/env bash +[ -n "${STUB_ARGV:-}" ] && printf '%s\n' "$*" >> "$STUB_ARGV" +case "$*" in *blobExec*) ;; *) exit 0 ;; esac +pos=(); skip=0; alt=0 +for a in "$@"; do + if [ "$skip" = 1 ]; then skip=0; continue; fi + case "$a" in + -jar) skip=1 ;; + --alternates) alt=1 ;; + -*) ;; + *) pos+=("$a") ;; + esac +done +src=${pos[0]} dst=${pos[1]} +[ -d "$dst" ] || git init -q --bare "$dst" +if [ "$alt" = 1 ]; then + echo "$(cd "$src/objects" && pwd)" > "$dst/objects/info/alternates" +else + git --git-dir="$dst" fetch -q "$src" "+refs/heads/*:refs/remotes/src/*" +fi +tree=$(git --git-dir="$src" rev-parse 'HEAD^{tree}') +c=$(echo rewritten | GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@x GIT_COMMITTER_NAME=t \ + GIT_COMMITTER_EMAIL=t@x GIT_AUTHOR_DATE='@1700000000 +0000' GIT_COMMITTER_DATE='@1700000000 +0000' \ + git --git-dir="$dst" commit-tree "$tree") +git --git-dir="$dst" update-ref refs/heads/master "$c" +git --git-dir="$dst" for-each-ref --format='%(refname)' refs/remotes | while read -r r; do + git --git-dir="$dst" update-ref -d "$r"; done +[ "${STUB_LOCK_PACK:-0}" = 1 ] && chmod a-w "$dst/objects/pack" +# A walk killed mid-way: dst borrows and holds a new object, but no ref yet. +if [ "${STUB_KILL_WALK:-0}" = 1 ]; then + git --git-dir="$dst" update-ref -d refs/heads/master + kill -9 $$ +fi +exit 0 +STUB + chmod +x "$1/java" +} + +# The runner takes CREGIT from $(pwd): tokenize/ for the identity, plus build +# files and placeholder artifacts, or the staleness guard would build. +repo_fixture() { + local f p + f=$(mktemp -d "${TMPDIR:-/tmp}/packrepo-XXXXXX") + cp "$RUNNER" "$f/run_pipeline_process.sh" + cp -r "$ROOT/tokenize" "$f/tokenize" + rm -rf "$f/tokenize/rustTokenizer/target" "$f/tokenize/srcMLtoken/srcml2token" + for p in blobExec slickGitLog persons remapCommits; do + mkdir -p "$f/$p/src" + cp "$ROOT/$p/build.sbt" "$f/$p/build.sbt" + cp -r "$ROOT/$p/project" "$f/$p/project" + rm -rf "$f/$p/project/target" "$f/$p/project/project" + done + for p in tokenize/srcMLtoken/srcml2token \ + tokenize/rustTokenizer/target/release/rust_tokenizer \ + blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar \ + slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar \ + persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar \ + remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar + do + mkdir -p "$f/$(dirname "$p")" + echo "placeholder v1" > "$f/$p" + chmod +x "$f/$p" + done + printf '#!/bin/sh\necho "%s/libsrcml"\n' "$f" > "$f/tokenize/srcMLtoken/srcml2token" + echo "placeholder v1" > "$f/libsrcml" + printf '%s' "$f" +} + +# A work dir resumable at step 2: a real original bare repo with two commits. +fixture() { + local w t + w=$(mktemp -d "${TMPDIR:-/tmp}/packwork-XXXXXX") + t="$w/tmp-src" + git init -q "$t" + echo "int a;" > "$t/a.c"; echo "readme" > "$t/README" + git -C "$t" add . && git -C "$t" -c user.name=t -c user.email=t@x commit -q -m one + echo "readme 2" > "$t/README" + git -C "$t" -c user.name=t -c user.email=t@x commit -q -am two + git clone -q --bare "$t" "$w/proj-original.git" + rm -rf "$t" + mkdir -p "$w/memo" + printf '%s' "$w" +} + +run_from() { # $1 = work dir, rest = extra runner args + FROM_STEP + local w=$1; shift + ( cd "$REPO" \ + && PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 timeout 300 \ + bash ./run_pipeline_process.sh \ + --repo-url /nonexistent/does-not-exist.git \ + --repo-name proj \ + --work "$w" \ + --skip-html --mask '\.c$' \ + "$@" 2>&1 ) +} + +DST=proj-cregit.git + +# True when dst holds every object it references, without any alternates file. +self_contained() { # $1 = dst + [ ! -e "$1/objects/info/alternates" ] && [ ! -e "$1/objects/info/alternates.packing" ] \ + && git --git-dir="$1" fsck --connectivity-only --no-dangling >/dev/null 2>&1 \ + && git --git-dir="$1" cat-file -e 'master:README' +} + +BIN=$(mktemp -d "${TMPDIR:-/tmp}/packbin-XXXXXX") +make_stub_java "$BIN" +REPO=$(repo_fixture) +trap 'rm -rf "$BIN" "$REPO"' EXIT + +# --------------------------------------------------------------------------- +echo "case 1: by default step 2 borrows, and the pack leaves dst whole on its own" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_ARGV="$ARGV" run_from "$W" 2) +grep -q 'Step 3' <<<"$OUT"; check "step 2 completed and the pipeline moved on" $? +grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; check "blobExec was given --alternates" $? +grep -q 'no longer borrows any' <<<"$OUT"; check "the runner packed the borrowed objects in" $? +self_contained "$W/$DST"; check "dst has no alternates file and fsck passes without it" $? +git --git-dir="$W/$DST" count-objects -v | grep -q '^count: 0' +check "every object is packed, none left loose" $? +[ ! -e "$W/PACK-FAILED" ]; check "no PACK-FAILED marker" $? +! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null; check "and no bitmap index" $? +git clone -q "$W/$DST" "$W/clone" && [ -f "$W/clone/README" ] && [ ! -e "$W/clone/.git/objects/info/alternates" ] +check "a step-6 style clone reads every file and borrows nothing" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 2: --copy-objects, CREGIT_COPY_OBJECTS=1 and --gc none all copy instead" +for how in flag env gcnone; do + W=$(fixture); ARGV="$W/argv.log" + case $how in + flag) OUT=$(STUB_ARGV="$ARGV" run_from "$W" --copy-objects 2) ;; + env) OUT=$(CREGIT_COPY_OBJECTS=1 STUB_ARGV="$ARGV" run_from "$W" 2) ;; + gcnone) OUT=$(STUB_ARGV="$ARGV" run_from "$W" --gc none 2) ;; + esac + grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; [ $? -ne 0 ] + check "$how: blobExec was NOT given --alternates" $? + [ ! -e "$W/$DST/objects/info/alternates" ] && git --git-dir="$W/$DST" cat-file -e 'master:README' + check "$how: dst holds its objects without borrowing" $? + if [ "$how" = flag ]; then + compgen -G "$W/$DST/objects/pack/*.pack" >/dev/null \ + && ! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null + check "$how: gc packed dst, without a bitmap index" $? + fi + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "case 2b: CREGIT_BLOBEXEC_OPTS and CREGIT_BLOBEXEC_JAVA_OPTS reach blobExec" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(CREGIT_BLOBEXEC_OPTS="--commits-per-transaction=1 --loose-compression=-1" \ + CREGIT_BLOBEXEC_JAVA_OPTS="-Dorg.eclipse.jgit.util.sha1.implementation=java -XX:ActiveProcessorCount=3" \ + STUB_ARGV="$ARGV" run_from "$W" 2) +grep blobExec "$ARGV" | grep -q -- '--commits-per-transaction=1 --loose-compression=-1 ' +check "the blobExec flags, before the positional arguments" $? +grep blobExec "$ARGV" | grep -q -- '^-Dorg.eclipse.jgit.util.sha1.implementation=java -XX:ActiveProcessorCount=3 -jar ' +check "the JVM flags, before -jar" $? +grep -q 'blobExec flags from CREGIT_BLOBEXEC_OPTS: --commits-per-transaction=1 --loose-compression=-1' <<<"$OUT" \ + && grep -q 'JVM flags from CREGIT_BLOBEXEC_JAVA_OPTS: -Dorg' <<<"$OUT" +check "and both are logged" $? +rm -rf "$W" +# The runner runs in $REPO: a file there is what an unquoted --x=* would match. +W=$(fixture); ARGV="$W/argv.log"; touch "$REPO/--x=bait" +OUT=$(CREGIT_BLOBEXEC_OPTS="--x=*" STUB_ARGV="$ARGV" run_from "$W" 2) +rm -f "$REPO/--x=bait" +grep blobExec "$ARGV" | grep -qF -- ' --x=* '; check "a * in the flags is passed as is, not glob-expanded" $? +OUT=$(STUB_ARGV="$W/argv2.log" run_from "$W" 2) +! grep -q 'from CREGIT_BLOBEXEC' <<<"$OUT"; check "unset, nothing is logged" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 3: a failed pack stops the run, resumably, and leaves dst readable" +W=$(fixture) +OUT=$(STUB_LOCK_PACK=1 run_from "$W" 2); RC=$? +[ "$RC" -ne 0 ]; check "the run failed (got exit $RC)" $? +grep -q 'Step 3' <<<"$OUT"; [ $? -ne 0 ]; check "and stopped before step 3" $? +[ -f "$W/PACK-FAILED" ]; check "a PACK-FAILED marker was written" $? +[ -f "$W/$DST/objects/info/alternates" ]; check "the alternates file is still in place" $? +git --git-dir="$W/$DST" cat-file -e 'master:README'; check "so dst still reads the borrowed blobs" $? +chmod u+w "$W/$DST/objects/pack" +OUT=$(run_from "$W" 3) +grep -q 'still borrows objects; packing it before step 3' <<<"$OUT" +check "a resume at step 3 packs first" $? +self_contained "$W/$DST"; check "and dst is then whole on its own" $? +[ ! -e "$W/PACK-FAILED" ]; check "and the marker is gone" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 4: a run killed while fsck ran (file set aside) is put back and redone" +W=$(fixture) +OUT=$(STUB_LOCK_PACK=1 run_from "$W" 2) +chmod u+w "$W/$DST/objects/pack" +mv "$W/$DST/objects/info/alternates" "$W/$DST/objects/info/alternates.packing" +OUT=$(run_from "$W" 4) +grep -q 'packing it before step 4' <<<"$OUT"; check "a resume at step 4 packs first" $? +self_contained "$W/$DST"; check "and dst is whole, with neither file left" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 5: a full run (FROM_STEP=1) keeps the work dir when the pack fails" +W=$(mktemp -d "${TMPDIR:-/tmp}/packfull-XXXXXX") +S=$(fixture) +OUT=$( cd "$REPO" && STUB_LOCK_PACK=1 PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 timeout 300 \ + bash ./run_pipeline_process.sh --repo-url "$S/proj-original.git" --repo-name proj \ + --work "$W/work" --skip-html --mask '\.c$' 2>&1 ); RC=$? +[ "$RC" -ne 0 ] && [ -f "$W/work/PACK-FAILED" ] && [ -d "$W/work/$DST" ] +check "the EXIT trap kept \$WORK: the marker says it is resumable (exit $RC)" $? +chmod -R u+w "$W" "$S"; rm -rf "$W" "$S" + +# --------------------------------------------------------------------------- +echo "case 6: --gc aggressive borrows too, and its repack leaves dst whole" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_ARGV="$ARGV" run_from "$W" --gc aggressive 2) +grep 'blobExec' "$ARGV" | grep -q -- '--alternates'; check "blobExec was given --alternates" $? +grep -q 'no longer borrows any' <<<"$OUT" && grep -q 'Step 3' <<<"$OUT" +check "the runner packed the borrowed objects in and moved on" $? +self_contained "$W/$DST"; check "dst has no alternates file and fsck passes without it" $? +! compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null; check "and no bitmap index" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 6b: CREGIT_WRITE_BITMAPS=1 brings the bitmap index back, on both paths" +for how in borrow copy; do + W=$(fixture) + case $how in + borrow) OUT=$(CREGIT_WRITE_BITMAPS=1 run_from "$W" 2) ;; + copy) OUT=$(CREGIT_WRITE_BITMAPS=1 run_from "$W" --copy-objects 2) ;; + esac + grep -q 'Step 3' <<<"$OUT" && compgen -G "$W/$DST/objects/pack/*.bitmap" >/dev/null + check "$how: the pack has a bitmap index" $? + git --git-dir="$W/$DST" cat-file -e 'master:README'; check "$how: dst reads its files" $? + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "case 7: a walk killed mid-way leaves dst borrowing; a step-2 resume finishes it" +W=$(fixture) +OUT=$(STUB_KILL_WALK=1 run_from "$W" 2); RC=$? +[ "$RC" -ne 0 ] && ! grep -q 'Step 3' <<<"$OUT"; check "the run failed in step 2 (exit $RC)" $? +[ -f "$W/$DST/objects/info/alternates" ] && [ -d "$W/proj-original.git" ] +check "dst still borrows, and src is kept" $? +OUT=$(run_from "$W" 2) +grep -q 'no longer borrows any' <<<"$OUT" && grep -q 'Step 3' <<<"$OUT" +check "the resume ran step 2 again, packed, and moved on" $? +self_contained "$W/$DST"; check "dst is whole on its own" $? +git clone -q "$W/$DST" "$W/clone" && [ -f "$W/clone/README" ] && [ ! -e "$W/clone/.git/objects/info/alternates" ] +check "a step-6 style clone reads every file and borrows nothing" $? +rm -rf "$W" + +echo "" +echo "pack_alternates: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] diff --git a/tests/pipeline_workdir_guard.sh b/tests/pipeline_workdir_guard.sh index 11229242..36dccc83 100755 --- a/tests/pipeline_workdir_guard.sh +++ b/tests/pipeline_workdir_guard.sh @@ -247,7 +247,7 @@ echo "case 17: without the flag nothing changes, and the flag is what reaches bl grep -q 'WIDENED_FLAG=""' "$RUNNER"; check "the flag defaults to absent" $? grep -q '\[ "\$MASK_WIDENED" = 1 \] && WIDENED_FLAG="--mask-widened"' "$RUNNER" check "and is only added when asked for" $? -grep -q 'java -jar "\$BFG" \$MODE_FLAG \$WIDENED_FLAG' "$RUNNER" +grep -qF 'java ${BLOBEXEC_JAVA_OPTS[@]+"${BLOBEXEC_JAVA_OPTS[@]}"} -jar "$BFG" $MODE_FLAG $WIDENED_FLAG' "$RUNNER" check "so it reaches blobExec's argv" $? echo