Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ jobs:
- name: Pipeline tests (shell)
run: |
devenv shell -- bash -c 'set -e
for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do
for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough pack_alternates; do
bash "tests/$t.sh"
done'

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ cd ../..
bash tests/test_token_worker.sh
prove tests/t
pytest -q
for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do
for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough pack_alternates; do
bash "tests/$t.sh"
done

Expand Down
105 changes: 100 additions & 5 deletions blobExec/src/main/scala/cregit/blobexec/Main.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ object Main {
/** Not 3 (memo mismatch): a no-op request must be told apart from a refusal. */
private[blobexec] val RetokenizeIneffectiveExitStatus = 7

/** zlib level 1 for dst's loose objects: they are temporary, because the pack
* after step 2 re-deflates every one at git's pack.compression. Measured -11%
* consumer CPU on psi-probe against JGit's default level. */
private[blobexec] val DefaultLooseCompression = 1

private val UsageExitStatus = 1
private val RefusedExitStatus = 3

Expand Down Expand Up @@ -74,6 +79,16 @@ object Main {
| --tokenizer-identity and --memo-dir; a no-op is refused (status ${RetokenizeIneffectiveExitStatus})
| --memo-dir=<dir> the tokenizer memo ($$BFG_MEMO_DIR) that --retokenize purges
| --tokenizer-worker=<path> tokenize through a pool of persistent workers; needs --pipeline(-trees)
| --alternates write <dst.git>/objects/info/alternates naming <src.git>'s objects
| and copy no original blob; the caller must then repack <dst.git>
| without -l and delete that file. Not with --shard
| --commits-per-transaction=<n>
| commits whose mapping rows share one SQLite transaction
| (default ${Walker.DefaultCommitsPerTransaction}; 1 = one per commit)
| --sqlite-cache-mb=<n> SQLite page cache for the mapping DB (default ${Mapping.DefaultCacheMiB}; 0 = SQLite's 2 MB)
| --sqlite-mmap-mb=<n> memory-map up to this much of the mapping DB (default ${Mapping.DefaultMmapMiB}; 0 = off)
| --loose-compression=<n> zlib level of the loose objects written to <dst.git>, in memory only
| (default ${DefaultLooseCompression}; -1 = JGit's default, 6). git repacks them anyway
| --blob-timeout=<seconds> budget for one <command> run (default ${BlobExec.DefaultTimeoutSeconds}); then its process
| tree is killed and the blob excluded
| --stall-timeout=<seconds> exit ${Walker.StalledExitStatus} after this long with no progress (default ${Walker.DefaultStallTimeoutSeconds});
Expand Down Expand Up @@ -103,6 +118,10 @@ object Main {
retokenize: Set[String] = Set.empty,
memoDir: Option[Path] = None,
tokenizerWorker: Option[Path] = None,
alternates: Boolean = false,
commitsPerTransaction: Int = Walker.DefaultCommitsPerTransaction,
sqlite: Mapping.Tuning = Mapping.Tuning(),
looseCompression: Int = DefaultLooseCompression,
blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds,
stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds,
stallExplicit: Boolean = false,
Expand Down Expand Up @@ -136,6 +155,8 @@ object Main {
"into, or the invalidation would leave the real memo's stale entries in place."

val PipelinesExclusive = withUsage("Error: --pipeline and --pipeline-trees are mutually exclusive")
val AlternatesUnderShard = withUsage(
"Error: --alternates cannot be used with --shard: shard_merge.py copies each shard's own objects")
val ShardWithPipeline = withUsage(
"Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees")
val MaskWidenedUnderShard = withUsage(
Expand Down Expand Up @@ -222,10 +243,23 @@ object Main {
private val TokenizerWorkerFlag = new ValuedFlag("--tokenizer-worker")
private val BlobTimeoutFlag = new ValuedFlag("--blob-timeout")
private val StallTimeoutFlag = new ValuedFlag("--stall-timeout")
private val CommitsPerTxFlag = new ValuedFlag("--commits-per-transaction")
private val SqliteCacheFlag = new ValuedFlag("--sqlite-cache-mb")
private val SqliteMmapFlag = new ValuedFlag("--sqlite-mmap-mb")
private val LooseCompressionFlag = new ValuedFlag("--loose-compression")

private def flagValue[A](flag: ValuedFlag, spec: String)(parse: String => Either[String, A]): Either[String, A] =
parse(spec).left.map(Messages.badFlagValue(flag.name, _))

private def positive(spec: String): Either[String, Int] =
spec.toIntOption.filter(_ > 0).toRight(s"must be a positive whole number [$spec]")

private def mebibytes(spec: String): Either[String, Int] =
spec.toIntOption.filter(_ >= 0).toRight(s"must be a whole number of MiB, 0 or more [$spec]")

private def level(spec: String): Either[String, Int] =
spec.toIntOption.filter(n => n >= -1 && n <= 9).toRight(s"must be a zlib level, -1 to 9 [$spec]")

private def seconds(spec: String): Either[String, Int] =
parsePositiveSeconds(spec).toRight(Messages.secondsWanted(spec))

Expand All @@ -249,6 +283,7 @@ object Main {
case "--pipeline" => Right(o.copy(pipeline = true))
case "--pipeline-trees" => Right(o.copy(pipelineTrees = true))
case "--mask-widened" => Right(o.copy(maskWidened = true))
case "--alternates" => Right(o.copy(alternates = true))
case TokenizerIdentityFlag(spec) =>
flagValue(TokenizerIdentityFlag, spec)(TokenizerIdentity.parse).map(id => o.copy(tokenizerIdentity = id))
case RetokenizeFlag(spec) =>
Expand All @@ -265,6 +300,14 @@ object Main {
Right(o.copy(tokenizerWorker = Some(Paths.get(spec))))
case BlobTimeoutFlag(spec) =>
flagValue(BlobTimeoutFlag, spec)(seconds).map(secs => o.copy(blobTimeoutSeconds = secs))
case CommitsPerTxFlag(spec) =>
flagValue(CommitsPerTxFlag, spec)(positive).map(n => o.copy(commitsPerTransaction = n))
case SqliteCacheFlag(spec) =>
flagValue(SqliteCacheFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(cacheMiB = n)))
case SqliteMmapFlag(spec) =>
flagValue(SqliteMmapFlag, spec)(mebibytes).map(n => o.copy(sqlite = o.sqlite.copy(mmapMiB = n)))
case LooseCompressionFlag(spec) =>
flagValue(LooseCompressionFlag, spec)(level).map(n => o.copy(looseCompression = n))
case StallTimeoutFlag(spec) =>
flagValue(StallTimeoutFlag, spec)(seconds).map(secs => o.copy(stallTimeoutSeconds = secs, stallExplicit = true))
case other => Left(Messages.unknownFlag(other))
Expand All @@ -285,6 +328,7 @@ object Main {
(retokenizing && o.shard.isDefined) -> (() => Messages.RetokenizeUnderShard),
(!retokenizing && o.memoDir.isDefined) -> (() => Messages.MemoDirWithoutRetokenize),
(o.shard.isDefined && o.anyPipeline) -> (() => Messages.ShardWithPipeline),
(o.shard.isDefined && o.alternates) -> (() => Messages.AlternatesUnderShard),
o.tokenizerWorker.exists(p => !Files.isRegularFile(p) || !Files.isExecutable(p)) ->
(() => Messages.workerNotExecutable(o.tokenizerWorker.get)),
(o.tokenizerWorker.isDefined && !o.anyPipeline) -> (() => Messages.WorkerNeedsPipeline)
Expand All @@ -311,7 +355,20 @@ object Main {
} yield full
}

/** JGit hashes with its own Java SHA-1, which also runs collision detection
* (SHA1DC); the JDK's intrinsic one is faster (-5% on psi-probe). Both give the
* same id for every input that is not a crafted SHA-1 collision; on one of
* those the Java one throws instead. Set before JGit hashes anything, and only
* when the operator did not choose: -Dorg.eclipse.jgit.util.sha1.implementation=java
* keeps the old one. JGit reads the property on every new hasher. */
private[blobexec] val Sha1ImplementationProperty = "org.eclipse.jgit.util.sha1.implementation"

private[blobexec] def defaultSha1Implementation(): Unit =
if (System.getProperty(Sha1ImplementationProperty) == null)
System.setProperty(Sha1ImplementationProperty, "jdkNative")

def main(args: Array[String]): Unit = {
defaultSha1Implementation()
val options = parse(args.toVector) match {
case Right(o) => o
case Left(why) => exit(why, UsageExitStatus)
Expand Down Expand Up @@ -339,7 +396,8 @@ object Main {
println(startLine(o, stallSeconds, incremental))

val src = openSrc(o.src)
val dst = openOrInitDst(o.dst)
val dst = openOrInitDst(o.dst, borrowFrom = Option.when(o.alternates)(src.getObjectsDirectory.toPath))
setLooseCompression(dst, o.looseCompression)
val mapping = openMapping(o, src, dst) match {
case Right(m) => m
case Left(refusal) =>
Expand Down Expand Up @@ -368,13 +426,17 @@ object Main {
s"maskWidened=${o.maskWidened} denylistEntries=${BlobDenylist.shipped.size} " +
s"tokenizerIdentity=${if (o.tokenizerIdentity.isEmpty) "none" else o.tokenizerIdentity.render} " +
s"retokenize=${if (o.retokenize.isEmpty) "none" else o.retokenize.toVector.sorted.mkString(",")} " +
s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)}"
s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)} " +
s"alternates=${o.alternates} commitsPerTransaction=${o.commitsPerTransaction} " +
s"sqliteCacheMiB=${o.sqlite.cacheMiB} sqliteMmapMiB=${o.sqlite.mmapMiB} " +
s"looseCompression=${o.looseCompression} sha1=${System.getProperty(Sha1ImplementationProperty)}"
}

private final case class Refusal(status: Int, message: String)

private def openMapping(o: Options, src: FileRepository, dst: FileRepository): Either[Refusal, Mapping] =
Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst)))
Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst),
o.sqlite))
.toEither
.left.map(e => refusalFor(e, o).getOrElse(throw e))

Expand Down Expand Up @@ -436,9 +498,11 @@ object Main {
src, dst, mapping, o.mask.r, o.command, o.abortOnError, parallelism,
o.pipeline, o.pipelineTrees, o.shard,
destinationMayContainObjects = incremental,
borrowOriginalObjects = o.alternates,
blobTimeoutSeconds = o.blobTimeoutSeconds,
stallTimeoutSeconds = stallSeconds,
workerPool = workerPool,
commitsPerTransaction = o.commitsPerTransaction,
denylist = BlobDenylist.shipped
).run()
finally workerPool.foreach(_.close())
Expand All @@ -456,6 +520,7 @@ object Main {
s"originalBlobDestinationLookups=${stats.originalBlobDestinationLookups} " +
s"originalBlobBytesCopied=${stats.originalBlobBytesCopied} " +
s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " +
s"originalBlobsBorrowed=${stats.originalBlobsBorrowed} " +
s"refsProjected=${stats.refsProjected} " +
s"blobsTimedOut=${stats.blobsTimedOut} " +
s"blobsOversized=${stats.blobsOversized} " +
Expand All @@ -480,10 +545,40 @@ object Main {
FileRepositoryBuilder.create(gitDir).asInstanceOf[FileRepository]
}

private def openOrInitDst(path: Path): FileRepository = {
/** `borrowFrom` names src's objects directory: it is written to dst's
* objects/info/alternates BEFORE dst is opened for the walk, because JGit reads
* that file once, and a walk that skipped a copy must find the object there. */
private[blobexec] def openOrInitDst(path: Path, borrowFrom: Option[Path] = None): FileRepository = {
val exists = Files.isDirectory(path)
val repo = FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository]
if (!exists) repo.create(true)
repo
borrowFrom match {
case None => repo
case Some(objects) =>
val info = repo.getObjectsDirectory.toPath.resolve("info")
repo.close()
Files.createDirectories(info)
writeAlternates(info.resolve("alternates"), objects)
FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository]
}
}

/** In memory only: dst's config file is not touched, so nothing that reads dst
* later (git repack, steps 3+) can see it. Object ids do not depend on it. JGit
* reads it when an inserter is created; a reload of the file would only bring
* back the default level, a speed loss, never a different object. */
private[blobexec] def setLooseCompression(dst: FileRepository, level: Int): Unit = {
val config = dst.getConfig // loads the file first, if it changed
config.setInt("core", null, "compression", level)
}

/** One absolute line, replacing whatever was there: a resume writes the same
* file again. Written to a temporary file and renamed, so git never reads half. */
private[blobexec] def writeAlternates(file: Path, objects: Path): Unit = {
val line = objects.toAbsolutePath.normalize.toString + "\n"
val tmp = file.resolveSibling(file.getFileName.toString + ".tmp")
Files.write(tmp, line.getBytes(java.nio.charset.StandardCharsets.UTF_8))
Files.move(tmp, file, java.nio.file.StandardCopyOption.REPLACE_EXISTING,
java.nio.file.StandardCopyOption.ATOMIC_MOVE)
}
}
92 changes: 90 additions & 2 deletions blobExec/src/main/scala/cregit/blobexec/Mapping.scala
Original file line number Diff line number Diff line change
Expand Up @@ -296,8 +296,10 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends
Mapping.RetokenizeResult(before, rowCounts, extensions, invalidated, memo)
}

/** Run `body` inside a transaction; commit on success, rollback on throw. */
/** Run `body` inside a transaction; commit on success, rollback on throw.
* An open group (see [[inGroupedTx]]) is committed first. */
def inTx[A](body: => A): A = {
commitGroup()
conn.setAutoCommit(false)
try {
val result = body
Expand All @@ -312,7 +314,67 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends
}
}

/** A PRAGMA's current value on this connection, for tests. */
private[blobexec] def pragmaValue(name: String): Long = {
val st = conn.createStatement()
try { val rs = st.executeQuery(s"PRAGMA $name"); rs.next(); rs.getLong(1) } finally st.close()
}

// -- grouped transactions ---------------------------------------------------
// A commit's rows are one body; up to `maxBodies` bodies share one transaction.
// Each SQLite transaction rewrites the pages it touched into the WAL, and the
// walk touches the same index pages for every commit, so per-commit
// transactions cost more than the rows. A group is all-or-nothing, like the
// one-commit transaction it replaces: a crash loses whole bodies, never part of
// one, so a commit row is still never durable without its trees and blobs, and
// the lost commits are re-walked on resume (their tokens come from the memo).

private var groupOpen = false
private var groupBodies = 0
private var groupStartNanos = 0L

/** As [[inTx]], but `body` joins the open group, which is committed once it
* holds `maxBodies` bodies or is `maxNanos` old. A throw in `body` rolls back
* the whole group. `maxBodies` <= 1 is plain [[inTx]]. Not thread-safe: the
* caller serializes access, as it does for every other method here. */
def inGroupedTx[A](maxBodies: Int, maxNanos: Long)(body: => A): A =
if (maxBodies <= 1) inTx(body)
else {
if (!groupOpen) {
conn.setAutoCommit(false)
groupOpen = true
groupBodies = 0
groupStartNanos = System.nanoTime()
}
val result =
try body
catch { case t: Throwable => abandonGroup(); throw t }
groupBodies += 1
if (groupBodies >= maxBodies || System.nanoTime() - groupStartNanos >= maxNanos) commitGroup()
result
}

/** Commit the open group, if any. The walk calls it when it stops, by abort too. */
def commitGroup(): Unit =
if (groupOpen) {
try conn.commit()
catch { case t: Throwable => abandonGroup(); throw t }
groupOpen = false
groupBodies = 0
conn.setAutoCommit(true)
}

private def abandonGroup(): Unit =
if (groupOpen) {
groupOpen = false
groupBodies = 0
try conn.rollback() catch { case _: Throwable => () }
try conn.setAutoCommit(true) catch { case _: Throwable => () }
}

override def close(): Unit = {
// Every body in the group finished (a failed one rolled the group back).
commitGroup()
(List(selBlob, insBlob, selCommit, insCommit, selTree, insTree,
selRef, insRef, delRef, selMeta, insMeta) ++ warmSelBlob.toList ++ warmSelTree.toList)
.foreach(s => try s.close() catch { case _: Throwable => () })
Expand Down Expand Up @@ -456,6 +518,30 @@ object Mapping {
} finally st.close()
}

/** Read-path sizing for the walk's DB. Both change speed only, never results.
* cacheMiB SQLite's own page cache, private anonymous memory, filled only
* as pages are read: the DB's size at most (default 2 MB).
* mmapMiB map up to this much of the file. Lookups then read the kernel's
* page cache in place instead of copying each page out with a
* syscall. Those are file pages: shared and reclaimable, and the
* same pages that reads cache anyway. 0 turns it off. */
final case class Tuning(cacheMiB: Int = DefaultCacheMiB, mmapMiB: Int = DefaultMmapMiB) {
require(cacheMiB >= 0 && mmapMiB >= 0, s"SQLite sizes must be >= 0 (cache=$cacheMiB MiB, mmap=$mmapMiB MiB)")
}
val DefaultCacheMiB = 256
val DefaultMmapMiB = 4096
/** SQLite's defaults: what every run used before. */
val UntunedSqlite: Tuning = Tuning(cacheMiB = 0, mmapMiB = 0)

private def applyTuning(conn: Connection, t: Tuning): Unit = {
val st = conn.createStatement()
try {
// Negative cache_size is KiB; 0 keeps the default.
if (t.cacheMiB > 0) st.execute(s"PRAGMA cache_size = -${t.cacheMiB.toLong * 1024L}")
if (t.mmapMiB > 0) st.execute(s"PRAGMA mmap_size = ${t.mmapMiB.toLong * 1024L * 1024L}")
} finally st.close()
}

/** Opt-in for `--mask-widened`. `newBlobResolves` has no default: `new_blob` ids
* live only in dst, and a wiped dst would dangle every reused row. */
final case class MaskWidening(newBlobResolves: String => Boolean,
Expand All @@ -468,7 +554,8 @@ object Mapping {
def open(path: Path, command: String, mask: String, warm: Option[Path] = None,
maskWidening: Option[MaskWidening] = None,
tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty,
retokenize: Option[Retokenize] = None): Mapping = {
retokenize: Option[Retokenize] = None,
tuning: Tuning = Tuning()): Mapping = {
// Refused, not composed: each invalidation is verified against a state the
// other would change. Run one, then the other as a resume.
require(!(maskWidening.isDefined && retokenize.isDefined),
Expand All @@ -492,6 +579,7 @@ object Mapping {
val url = s"jdbc:sqlite:${path.toAbsolutePath}"
val conn = DriverManager.getConnection(url)
enableForeignKeys(conn)
applyTuning(conn, tuning)
val st = conn.createStatement()
try Schema.foreach(st.execute) finally st.close()

Expand Down
Loading
Loading