Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions .github/workflows/linux-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Builds and publishes Linux x86_64 binaries (tarball + .deb + .rpm) on every
# GitHub release. The AUR -bin package is a maintainer push from the same
# tarball (see packaging/PKGBUILD-bin): bump pkgver + updpkgsums per release.
name: Linux binaries (x86_64)

on:
release:
types: [published]
# Validates the Linux port on every PR (build + test + package, no upload).
pull_request:
paths:
- 'Sources/**'
- 'Tests/**'
- 'Package.swift'
- 'Package.resolved'
- 'scripts/build-llama.sh'
- 'scripts/package-linux.sh'
- '.github/workflows/linux-release.yml'
workflow_dispatch:
inputs:
tag:
description: Release tag, like v0.7.0
required: true

permissions:
contents: write # upload dist/ to the release

jobs:
x86_64:
runs-on: ubuntu-24.04
env:
TAG: ${{ github.event.release.tag_name || inputs.tag }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ env.TAG }}
- name: System deps
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq cmake ninja-build libsqlite3-dev ffmpeg poppler-utils rpm dpkg-dev curl
- name: Swift toolchain
uses: swift-actions/setup-swift@v2
with:
swift-version: "6.2"
- name: Version check
run: |
[[ "$TAG" =~ ^v[0-9]+(\.[0-9]+)*$ ]] || { echo "Not a release tag: $TAG"; exit 1; }
swift --version
- name: Build llama.cpp (Linux CPU, portable)
run: scripts/build-llama.sh
- name: Build + test (CLI engine only)
run: |
swift build -c release --product digup
swift test --filter DigUpKitTests
- name: Package tarball + .deb
run: scripts/package-linux.sh --version "${TAG#v}" --no-build
- name: Package .rpm from the tarball
run: |
mkdir -p ~/rpmbuild/SOURCES
cp "dist/digup-${TAG#v}-x86_64.tar.gz" ~/rpmbuild/SOURCES/
rpmbuild -bb --define "version ${TAG#v}" packaging/digup.spec \
--define "_topdir $HOME/rpmbuild"
cp ~/rpmbuild/RPMS/x86_64/digup-*.rpm dist/
(cd dist && sha256sum digup-*.tar.gz digup_*.deb digup-*.rpm > SHA256SUMS)
- name: Upload to the release
if: github.event_name == 'release'
env:
GH_TOKEN: ${{ github.token }}
run: gh release upload "$TAG" dist/* --clobber
- name: Build artifacts (dispatch runs and review)
uses: actions/upload-artifact@v4
with:
name: digup-linux-x86_64
path: dist/*
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
.build/
.swiftpm/
build.noindex/
dist/
/Vendor/

# Dev runs (scripts/dev-run.sh): their indexes, models and settings
Expand Down
19 changes: 14 additions & 5 deletions Package.resolved

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

120 changes: 87 additions & 33 deletions Package.swift
Original file line number Diff line number Diff line change
@@ -1,40 +1,94 @@
// swift-tools-version: 6.2
import PackageDescription

// Linux (x86_64 first): only the `digup` CLI and its engine are portable.
// The menubar app (AppKit/SwiftUI/Sparkle) stays macOS-only and is appended below.
var products: [Product] = [
.executable(name: "digup", targets: ["digup"]),
.library(name: "DigUpKit", targets: ["DigUpKit"]),
]
var dependencies: [Package.Dependency] = []

#if os(Linux)
let digUpKitDeps: [Target.Dependency] = ["CSQLite", .product(name: "Crypto", package: "swift-crypto")]
#else
let digUpKitDeps: [Target.Dependency] = []
#endif

#if os(Linux)
// Linux links llama.cpp's original static archives directly: several members
// share basenames across archives, so no merge can hold them all, and the
// archives reference each other cyclically (--start-group). scripts/build-llama.sh
// must run first (it builds Vendor/llama-build and Vendor/llama/include).
let _llamaDir = Context.packageDirectory + "/Vendor/llama-build"
let llamaLinkerSettings: [LinkerSetting] = [
.unsafeFlags(["-L\(Context.packageDirectory)/Vendor/llama/lib"]),
.unsafeFlags(["-Xlinker", "--start-group",
"\(_llamaDir)/src/libllama.a",
"\(_llamaDir)/tools/mtmd/libmtmd.a",
"\(_llamaDir)/ggml/src/libggml.a",
"\(_llamaDir)/ggml/src/libggml-base.a",
"\(_llamaDir)/ggml/src/libggml-cpu.a",
"\(_llamaDir)/vendor/hash/libvendor-hash.a",
"-Xlinker", "--end-group"]),
.linkedLibrary("stdc++"),
.linkedLibrary("gomp"),
]
#else
let llamaLinkerSettings: [LinkerSetting] = [
.unsafeFlags(["-L\(Context.packageDirectory)/Vendor/llama/lib"]),
// Moved out of Sources/CLlama/module.modulemap (modulemaps have no
// conditionals, and `link framework` / `-lc++` break Linux linking).
.linkedFramework("Accelerate"),
.linkedFramework("Metal"),
.linkedFramework("MetalKit"),
.linkedLibrary("c++"),
.linkedLibrary("llamacpp"),
]
#endif
var targets: [Target] = [
// The engine: crawler, extractors, index store, the embedder interface and worker client, search. No UI.
.target(name: "DigUpKit", dependencies: digUpKitDeps),
// llama.cpp (scripts/build-llama.sh builds it into Vendor/llama) and EmbeddingGemma 2 on it.
.systemLibrary(name: "CLlama", path: "Sources/CLlama"),
.target(
name: "LlamaRuntime",
dependencies: ["CLlama", "DigUpKit"],
linkerSettings: llamaLinkerSettings
),
// Developer CLI: estimate / index / search / status / eval. Doubles as the eval harness, and the app runs it as
// its indexing helper and its query encoder.
.executableTarget(name: "digup", dependencies: ["DigUpKit", "LlamaRuntime"]),
.testTarget(name: "DigUpKitTests", dependencies: ["DigUpKit"]),
.testTarget(name: "LlamaRuntimeTests", dependencies: ["LlamaRuntime"]),
]

#if os(macOS)
products.append(.executable(name: "DigUpApp", targets: ["DigUpApp"]))
// In-app updates (the app only). build.sh embeds the framework in Contents/Frameworks and signs it.
dependencies.append(.package(url: "https://github.com/sparkle-project/Sparkle", exact: "2.10.0"))
targets += [
// The menubar + hotkey app. build.sh wraps it into build.noindex/DigUp.app. (The product can't be called
// "DigUp": on a case-insensitive disk it would collide with the `digup` CLI binary.)
.executableTarget(
name: "DigUpApp",
dependencies: ["DigUpKit", .product(name: "Sparkle", package: "Sparkle")],
swiftSettings: [.defaultIsolation(MainActor.self)],
linkerSettings: [.unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@executable_path/../Frameworks"])]
),
]
#else
// Linux SQLite headers (macOS gets them from the SDK). Needs libsqlite3-dev (apt) / sqlite (Arch).
targets.append(.systemLibrary(name: "CSQLite", path: "Sources/CSQLite", pkgConfig: "sqlite3",
providers: [.apt(["libsqlite3-dev"])]))
// Linux CryptoKit (SHA-256 for model verification): swift-crypto's Crypto module has the same API.
dependencies.append(.package(url: "https://github.com/apple/swift-crypto", from: "3.0.0"))
#endif

let package = Package(
name: "DigUp",
platforms: [.macOS(.v14)],
products: [
.executable(name: "digup", targets: ["digup"]),
.executable(name: "DigUpApp", targets: ["DigUpApp"]),
.library(name: "DigUpKit", targets: ["DigUpKit"]),
],
dependencies: [
// In-app updates (the app only). build.sh embeds the framework in Contents/Frameworks and signs it.
.package(url: "https://github.com/sparkle-project/Sparkle", exact: "2.10.0"),
],
targets: [
// The engine: crawler, extractors, index store, the embedder interface and worker client, search. No UI.
.target(name: "DigUpKit"),
// llama.cpp (scripts/build-llama.sh builds it into Vendor/llama) and EmbeddingGemma 2 on it.
.systemLibrary(name: "CLlama", path: "Sources/CLlama"),
.target(
name: "LlamaRuntime",
dependencies: ["CLlama", "DigUpKit"],
linkerSettings: [.unsafeFlags(["-L\(Context.packageDirectory)/Vendor/llama/lib"])]
),
// Developer CLI: estimate / index / search / status / eval. Doubles as the eval harness, and the app runs it as
// its indexing helper and its query encoder.
.executableTarget(name: "digup", dependencies: ["DigUpKit", "LlamaRuntime"]),
// The menubar + hotkey app. build.sh wraps it into build.noindex/DigUp.app. (The product can't be called
// "DigUp": on a case-insensitive disk it would collide with the `digup` CLI binary.)
.executableTarget(
name: "DigUpApp",
dependencies: ["DigUpKit", .product(name: "Sparkle", package: "Sparkle")],
swiftSettings: [.defaultIsolation(MainActor.self)],
linkerSettings: [.unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@executable_path/../Frameworks"])]
),
.testTarget(name: "DigUpKitTests", dependencies: ["DigUpKit"]),
.testTarget(name: "LlamaRuntimeTests", dependencies: ["LlamaRuntime"]),
]
products: products,
dependencies: dependencies,
targets: targets
)
10 changes: 4 additions & 6 deletions Sources/CLlama/module.modulemap
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,9 @@ module CLlama [system] {
header "../../Vendor/llama/include/mtmd.h"
header "../../Vendor/llama/include/mtmd-helper.h"
header "../../Vendor/llama/include/llama-build.h"
link "llamacpp"
link "c++"
link framework "Accelerate"
link framework "Foundation"
link framework "Metal"
link framework "MetalKit"
// NOTE: linking lives in the LlamaRuntime target's linkerSettings in
// Package.swift (modulemaps have no per-OS conditionals): -lllamacpp and
// the C++/system frameworks on macOS, the original static archives with
// --start-group plus stdc++/gomp on Linux.
export *
}
5 changes: 5 additions & 0 deletions Sources/CSQLite/module.modulemap
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module CSQLite [system] {
header "/usr/include/sqlite3.h"
link "sqlite3"
export *
}
39 changes: 39 additions & 0 deletions Sources/DigUpKit/Crawler.swift
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import Foundation
#if canImport(UniformTypeIdentifiers)
import UniformTypeIdentifiers
#endif
#if os(Linux)
import Glibc
#endif

/// A file the crawler decided to index.
public struct Candidate: Sendable {
Expand Down Expand Up @@ -27,8 +32,13 @@ public enum Crawler {
public static func crawl(_ roots: [URL], options: IndexOptions) -> CrawlReport {
let fm = FileManager.default
let home = fm.homeDirectoryForCurrentUser.path
#if os(macOS)
let keys: [URLResourceKey] = [.isDirectoryKey, .isPackageKey, .contentTypeKey, .isUbiquitousItemKey,
.ubiquitousItemDownloadingStatusKey]
#else
// Linux: no content-type or iCloud metadata keys.
let keys: [URLResourceKey] = [.isDirectoryKey, .isPackageKey]
#endif
var report = CrawlReport()
var projectRoots = Set<String>()
var repos = GitRepos()
Expand Down Expand Up @@ -69,8 +79,13 @@ public enum Crawler {
report.skipped["app or bundle", default: 0] += 1
return
}
#if os(macOS)
let classification = options.code ? Rules.classifyCode(url, options: options)
: Rules.classify(url, type: values?.contentType, options: options)
#else
let classification = options.code ? Rules.classifyCode(url, options: options)
: Rules.classify(url, options: options)
#endif
switch classification {
case .ignore:
report.skipped["other type", default: 0] += 1
Expand All @@ -83,12 +98,14 @@ public enum Crawler {
return
}
// Reading a dataless placeholder (iCloud "Optimize Mac Storage", File Providers) downloads it.
#if os(macOS)
let dataless = info.st_flags & 0x4000_0000 != 0 // SF_DATALESS
let inCloud = values?.isUbiquitousItem == true && values?.ubiquitousItemDownloadingStatus != .current
if dataless || inCloud {
report.skipped["in the cloud, not downloaded", default: 0] += 1
return
}
#endif
if options.code {
// Too big to be written by hand; a notebook's file holds its outputs too.
let limit = url.pathExtension.lowercased() == "ipynb" ? options.maxNotebookBytes
Expand All @@ -103,7 +120,11 @@ public enum Crawler {
}
}
guard foundPaths.insert(url.path).inserted else { return }
#if os(macOS)
let mtime = Double(info.st_mtimespec.tv_sec) + Double(info.st_mtimespec.tv_nsec) / 1e9
#else
let mtime = Double(info.st_mtim.tv_sec) + Double(info.st_mtim.tv_nsec) / 1e9
#endif
found.append(Candidate(path: url.path, kind: kind, size: Int64(info.st_size), mtime: mtime,
inode: UInt64(info.st_ino), device: UInt64(bitPattern: Int64(info.st_dev))))
}
Expand Down Expand Up @@ -213,6 +234,23 @@ enum GitFiles {
/// A git that's really installed (Xcode's, the command line tools', Homebrew's). /usr/bin/git on a Mac without the
/// developer tools only asks to install them.
static let git: String? = {
#if os(Linux)
let candidates = ["/usr/bin/git", "/usr/local/bin/git"]
if let found = candidates.first(where: { FileManager.default.isExecutableFile(atPath: $0) }) { return found }
// Anything else on PATH (nix, ~/.local/bin).
let probe = Process()
probe.executableURL = URL(fileURLWithPath: "/usr/bin/env")
probe.arguments = ["sh", "-c", "command -v git"]
let output = Pipe()
probe.standardOutput = output
probe.standardError = FileHandle.nullDevice
probe.standardInput = FileHandle.nullDevice
guard (try? probe.run()) != nil else { return nil }
let path = String(data: output.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8)?
.trimmingCharacters(in: .whitespacesAndNewlines)
probe.waitUntilExit()
return path.flatMap { $0.isEmpty ? nil : $0 }
#else
var candidates: [String] = []
if let developer = try? FileManager.default.destinationOfSymbolicLink(atPath: "/var/db/xcode_select_link") {
candidates.append(developer + "/usr/bin/git")
Expand All @@ -221,6 +259,7 @@ enum GitFiles {
"/Applications/Xcode.app/Contents/Developer/usr/bin/git", "/opt/homebrew/bin/git",
"/usr/local/bin/git"]
return candidates.first { FileManager.default.isExecutableFile(atPath: $0) }
#endif
}()

/// Paths relative to `repo`, or nil when git can't say (no git, not a repo, an error).
Expand Down
4 changes: 4 additions & 0 deletions Sources/DigUpKit/Database.swift
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import Foundation
#if canImport(SQLite3)
import SQLite3
#elseif canImport(CSQLite)
import CSQLite
#endif

/// A thin wrapper over the system SQLite (3.51 on macOS 26, FTS5 built in). One connection, used from one thread.
public final class Database {
Expand Down
Loading