diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml new file mode 100644 index 00000000..382f6fbe --- /dev/null +++ b/.github/workflows/linux-release.yml @@ -0,0 +1,74 @@ +# Builds and publishes Linux x86_64 binaries (tarball + .deb + .rpm) on every +# GitHub release. The AUR -bin package is a maintainer push from the same +# tarball (see packaging/PKGBUILD-bin): bump pkgver + updpkgsums per release. +name: Linux binaries (x86_64) + +on: + release: + types: [published] + # Validates the Linux port on every PR (build + test + package, no upload). + pull_request: + paths: + - 'Sources/**' + - 'Tests/**' + - 'Package.swift' + - 'Package.resolved' + - 'scripts/build-llama.sh' + - 'scripts/package-linux.sh' + - '.github/workflows/linux-release.yml' + workflow_dispatch: + inputs: + tag: + description: Release tag, like v0.7.0 + required: true + +permissions: + contents: write # upload dist/ to the release + +jobs: + x86_64: + runs-on: ubuntu-24.04 + env: + TAG: ${{ github.event.release.tag_name || inputs.tag }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.TAG }} + - name: System deps + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq cmake ninja-build libsqlite3-dev ffmpeg poppler-utils rpm dpkg-dev curl + - name: Swift toolchain + uses: swift-actions/setup-swift@v2 + with: + swift-version: "6.2" + - name: Version check + run: | + [[ "$TAG" =~ ^v[0-9]+(\.[0-9]+)*$ ]] || { echo "Not a release tag: $TAG"; exit 1; } + swift --version + - name: Build llama.cpp (Linux CPU, portable) + run: scripts/build-llama.sh + - name: Build + test (CLI engine only) + run: | + swift build -c release --product digup + swift test --filter DigUpKitTests + - name: Package tarball + .deb + run: scripts/package-linux.sh --version "${TAG#v}" --no-build + - name: Package .rpm from the tarball + run: | + mkdir -p ~/rpmbuild/SOURCES + cp "dist/digup-${TAG#v}-x86_64.tar.gz" ~/rpmbuild/SOURCES/ + rpmbuild -bb --define "version ${TAG#v}" packaging/digup.spec \ + --define "_topdir $HOME/rpmbuild" + cp ~/rpmbuild/RPMS/x86_64/digup-*.rpm dist/ + (cd dist && sha256sum digup-*.tar.gz digup_*.deb digup-*.rpm > SHA256SUMS) + - name: Upload to the release + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ github.token }} + run: gh release upload "$TAG" dist/* --clobber + - name: Build artifacts (dispatch runs and review) + uses: actions/upload-artifact@v4 + with: + name: digup-linux-x86_64 + path: dist/* diff --git a/.gitignore b/.gitignore index 7a1bb706..cddda8dd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .build/ .swiftpm/ build.noindex/ +dist/ /Vendor/ # Dev runs (scripts/dev-run.sh): their indexes, models and settings diff --git a/Package.resolved b/Package.resolved index a715522c..27ae9c3b 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,13 +1,22 @@ { - "originHash" : "23ef967e422558ab1161cd009bb19dc5cb4b48f087f3d588c0e253fedf19490f", + "originHash" : "5bd155cab56dc5e105f8bddde92f8f84fa7a7bc8e181afd916891073f4dc4f54", "pins" : [ { - "identity" : "sparkle", + "identity" : "swift-asn1", "kind" : "remoteSourceControl", - "location" : "https://github.com/sparkle-project/Sparkle", + "location" : "https://github.com/apple/swift-asn1.git", "state" : { - "revision" : "eef1a539a373c1f1a320624b1130fc5de7b2e100", - "version" : "2.10.0" + "revision" : "3b6410f7dee09eb33cdd26260c5fd47fda19b0e2", + "version" : "1.7.3" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto", + "state" : { + "revision" : "95ba0316a9b733e92bb6b071255ff46263bbe7dc", + "version" : "3.15.1" } } ], diff --git a/Package.swift b/Package.swift index 91709b06..d9b11ecb 100644 --- a/Package.swift +++ b/Package.swift @@ -1,40 +1,94 @@ // swift-tools-version: 6.2 import PackageDescription +// Linux (x86_64 first): only the `digup` CLI and its engine are portable. +// The menubar app (AppKit/SwiftUI/Sparkle) stays macOS-only and is appended below. +var products: [Product] = [ + .executable(name: "digup", targets: ["digup"]), + .library(name: "DigUpKit", targets: ["DigUpKit"]), +] +var dependencies: [Package.Dependency] = [] + +#if os(Linux) +let digUpKitDeps: [Target.Dependency] = ["CSQLite", .product(name: "Crypto", package: "swift-crypto")] +#else +let digUpKitDeps: [Target.Dependency] = [] +#endif + +#if os(Linux) +// Linux links llama.cpp's original static archives directly: several members +// share basenames across archives, so no merge can hold them all, and the +// archives reference each other cyclically (--start-group). scripts/build-llama.sh +// must run first (it builds Vendor/llama-build and Vendor/llama/include). +let _llamaDir = Context.packageDirectory + "/Vendor/llama-build" +let llamaLinkerSettings: [LinkerSetting] = [ + .unsafeFlags(["-L\(Context.packageDirectory)/Vendor/llama/lib"]), + .unsafeFlags(["-Xlinker", "--start-group", + "\(_llamaDir)/src/libllama.a", + "\(_llamaDir)/tools/mtmd/libmtmd.a", + "\(_llamaDir)/ggml/src/libggml.a", + "\(_llamaDir)/ggml/src/libggml-base.a", + "\(_llamaDir)/ggml/src/libggml-cpu.a", + "\(_llamaDir)/vendor/hash/libvendor-hash.a", + "-Xlinker", "--end-group"]), + .linkedLibrary("stdc++"), + .linkedLibrary("gomp"), +] +#else +let llamaLinkerSettings: [LinkerSetting] = [ + .unsafeFlags(["-L\(Context.packageDirectory)/Vendor/llama/lib"]), + // Moved out of Sources/CLlama/module.modulemap (modulemaps have no + // conditionals, and `link framework` / `-lc++` break Linux linking). + .linkedFramework("Accelerate"), + .linkedFramework("Metal"), + .linkedFramework("MetalKit"), + .linkedLibrary("c++"), + .linkedLibrary("llamacpp"), +] +#endif +var targets: [Target] = [ + // The engine: crawler, extractors, index store, the embedder interface and worker client, search. No UI. + .target(name: "DigUpKit", dependencies: digUpKitDeps), + // llama.cpp (scripts/build-llama.sh builds it into Vendor/llama) and EmbeddingGemma 2 on it. + .systemLibrary(name: "CLlama", path: "Sources/CLlama"), + .target( + name: "LlamaRuntime", + dependencies: ["CLlama", "DigUpKit"], + linkerSettings: llamaLinkerSettings + ), + // Developer CLI: estimate / index / search / status / eval. Doubles as the eval harness, and the app runs it as + // its indexing helper and its query encoder. + .executableTarget(name: "digup", dependencies: ["DigUpKit", "LlamaRuntime"]), + .testTarget(name: "DigUpKitTests", dependencies: ["DigUpKit"]), + .testTarget(name: "LlamaRuntimeTests", dependencies: ["LlamaRuntime"]), +] + +#if os(macOS) +products.append(.executable(name: "DigUpApp", targets: ["DigUpApp"])) +// In-app updates (the app only). build.sh embeds the framework in Contents/Frameworks and signs it. +dependencies.append(.package(url: "https://github.com/sparkle-project/Sparkle", exact: "2.10.0")) +targets += [ + // The menubar + hotkey app. build.sh wraps it into build.noindex/DigUp.app. (The product can't be called + // "DigUp": on a case-insensitive disk it would collide with the `digup` CLI binary.) + .executableTarget( + name: "DigUpApp", + dependencies: ["DigUpKit", .product(name: "Sparkle", package: "Sparkle")], + swiftSettings: [.defaultIsolation(MainActor.self)], + linkerSettings: [.unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@executable_path/../Frameworks"])] + ), +] +#else +// Linux SQLite headers (macOS gets them from the SDK). Needs libsqlite3-dev (apt) / sqlite (Arch). +targets.append(.systemLibrary(name: "CSQLite", path: "Sources/CSQLite", pkgConfig: "sqlite3", + providers: [.apt(["libsqlite3-dev"])])) +// Linux CryptoKit (SHA-256 for model verification): swift-crypto's Crypto module has the same API. +dependencies.append(.package(url: "https://github.com/apple/swift-crypto", from: "3.0.0")) +#endif + let package = Package( name: "DigUp", platforms: [.macOS(.v14)], - products: [ - .executable(name: "digup", targets: ["digup"]), - .executable(name: "DigUpApp", targets: ["DigUpApp"]), - .library(name: "DigUpKit", targets: ["DigUpKit"]), - ], - dependencies: [ - // In-app updates (the app only). build.sh embeds the framework in Contents/Frameworks and signs it. - .package(url: "https://github.com/sparkle-project/Sparkle", exact: "2.10.0"), - ], - targets: [ - // The engine: crawler, extractors, index store, the embedder interface and worker client, search. No UI. - .target(name: "DigUpKit"), - // llama.cpp (scripts/build-llama.sh builds it into Vendor/llama) and EmbeddingGemma 2 on it. - .systemLibrary(name: "CLlama", path: "Sources/CLlama"), - .target( - name: "LlamaRuntime", - dependencies: ["CLlama", "DigUpKit"], - linkerSettings: [.unsafeFlags(["-L\(Context.packageDirectory)/Vendor/llama/lib"])] - ), - // Developer CLI: estimate / index / search / status / eval. Doubles as the eval harness, and the app runs it as - // its indexing helper and its query encoder. - .executableTarget(name: "digup", dependencies: ["DigUpKit", "LlamaRuntime"]), - // The menubar + hotkey app. build.sh wraps it into build.noindex/DigUp.app. (The product can't be called - // "DigUp": on a case-insensitive disk it would collide with the `digup` CLI binary.) - .executableTarget( - name: "DigUpApp", - dependencies: ["DigUpKit", .product(name: "Sparkle", package: "Sparkle")], - swiftSettings: [.defaultIsolation(MainActor.self)], - linkerSettings: [.unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@executable_path/../Frameworks"])] - ), - .testTarget(name: "DigUpKitTests", dependencies: ["DigUpKit"]), - .testTarget(name: "LlamaRuntimeTests", dependencies: ["LlamaRuntime"]), - ] + products: products, + dependencies: dependencies, + targets: targets ) diff --git a/Sources/CLlama/module.modulemap b/Sources/CLlama/module.modulemap index 1f19a1a8..31423129 100644 --- a/Sources/CLlama/module.modulemap +++ b/Sources/CLlama/module.modulemap @@ -3,11 +3,9 @@ module CLlama [system] { header "../../Vendor/llama/include/mtmd.h" header "../../Vendor/llama/include/mtmd-helper.h" header "../../Vendor/llama/include/llama-build.h" - link "llamacpp" - link "c++" - link framework "Accelerate" - link framework "Foundation" - link framework "Metal" - link framework "MetalKit" + // NOTE: linking lives in the LlamaRuntime target's linkerSettings in + // Package.swift (modulemaps have no per-OS conditionals): -lllamacpp and + // the C++/system frameworks on macOS, the original static archives with + // --start-group plus stdc++/gomp on Linux. export * } diff --git a/Sources/CSQLite/module.modulemap b/Sources/CSQLite/module.modulemap new file mode 100644 index 00000000..721c678a --- /dev/null +++ b/Sources/CSQLite/module.modulemap @@ -0,0 +1,5 @@ +module CSQLite [system] { + header "/usr/include/sqlite3.h" + link "sqlite3" + export * +} diff --git a/Sources/DigUpKit/Crawler.swift b/Sources/DigUpKit/Crawler.swift index c03bdba6..3e335bcd 100644 --- a/Sources/DigUpKit/Crawler.swift +++ b/Sources/DigUpKit/Crawler.swift @@ -1,5 +1,10 @@ import Foundation +#if canImport(UniformTypeIdentifiers) import UniformTypeIdentifiers +#endif +#if os(Linux) +import Glibc +#endif /// A file the crawler decided to index. public struct Candidate: Sendable { @@ -27,8 +32,13 @@ public enum Crawler { public static func crawl(_ roots: [URL], options: IndexOptions) -> CrawlReport { let fm = FileManager.default let home = fm.homeDirectoryForCurrentUser.path + #if os(macOS) let keys: [URLResourceKey] = [.isDirectoryKey, .isPackageKey, .contentTypeKey, .isUbiquitousItemKey, .ubiquitousItemDownloadingStatusKey] + #else + // Linux: no content-type or iCloud metadata keys. + let keys: [URLResourceKey] = [.isDirectoryKey, .isPackageKey] + #endif var report = CrawlReport() var projectRoots = Set() var repos = GitRepos() @@ -69,8 +79,13 @@ public enum Crawler { report.skipped["app or bundle", default: 0] += 1 return } + #if os(macOS) let classification = options.code ? Rules.classifyCode(url, options: options) : Rules.classify(url, type: values?.contentType, options: options) + #else + let classification = options.code ? Rules.classifyCode(url, options: options) + : Rules.classify(url, options: options) + #endif switch classification { case .ignore: report.skipped["other type", default: 0] += 1 @@ -83,12 +98,14 @@ public enum Crawler { return } // Reading a dataless placeholder (iCloud "Optimize Mac Storage", File Providers) downloads it. + #if os(macOS) let dataless = info.st_flags & 0x4000_0000 != 0 // SF_DATALESS let inCloud = values?.isUbiquitousItem == true && values?.ubiquitousItemDownloadingStatus != .current if dataless || inCloud { report.skipped["in the cloud, not downloaded", default: 0] += 1 return } + #endif if options.code { // Too big to be written by hand; a notebook's file holds its outputs too. let limit = url.pathExtension.lowercased() == "ipynb" ? options.maxNotebookBytes @@ -103,7 +120,11 @@ public enum Crawler { } } guard foundPaths.insert(url.path).inserted else { return } + #if os(macOS) let mtime = Double(info.st_mtimespec.tv_sec) + Double(info.st_mtimespec.tv_nsec) / 1e9 + #else + let mtime = Double(info.st_mtim.tv_sec) + Double(info.st_mtim.tv_nsec) / 1e9 + #endif found.append(Candidate(path: url.path, kind: kind, size: Int64(info.st_size), mtime: mtime, inode: UInt64(info.st_ino), device: UInt64(bitPattern: Int64(info.st_dev)))) } @@ -213,6 +234,23 @@ enum GitFiles { /// A git that's really installed (Xcode's, the command line tools', Homebrew's). /usr/bin/git on a Mac without the /// developer tools only asks to install them. static let git: String? = { + #if os(Linux) + let candidates = ["/usr/bin/git", "/usr/local/bin/git"] + if let found = candidates.first(where: { FileManager.default.isExecutableFile(atPath: $0) }) { return found } + // Anything else on PATH (nix, ~/.local/bin). + let probe = Process() + probe.executableURL = URL(fileURLWithPath: "/usr/bin/env") + probe.arguments = ["sh", "-c", "command -v git"] + let output = Pipe() + probe.standardOutput = output + probe.standardError = FileHandle.nullDevice + probe.standardInput = FileHandle.nullDevice + guard (try? probe.run()) != nil else { return nil } + let path = String(data: output.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) + probe.waitUntilExit() + return path.flatMap { $0.isEmpty ? nil : $0 } + #else var candidates: [String] = [] if let developer = try? FileManager.default.destinationOfSymbolicLink(atPath: "/var/db/xcode_select_link") { candidates.append(developer + "/usr/bin/git") @@ -221,6 +259,7 @@ enum GitFiles { "/Applications/Xcode.app/Contents/Developer/usr/bin/git", "/opt/homebrew/bin/git", "/usr/local/bin/git"] return candidates.first { FileManager.default.isExecutableFile(atPath: $0) } + #endif }() /// Paths relative to `repo`, or nil when git can't say (no git, not a repo, an error). diff --git a/Sources/DigUpKit/Database.swift b/Sources/DigUpKit/Database.swift index f2eea3df..b9100b5c 100644 --- a/Sources/DigUpKit/Database.swift +++ b/Sources/DigUpKit/Database.swift @@ -1,5 +1,9 @@ import Foundation +#if canImport(SQLite3) import SQLite3 +#elseif canImport(CSQLite) +import CSQLite +#endif /// A thin wrapper over the system SQLite (3.51 on macOS 26, FTS5 built in). One connection, used from one thread. public final class Database { diff --git a/Sources/DigUpKit/Estimator.swift b/Sources/DigUpKit/Estimator.swift index e0f6d797..47ff636e 100644 --- a/Sources/DigUpKit/Estimator.swift +++ b/Sources/DigUpKit/Estimator.swift @@ -1,5 +1,9 @@ +#if canImport(CoreGraphics) import CoreGraphics +#endif +#if canImport(CoreServices) import CoreServices +#endif import Foundation /// Seconds of indexing per unit, measured 2026-10-07 on an M4 Pro with llama.cpp (Q8_0), reading files while the @@ -30,6 +34,8 @@ public struct Estimate: Sendable { public var files: [FileKind: Int] = [:] public var pdfPages = 0 // pages the first pass will read public var pdfPagesTotal = 0 + public var slidePages = 0 // slides the first pass will read + public var slidePagesTotal = 0 public var audioSeconds = 0.0 public var videoSeconds = 0.0 public var docChunks = 0 // chunks the first pass will read @@ -49,6 +55,8 @@ public struct Estimate: Sendable { sum.files.merge(b.files, uniquingKeysWith: +) sum.pdfPages += b.pdfPages sum.pdfPagesTotal += b.pdfPagesTotal + sum.slidePages += b.slidePages + sum.slidePagesTotal += b.slidePagesTotal sum.audioSeconds += b.audioSeconds sum.videoSeconds += b.videoSeconds sum.docChunks += b.docChunks @@ -82,6 +90,18 @@ public enum Estimator { estimate.longFiles += 1 estimate.laterSeconds += Double(all - first) * costs.pdfPage } + case .slides: + // Slide count without converting (zip directory for .pptx); legacy .ppt gets a size guess. + let total = SlideExtractor.slideCount(of: URL(fileURLWithPath: file.path)) ?? max(1, Int(file.size / 50000)) + let all = min(total, options.maxPages) + let first = total > options.bigPDFPages ? min(options.pdfStep, all) : all + estimate.slidePagesTotal += total + estimate.slidePages += first + estimate.seconds += Double(first) * costs.pdfPage + if all > first { + estimate.longFiles += 1 + estimate.laterSeconds += Double(all - first) * costs.pdfPage + } case .doc: let all = max(1, min(options.maxChunks, Int(file.size / 2000))) let first = min(all, options.docStep) @@ -131,17 +151,69 @@ public enum Estimator { } static func pages(of path: String) -> Int { + #if os(macOS) if let pages = spotlight(path, kMDItemNumberOfPages), pages > 0 { return Int(pages) } return CGPDFDocument(URL(fileURLWithPath: path) as CFURL)?.numberOfPages ?? 1 + #else + // Linux: `pdfinfo` (poppler-utils) when present, else a size heuristic. + if let pages = pdfinfoPages(path), pages > 0 { return pages } + let size = (try? FileManager.default.attributesOfItem(atPath: path)[.size] as? Int) ?? 0 + return max(1, size / 3000) + #endif } static func duration(of path: String) -> Double { + #if os(macOS) if let seconds = spotlight(path, kMDItemDurationSeconds), seconds > 0 { return seconds } return MediaInfo.duration(of: URL(fileURLWithPath: path)) ?? 0 + #else + // Linux: `ffprobe` when present (AVFoundation's MediaInfo is macOS-only). + return ffprobeDuration(path) ?? 0 + #endif } + #if os(macOS) private static func spotlight(_ path: String, _ attribute: CFString) -> Double? { guard let item = MDItemCreate(kCFAllocatorDefault, path as CFString) else { return nil } return (MDItemCopyAttribute(item, attribute) as? NSNumber)?.doubleValue } + #else + /// Page count via poppler's `pdfinfo` ("Pages: N"), nil when it's missing or fails. + private static func pdfinfoPages(_ path: String) -> Int? { + guard let output = runHelper("/usr/bin/pdfinfo", [path]) + ?? runHelper("/usr/bin/env", ["pdfinfo", path]) else { return nil } + for line in output.split(separator: "\n") where line.hasPrefix("Pages:") { + if let pages = Int(line.dropFirst("Pages:".count).trimmingCharacters(in: .whitespaces)), pages > 0 { + return pages + } + } + return nil + } + + /// Media duration via `ffprobe`, nil when it's missing or fails. + private static func ffprobeDuration(_ path: String) -> Double? { + guard let output = runHelper("/usr/bin/ffprobe", + ["-v", "error", "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", path]) + ?? runHelper("/usr/bin/env", ["ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", path]) else { return nil } + return Double(output.trimmingCharacters(in: .whitespacesAndNewlines)) + } + + private static func runHelper(_ executable: String, _ arguments: [String]) -> String? { + guard FileManager.default.isExecutableFile(atPath: executable) || executable.hasSuffix("/env") else { return nil } + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = arguments + let output = Pipe() + process.standardOutput = output + process.standardError = FileHandle.nullDevice + process.standardInput = FileHandle.nullDevice + guard (try? process.run()) != nil else { return nil } + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + guard process.terminationStatus == 0 else { return nil } + return String(data: data, encoding: .utf8) + } + #endif } diff --git a/Sources/DigUpKit/Extractors.swift b/Sources/DigUpKit/Extractors.swift index 1bed1023..f7fd0549 100644 --- a/Sources/DigUpKit/Extractors.swift +++ b/Sources/DigUpKit/Extractors.swift @@ -1,9 +1,11 @@ -import AppKit import Foundation +#if os(macOS) +import AppKit import ImageIO import PDFKit import UniformTypeIdentifiers import Vision +#endif enum ExtractError: Error, CustomStringConvertible { case unreadable(String) @@ -24,6 +26,7 @@ enum ExtractError: Error, CustomStringConvertible { } } +#if os(macOS) enum ImageExtractor { struct Prepared: Sendable { let path: String @@ -157,12 +160,95 @@ enum PDFExtractor { .cgImage(forProposedRect: nil, context: nil, hints: nil) } } +#endif + +/// Presentation decks: converted to PDF with LibreOffice (`soffice`), then read +/// as PDFs. Portable (a `Process` call); only Linux classifies decks for indexing. +enum SlideExtractor { + /// Converts a deck to PDF in `folder` and returns it. Throws when LibreOffice + /// is missing or the conversion fails. + static func toPDF(_ url: URL, to folder: URL) throws -> URL { + let soffice: URL + let prefix: [String] + if FileManager.default.isExecutableFile(atPath: "/usr/bin/soffice") { + soffice = URL(fileURLWithPath: "/usr/bin/soffice") + prefix = [] + } else { + soffice = URL(fileURLWithPath: "/usr/bin/env") + prefix = ["soffice"] + } + let process = Process() + process.executableURL = soffice + process.arguments = prefix + ["--headless", "--convert-to", "pdf", "--outdir", folder.path, url.path] + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + process.standardInput = FileHandle.nullDevice + do { + try process.run() + } catch { + throw ExtractError.unreadable("can't run soffice (LibreOffice installed?)") + } + process.waitUntilExit() + let pdf = folder.appendingPathComponent(url.deletingPathExtension().lastPathComponent + ".pdf") + guard process.terminationStatus == 0, FileManager.default.fileExists(atPath: pdf.path) else { + throw ExtractError.unreadable("LibreOffice couldn't convert it") + } + return pdf + } + + /// Slide count of a .pptx without converting it: a .pptx is a zip, and each + /// slide is `ppt/slides/slideN.xml` in its central directory. Nil for anything + /// else (legacy .ppt, broken files). + static func slideCount(of url: URL) -> Int? { + guard url.pathExtension.lowercased() == "pptx", + let data = try? Data(contentsOf: url), data.count > 22 else { return nil } + // End of central directory: the last 22 bytes at least (plus an optional comment). + let searchStart = max(0, data.count - 65557) + var eocd: Int? + var i = data.count - 22 + while i >= searchStart { + if data[i] == 0x50, data[i + 1] == 0x4B, data[i + 2] == 0x05, data[i + 3] == 0x06 { + eocd = i + break + } + i -= 1 + } + guard let eocd, + data.count >= eocd + 22, + data[eocd + 8] < 0xFF || data[eocd + 9] < 0xFF else { return nil } + let entries = Int(data[eocd + 8]) | (Int(data[eocd + 9]) << 8) + let dirOffset = Int(data[eocd + 16]) | (Int(data[eocd + 17]) << 8) + | (Int(data[eocd + 18]) << 16) | (Int(data[eocd + 19]) << 24) + guard entries > 0, dirOffset >= 0, dirOffset < data.count else { return nil } + var slides = 0 + var offset = dirOffset + for _ in 0.. 0 ? slides : nil + } +} enum DocExtractor { /// Plain text from txt/md directly, HTML by stripping tags, and rtf/docx/doc/odt through the Cocoa text system. /// (Cocoa's HTML import runs WebKit and must be on the main thread; the app indexes on a background queue.) + /// Linux reads txt/md/html only; rich formats need a backend (pandoc/libreoffice) — not ported yet. static func text(of url: URL) throws -> String { let ext = url.pathExtension.lowercased() + #if os(macOS) let type: NSAttributedString.DocumentType switch ext { case "docx": type = .officeOpenXML @@ -173,6 +259,14 @@ enum DocExtractor { default: return try readText(url) } return try NSAttributedString(url: url, options: [.documentType: type], documentAttributes: nil).string + #else + switch ext { + case "html", "htm": return plainText(fromHTML: try readText(url)) + case "docx", "doc", "odt", "rtf": + throw ExtractError.unreadable("\(ext) needs a Linux document backend — not ported yet") + default: return try readText(url) + } + #endif } private static func readText(_ url: URL) throws -> String { @@ -220,8 +314,9 @@ enum DocExtractor { /// Splits text into ~`size`-character pieces (~450 tokens) that overlap a little, breaking at whitespace. static func chunks(_ text: String, size: Int = 1800, overlap: Int = 200, limit: Int) -> [String] { + // NBSP written braceless: ICU has no braced escape form, and Linux answers a bad pattern with "". let clean = text - .replacingOccurrences(of: #"[ \t\u{00A0}]+"#, with: " ", options: .regularExpression) + .replacingOccurrences(of: #"[ \t\u00A0]+"#, with: " ", options: .regularExpression) .replacingOccurrences(of: #"\n\s*\n\s*\n+"#, with: "\n\n", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) var chunks: [String] = [] diff --git a/Sources/DigUpKit/Extractors_Linux.swift b/Sources/DigUpKit/Extractors_Linux.swift new file mode 100644 index 00000000..74c791bf --- /dev/null +++ b/Sources/DigUpKit/Extractors_Linux.swift @@ -0,0 +1,148 @@ +#if os(Linux) +import Foundation + +// Linux stubs with the same API so the indexer compiles and runs. +// Text and code index normally; images, scanned PDFs and OCR'd pages report +// why they were left out until the Linux backends land (stb/VIPS, poppler, +// tesseract). See Extractors.swift (macOS implementations). + +enum ImageExtractor { + struct Prepared: Sendable { + let path: String + let width: Int + let height: Int + let taken: String? + } + + static func prepare(_ url: URL, maxPixel: Int, minSide: Int, to folder: URL, png: Bool) throws -> Prepared { + throw ExtractError.unreadable("images need a Linux image backend (stb/VIPS) — not ported yet") + } +} + +/// PDFs on Linux via poppler (`pdfinfo` for the count/title, `pdftotext -layout` +/// for text pages, `pdftoppm` for pages rendered as images), mirroring what the +/// macOS reader returns: text pages as text, the rest as pictures (with garbled +/// layers kept for meaning, never shown). No OCR yet: pages whose text layer is +/// lost keep the layer as-is instead of being re-read. +enum PDFExtractor { + struct Page: Sendable { + let number: Int + let text: String + let imagePath: String? + var ocr = false + var garbled: String? + } + + static func pages(of url: URL, range: (Int) -> Range, minText: Int, renderPixels: Int, to folder: URL) + throws -> (pages: [Page], total: Int, title: String?) { + let info = try pdfinfo(of: url) + let total = info.pages + var pages: [Page] = [] + for index in range(total).clamped(to: 0..= minText { + pages.append(Page(number: number, text: text, imagePath: nil)) + continue + } + case .glyphOrder: + (text, garbled) = ("", text) + case nil: + if text.count >= minText { + pages.append(Page(number: number, text: text, imagePath: nil)) + continue + } + } + guard let image = render(url, page: number, pixels: renderPixels, to: folder) else { continue } + pages.append(Page(number: number, text: text, imagePath: image, garbled: garbled)) + } + return (pages, total, info.title) + } + + // MARK: poppler + + private struct Info { + var pages = 0 + var title: String? + var encrypted = false + } + + private static func pdfinfo(of url: URL) throws -> Info { + let output: String + do { + output = try run("pdfinfo", [url.path]) + } catch { + throw ExtractError.unreadable("can't open PDF (\(error))") + } + var info = Info() + for line in output.split(separator: "\n") { + if line.hasPrefix("Pages:") { + info.pages = Int(line.dropFirst("Pages:".count).trimmingCharacters(in: .whitespaces)) ?? 0 + } else if line.hasPrefix("Title:") { + let title = line.dropFirst("Title:".count).trimmingCharacters(in: .whitespaces) + if !title.isEmpty { info.title = title } + } else if line.hasPrefix("Encrypted:") { + info.encrypted = line.dropFirst("Encrypted:".count).trimmingCharacters(in: .whitespaces) + .hasPrefix("yes") + } + } + guard info.pages > 0 else { + if info.encrypted { throw ExtractError.locked } + throw ExtractError.unreadable("no pages") + } + return info + } + + private static func pageText(of url: URL, page: Int) throws -> String { + do { + return try run("pdftotext", ["-layout", "-f", "\(page)", "-l", "\(page)", url.path, "-"]) + } catch { + // A locked file fails every page the same way. + if (try? pdfinfo(of: url))?.encrypted == true { throw ExtractError.locked } + throw error + } + } + + /// The page as a PNG fitted roughly into a square of `pixels` (like the mac render). + private static func render(_ url: URL, page: Int, pixels: Int, to folder: URL) -> String? { + let dpi = max(100, min(300, pixels / 8)) // 1400 px -> 175 dpi (~1500x1900 for Letter) + let prefix = folder.appendingPathComponent(UUID().uuidString).path + guard (try? run("pdftoppm", ["-png", "-r", "\(dpi)", "-f", "\(page)", "-l", "\(page)", + url.path, prefix])) != nil else { return nil } + let out = "\(prefix)-\(page).png" + return FileManager.default.fileExists(atPath: out) ? out : nil + } + + /// poppler's stdout, or an error. Looks in /usr/bin, then on PATH. + private static func run(_ tool: String, _ arguments: [String]) throws -> String { + let process = Process() + if FileManager.default.isExecutableFile(atPath: "/usr/bin/\(tool)") { + process.executableURL = URL(fileURLWithPath: "/usr/bin/\(tool)") + process.arguments = arguments + } else { + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = [tool] + arguments + } + let output = Pipe() + process.standardOutput = output + process.standardError = FileHandle.nullDevice + process.standardInput = FileHandle.nullDevice + do { + try process.run() + } catch { + throw ExtractError.unreadable("can't run \(tool) (poppler-utils installed?)") + } + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + guard process.terminationStatus == 0, let text = String(data: data, encoding: .utf8) else { + throw ExtractError.unreadable("\(tool) failed") + } + return text + } +} +#endif diff --git a/Sources/DigUpKit/FolderSelection.swift b/Sources/DigUpKit/FolderSelection.swift index f2e8241d..2ba1ec83 100644 --- a/Sources/DigUpKit/FolderSelection.swift +++ b/Sources/DigUpKit/FolderSelection.swift @@ -1,4 +1,7 @@ import Foundation +#if os(Linux) +import Glibc +#endif /// Which folders DigUp searches: the folders you chose, minus the folders you skipped inside them. A skip applies /// inside the chosen folders that contain it, and a chosen folder is searched whole even inside a skipped one (skip diff --git a/Sources/DigUpKit/Indexer.swift b/Sources/DigUpKit/Indexer.swift index a82477d8..8f34fb28 100644 --- a/Sources/DigUpKit/Indexer.swift +++ b/Sources/DigUpKit/Indexer.swift @@ -355,6 +355,18 @@ public final class Indexer { return pending } + /// A PDF's pages for a job (all of a short one, a step of a long one). + /// Long PDFs (books) get their first pages now and the rest after everything else. + static func preparePDF(of url: URL, job: Job, options: IndexOptions, into folder: URL) throws -> Payload { + var wanted = 0..<0 + let read = try PDFExtractor.pages(of: url, range: { total in + wanted = (job.step ?? (total > options.bigPDFPages ? 0.. Prepared { let batch = job.files var prepared = Prepared(batch: batch, step: job.step, folder: folder) @@ -371,7 +383,11 @@ public final class Indexer { let image = try ImageExtractor.prepare(url, maxPixel: screenshot ? 1536 : 1024, minSide: options.minImageSide, to: folder, png: screenshot) + #if os(macOS) let ocr = screenshot ? ImageExtractor.fullImage(url).map(OCR.text(in:)) ?? "" : "" + #else + let ocr = "" + #endif ready.append(ReadyImage(slot: slot, image: image, ocr: ocr)) } catch let error as ExtractError { if case .tooSmall = error { @@ -383,14 +399,11 @@ public final class Indexer { } prepared.payload = .images(screenshot: screenshot, ready: ready) case .pdf: - // Long PDFs (books) get their first pages now and the rest after everything else. - var wanted = 0..<0 - let read = try PDFExtractor.pages(of: url, range: { total in - wanted = (job.step ?? (total > options.bigPDFPages ? 0..(invoking body: () throws -> T) rethrows -> T { + try body() +} +#endif diff --git a/Sources/DigUpKit/MediaExtractors.swift b/Sources/DigUpKit/MediaExtractors.swift index f2275bbf..cfc6a8c7 100644 --- a/Sources/DigUpKit/MediaExtractors.swift +++ b/Sources/DigUpKit/MediaExtractors.swift @@ -1,30 +1,8 @@ +#if canImport(AVFoundation) @preconcurrency import AVFoundation import CoreGraphics import Foundation -private final class ResultBox: @unchecked Sendable { - var result: Result? -} - -/// Runs async AVFoundation work from the synchronous indexer (one thread on purpose: simple and easy to reason about). -func runBlocking(_ operation: @escaping @Sendable () async throws -> T) throws -> T { - let box = ResultBox() - let done = DispatchSemaphore(value: 0) - Task.detached { - do { box.result = .success(try await operation()) } catch { box.result = .failure(error) } - done.signal() - } - done.wait() - return try box.result!.get() -} - -/// A stretch of audio written to a temporary WAV for the model, or a video frame written to a JPEG. -struct MediaPiece: Sendable { - let path: String - let start: Double - var end: Double -} - enum AudioExtractor { static let sampleRate = 16_000.0 @@ -163,3 +141,4 @@ enum MediaInfo { return seconds.flatMap { $0.isFinite && $0 > 0 ? $0 : nil } } } +#endif diff --git a/Sources/DigUpKit/MediaExtractors_Linux.swift b/Sources/DigUpKit/MediaExtractors_Linux.swift new file mode 100644 index 00000000..810a89b3 --- /dev/null +++ b/Sources/DigUpKit/MediaExtractors_Linux.swift @@ -0,0 +1,27 @@ +#if os(Linux) +import Foundation + +// Linux stubs with the same API so the indexer compiles and runs. +// Audio and video report why they were left out until the ffmpeg backend +// lands. See MediaExtractors.swift (macOS implementations). + +enum AudioExtractor { + static let sampleRate = 16_000.0 + + static func windows(of url: URL, window: Double, hop: Double, minimum: Double = 2, to folder: URL) async throws + -> (pieces: [MediaPiece], duration: Double) { + throw ExtractError.unreadable("audio needs a Linux audio backend (ffmpeg) — not ported yet") + } +} + +enum VideoExtractor { + static func keyframes(of url: URL, every interval: Double, maxPixel: Int, maxFrames: Int = 1200, to folder: URL) + async throws -> (frames: [MediaPiece], duration: Double, hasAudio: Bool) { + throw ExtractError.unreadable("video needs a Linux video backend (ffmpeg) — not ported yet") + } +} + +enum MediaInfo { + static func duration(of url: URL) -> Double? { nil } +} +#endif diff --git a/Sources/DigUpKit/MediaTypes.swift b/Sources/DigUpKit/MediaTypes.swift new file mode 100644 index 00000000..03a86748 --- /dev/null +++ b/Sources/DigUpKit/MediaTypes.swift @@ -0,0 +1,28 @@ +import Foundation + +// Portable bits of the media layer (used on every OS). The AVFoundation +// decoders live in MediaExtractors.swift (macOS); the Linux stubs that throw +// until the ffmpeg backend lands live in MediaExtractors_Linux.swift. + +private final class ResultBox: @unchecked Sendable { + var result: Result? +} + +/// Runs async extractor work from the synchronous indexer (one thread on purpose: simple and easy to reason about). +func runBlocking(_ operation: @escaping @Sendable () async throws -> T) throws -> T { + let box = ResultBox() + let done = DispatchSemaphore(value: 0) + Task.detached { + do { box.result = .success(try await operation()) } catch { box.result = .failure(error) } + done.signal() + } + done.wait() + return try box.result!.get() +} + +/// A stretch of audio written to a temporary WAV for the model, or a video frame written to a JPEG. +struct MediaPiece: Sendable { + let path: String + let start: Double + var end: Double +} diff --git a/Sources/DigUpKit/Model.swift b/Sources/DigUpKit/Model.swift index 72cb6726..b11e3ba7 100644 --- a/Sources/DigUpKit/Model.swift +++ b/Sources/DigUpKit/Model.swift @@ -3,13 +3,16 @@ import Foundation /// What a file is, for indexing and for grouping results. public enum FileKind: String, CaseIterable, Sendable, Codable { case screenshot, image, pdf, doc, audio, video + /// Presentation decks (Linux: ppt/pptx via LibreOffice; macOS leaves them out). + case slides /// Source code, a repo's docs and notebooks: only in the code index (`IndexOptions.code`). case code /// The order of the first indexing pass: what people look for most comes first. + /// Slides go last: each needs a LibreOffice conversion first. static let indexingOrderSQL = """ CASE kind WHEN 'screenshot' THEN 0 WHEN 'image' THEN 1 WHEN 'pdf' THEN 2 WHEN 'doc' THEN 3 - WHEN 'audio' THEN 4 WHEN 'video' THEN 5 ELSE 6 END + WHEN 'audio' THEN 4 WHEN 'video' THEN 5 WHEN 'slides' THEN 6 ELSE 7 END """ } diff --git a/Sources/DigUpKit/ModelFiles.swift b/Sources/DigUpKit/ModelFiles.swift index d864a62a..f62ffb75 100644 --- a/Sources/DigUpKit/ModelFiles.swift +++ b/Sources/DigUpKit/ModelFiles.swift @@ -1,4 +1,8 @@ +#if canImport(CryptoKit) import CryptoKit +#elseif canImport(Crypto) +import Crypto +#endif import Foundation /// The EmbeddingGemma 2 files the app runs: ggml-org's Q8_0 GGUF conversion at one pinned revision of the repo (made @@ -33,8 +37,17 @@ public enum ModelFiles { public static let totalBytes = all.reduce(0) { $0 + $1.bytes } /// Where the app keeps them. + #if os(macOS) public static let installed = FileManager.default.homeDirectoryForCurrentUser .appendingPathComponent("Library/Application Support/DigUp/Models") + #else + /// Where the CLI keeps them on Linux (`$XDG_DATA_HOME`, else `~/.local/share`). + public static let installed: URL = { + let base = ProcessInfo.processInfo.environment["XDG_DATA_HOME"].map { URL(fileURLWithPath: $0) } + ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".local/share") + return base.appendingPathComponent("DigUp/Models") + }() + #endif /// The pinned revision's folder in the Hugging Face cache (`hf download` puts the files there): `$HF_HUB_CACHE`, /// `$HF_HOME/hub`, or `~/.cache/huggingface/hub`. diff --git a/Sources/DigUpKit/Rules.swift b/Sources/DigUpKit/Rules.swift index 9320aa1c..81f4ff24 100644 --- a/Sources/DigUpKit/Rules.swift +++ b/Sources/DigUpKit/Rules.swift @@ -1,5 +1,7 @@ import Foundation +#if canImport(UniformTypeIdentifiers) import UniformTypeIdentifiers +#endif public enum Classification: Equatable, Sendable { case index(FileKind) @@ -105,6 +107,41 @@ public enum Rules { return nil } + /// File extensions mapped without Apple's type system (Linux: no UniformTypeIdentifiers). + /// Kept in sync with the UTType branches of `classify(_:type:options:)` above. + static let imageExtensions: Set = [ + "heic", "heif", "jpg", "jpeg", "png", "gif", "webp", "tiff", "tif", "bmp", "ico", + "raw", "cr2", "cr3", "nef", "arw", "dng", "raf", "rw2", "orf", "srw", "pef", "psd", + ] + static let audioExtensions: Set = [ + "mp3", "wav", "m4a", "aac", "flac", "ogg", "oga", "opus", "wma", "aiff", "aif", "caf", + ] + static let videoExtensions: Set = [ + "mp4", "mov", "m4v", "mkv", "avi", "webm", "mpg", "mpeg", "3gp", "flv", "wmv", "ogv", + ] + + /// Presentations read through LibreOffice (Linux only; macOS leaves them out). + static let slidesExtensions: Set = ["ppt", "pptx", "odp"] + + /// Extension-only classification for platforms without UniformTypeIdentifiers (Linux). + public static func classify(_ url: URL, options: IndexOptions) -> Classification { + let ext = url.pathExtension.lowercased() + if options.excludedExtensions.contains(ext) { return .skip("excluded type") } + if ext == "ts" || ext == "mts" || ext == "m2ts" { + if isMPEGTransportStream(url) { return .index(.video) } + return options.includeCode ? .index(.doc) : .skip("code or data file") + } + if ext == "pdf" { return .index(.pdf) } + if slidesExtensions.contains(ext) { return .index(.slides) } + if docExtensions.contains(ext) { return .index(.doc) } + if codeExtensions.contains(ext) { return options.includeCode ? .index(.doc) : .skip("code or data file") } + if imageExtensions.contains(ext) { return .index(isScreenshot(url) ? .screenshot : .image) } + if videoExtensions.contains(ext) { return .index(.video) } + if audioExtensions.contains(ext) { return .index(.audio) } + return .ignore + } + + #if canImport(UniformTypeIdentifiers) public static func classify(_ url: URL, type: UTType?, options: IndexOptions) -> Classification { let ext = url.pathExtension.lowercased() if options.excludedExtensions.contains(ext) { return .skip("excluded type") } @@ -125,6 +162,7 @@ public enum Rules { if type.conforms(to: .audio) { return .index(.audio) } return .ignore } + #endif /// MPEG-TS packets start with 0x47 every 188 bytes (or every 192 with a 4-byte timecode, as in .m2ts / AVCHD .mts). static func isMPEGTransportStream(_ url: URL) -> Bool { @@ -137,8 +175,11 @@ public enum Rules { } /// macOS marks screenshots with an extended attribute that survives moves and copies; names are the fallback. + /// (Linux has no such attribute: names only.) public static func isScreenshot(_ url: URL) -> Bool { + #if os(macOS) if getxattr(url.path, "com.apple.metadata:kMDItemIsScreenCapture", nil, 0, 0, 0) > 0 { return true } + #endif let name = url.lastPathComponent return screenshotPrefixes.contains { name.hasPrefix($0) } } diff --git a/Sources/DigUpKit/Searcher.swift b/Sources/DigUpKit/Searcher.swift index ca51eb1c..25ab2473 100644 --- a/Sources/DigUpKit/Searcher.swift +++ b/Sources/DigUpKit/Searcher.swift @@ -1,5 +1,7 @@ -import Accelerate import Foundation +#if canImport(Accelerate) +import Accelerate +#endif public struct SearchHit: Sendable { public let path: String @@ -293,9 +295,21 @@ public final class Searcher { let count = min(VectorStorage.blockRows, rows - row) Vectors.floats(fromHalf: matrix.halves + row * Vectors.dimension, count: count * Vectors.dimension, to: matrix.block) + #if canImport(Accelerate) // (count × 768) · (768 × 1) vDSP_mmul(matrix.block, 1, query.baseAddress!, 1, scores.baseAddress! + row, 1, vDSP_Length(count), 1, vDSP_Length(Vectors.dimension)) + #else + // Portable fallback (Linux): plain dot products, same result. + let q = query.baseAddress! + let s = scores.baseAddress! + row + for r in 0.., count: Int, to destination: UnsafeMutablePointer) { + #if canImport(Accelerate) var from = vImage_Buffer(data: UnsafeMutableRawPointer(mutating: source), height: 1, width: vImagePixelCount(count), rowBytes: count * 2) var to = vImage_Buffer(data: destination, height: 1, width: vImagePixelCount(count), rowBytes: count * 4) vImageConvert_Planar16FtoPlanarF(&from, &to, vImage_Flags(kvImageNoFlags)) + #else + // Portable fallback (Linux): same result, no Accelerate. + for i in 0.. 0 else { continue } switch kind { case .pdf: kinds.append("\(number(count)) PDFs (\(number(estimate.pdfPagesTotal)) pages; \(number(estimate.pdfPages)) in the first pass)") + case .slides: kinds.append("\(number(count)) slide decks (\(number(estimate.slidePagesTotal)) slides; \(number(estimate.slidePages)) in the first pass)") case .audio: kinds.append("\(number(count)) audio (\(duration(estimate.audioSeconds)))") case .video: kinds.append("\(number(count)) video (\(duration(estimate.videoSeconds)))") case .code: kinds.append("\(number(count)) code \(count == 1 ? "file" : "files") " diff --git a/Tests/DigUpKitTests/FolderSelectionTests.swift b/Tests/DigUpKitTests/FolderSelectionTests.swift index 8e331762..cb4e0d13 100644 --- a/Tests/DigUpKitTests/FolderSelectionTests.swift +++ b/Tests/DigUpKitTests/FolderSelectionTests.swift @@ -1,4 +1,7 @@ import Foundation +#if os(Linux) +import Glibc +#endif import Testing @testable import DigUpKit diff --git a/Tests/DigUpKitTests/IndexStoreTests.swift b/Tests/DigUpKitTests/IndexStoreTests.swift index ec1568e0..032238b4 100644 --- a/Tests/DigUpKitTests/IndexStoreTests.swift +++ b/Tests/DigUpKitTests/IndexStoreTests.swift @@ -1,5 +1,12 @@ +#if canImport(CryptoKit) import CryptoKit +#elseif canImport(Crypto) +import Crypto +#endif import Foundation +#if os(Linux) +import Glibc +#endif import Testing @testable import DigUpKit diff --git a/Tests/DigUpKitTests/LongFilesTests.swift b/Tests/DigUpKitTests/LongFilesTests.swift index 3a39767c..fd36640e 100644 --- a/Tests/DigUpKitTests/LongFilesTests.swift +++ b/Tests/DigUpKitTests/LongFilesTests.swift @@ -1,5 +1,10 @@ +#if os(macOS) import AppKit +#endif import Foundation +#if os(Linux) +import Glibc +#endif import Testing @testable import DigUpKit @@ -52,6 +57,7 @@ import Testing return out } + #if os(macOS) // needs CoreGraphics PDF output + PDFKit reading (macOS-only for now) @Test func longPDFIsReadInSteps() throws { let pdf = try makePDF("Field guide.pdf", pages: 120, special: [100: "quokka"]) let store = try store() @@ -80,7 +86,9 @@ import Testing #expect(try indexer.finishLongFiles().isEmpty) // nothing left #expect(embedder.texts == 120) // every page embedded once } + #endif + #if os(macOS) @Test func stoppingBetweenStepsLosesNothing() throws { let pdf = try makePDF("Book.pdf", pages: 100) let store = try store() @@ -95,7 +103,9 @@ import Testing #expect(try places(store, pdf.path) == Array(1...100)) #expect(try state(store, pdf.path).state == "done") } + #endif + #if os(macOS) @Test func aChangedFileStartsOver() throws { let pdf = try makePDF("Book.pdf", pages: 80) let store = try store() @@ -110,6 +120,7 @@ import Testing try indexer.finishLongFiles() #expect(try places(store, pdf.path) == Array(1...90)) } + #endif @Test func longDocumentGoesOnWhereItStopped() throws { let text = prose(characters: 150_000, special: (at: 120_000, word: "axolotl")) @@ -162,6 +173,7 @@ import Testing #expect(try found(store, "4815162342").count == 1) } + #if os(macOS) @Test func machineMadeGiantsStopAtTheCap() throws { let pdf = try makePDF("Export.pdf", pages: 120) var options = IndexOptions() @@ -178,6 +190,7 @@ import Testing try store.db.query("SELECT detail FROM files") { detail = $0.text(0) } #expect(detail == "the first 45 of 120 pages") } + #endif @Test func oldIndexesFinishWhatTheyCut() throws { // As schema 4 left them: a long PDF read to page 30, a document cut at its 40th chunk, and a chunk of digits @@ -218,6 +231,7 @@ import Testing #expect(try found(migrated, "9153") == [1]) // still found by keyword search } + #if os(macOS) @Test func estimatesCountTheRest() throws { let book = try makePDF("Book.pdf", pages: 120) let short = try makePDF("Leaflet.pdf", pages: 3) @@ -232,6 +246,7 @@ import Testing #expect(abs(estimate.seconds - 33 * costs.pdfPage) < 1e-9) #expect(abs(estimate.laterSeconds - 90 * costs.pdfPage) < 1e-9) } + #endif @Test func float16VectorsScoreAcrossBlocks() throws { // More vectors than one block (`VectorStorage.blockRows`): every one must be scored, the last ones too. @@ -256,6 +271,7 @@ import Testing // MARK: Files + #if os(macOS) /// A text PDF with `pages` pages of a few sentences each; `special` puts a word on a page (from 1). @discardableResult func makePDF(_ name: String, pages: Int, special: [Int: String] = [:]) throws -> URL { @@ -277,6 +293,7 @@ import Testing context.closePDF() return url } + #endif /// Plain English-looking sentences, the same for the same seed; `special`'s word goes in once, past `at`. func prose(characters: Int, seed: Int = 1, special: (at: Int, word: String)? = nil) -> String { diff --git a/Tests/DigUpKitTests/RulesTests.swift b/Tests/DigUpKitTests/RulesTests.swift index 2e54b944..0c0e4664 100644 --- a/Tests/DigUpKitTests/RulesTests.swift +++ b/Tests/DigUpKitTests/RulesTests.swift @@ -12,7 +12,11 @@ import Testing @Test func typeScriptIsNotVideo() throws { let file = folder.appendingPathComponent("app.ts") try "export const x: number = 1;\n".write(to: file, atomically: true, encoding: .utf8) + #if canImport(UniformTypeIdentifiers) #expect(Rules.classify(file, type: .init(filenameExtension: "ts"), options: IndexOptions()) == .skip("code or data file")) + #else + #expect(Rules.classify(file, options: IndexOptions()) == .skip("code or data file")) + #endif } @Test func realTransportStreamIsVideo() throws { @@ -20,14 +24,22 @@ import Testing for packet in 0..<4 { bytes[packet * 188] = 0x47 } let file = folder.appendingPathComponent("clip.ts") try Data(bytes).write(to: file) + #if canImport(UniformTypeIdentifiers) #expect(Rules.classify(file, type: .init(filenameExtension: "ts"), options: IndexOptions()) == .index(.video)) + #else + #expect(Rules.classify(file, options: IndexOptions()) == .index(.video)) + #endif } @Test func codeCanBeIncluded() { var options = IndexOptions() options.includeCode = true let file = folder.appendingPathComponent("main.py") + #if canImport(UniformTypeIdentifiers) #expect(Rules.classify(file, type: .init(filenameExtension: "py"), options: options) == .index(.doc)) + #else + #expect(Rules.classify(file, options: options) == .index(.doc)) + #endif } @Test func machineNames() { diff --git a/docs/linux.md b/docs/linux.md new file mode 100644 index 00000000..f35a2a7f --- /dev/null +++ b/docs/linux.md @@ -0,0 +1,57 @@ +# Linux port (branch `linux-port-x86_64`, x86_64 first) + +Scope: the `digup` CLI (estimate / index / search / status / eval / +index-service / worker) and its engine. The menubar app (`DigUpApp`, +AppKit/SwiftUI/Sparkle) stays macOS-only and is excluded from the Linux +package graph (`Package.swift` appends it only on macOS). + +## What works on Linux today + +- text, markdown, HTML and code indexing + search (pure `Foundation` paths) +- PDFs via poppler (`pdftotext`/`pdftoppm`/`pdfinfo`), same page model as macOS +- slide decks via LibreOffice (`soffice` converts ppt/pptx/odp to PDF, then the PDF path) +- SQLite index + FTS5 keyword search + float16 meaning search (portable + `Float` fallback where Accelerate is missing) +- extension-based file typing (no `UniformTypeIdentifiers`) +- `pdfinfo` page counts and `ffprobe` durations for estimates +- llama.cpp CPU build (`scripts/build-llama.sh` detects Linux, uses `ar`, + keeps `GGML_NATIVE=OFF` so `-bin` packages run on any x86_64) + +## Honest stubs (report why, never crash) + +- images/screenshots: `Extractors_Linux.swift` (needs stb/VIPS + resize) +- OCR: names only for now (needs tesseract) +- audio/video: `MediaExtractors_Linux.swift` (needs ffmpeg decode) +- rich documents (docx/doc/odt/rtf): needs pandoc/libreoffice + +The indexer records these as skipped/failed per file, so mixed folders +index everything else. + +## Build + +```sh +sudo apt install cmake ninja-build libsqlite3-dev ffmpeg poppler-utils # Debian/Ubuntu +# Arch: yay -S swift-bin cmake ninja sqlite ffmpeg poppler +scripts/build-llama.sh +swift build -c release --product digup +swift test --filter DigUpKitTests # PDF/output tests run on macOS only +``` + +Index and model locations follow `$XDG_DATA_HOME` (`~/.local/share` fallback) +instead of `~/Library/Application Support`. + +## Binary installs (no compile) + +```sh +scripts/package-linux.sh # → dist/*.tar.gz (+ .deb with dpkg-deb) +``` + +- generic tarball → manual install, and the base for `digup-bin` on AUR + (`packaging/PKGBUILD-bin`: bump `pkgver`, `updpkgsums`) +- `.deb` → Debian/Ubuntu (`Depends: libsqlite3-0, ffmpeg, poppler-utils`) +- `.rpm` → built in CI from `packaging/digup.spec` +- releases: `.github/workflows/linux-release.yml` uploads all three plus + `SHA256SUMS` on every published tag. aarch64 is a second job, not started. + +The 865 MB model is never bundled: it downloads on first use +(`ModelFiles`, `LlamaModels`), same as macOS. diff --git a/packaging/PKGBUILD-bin b/packaging/PKGBUILD-bin new file mode 100644 index 00000000..9751ad06 --- /dev/null +++ b/packaging/PKGBUILD-bin @@ -0,0 +1,22 @@ +# Maintainer template for the AUR -bin package (installs the release tarball, +# no compilation). Copy to an AUR repo as PKGBUILD, set pkgver to the release, +# and refresh sha256sums with: updpkgsums +# (https://wiki.archlinux.org/title/Arch_package_guidelines) +pkgname=digup-bin +pkgver=0.6.0 +pkgrel=1 +pkgdesc="Search your files by meaning (EmbeddingGemma 2, local-first)" +arch=('x86_64') +url="https://github.com/ARahim3/DigUp" +license=('MIT') +depends=('sqlite' 'ffmpeg' 'poppler') +provides=('digup') +conflicts=('digup') +source=("digup-$pkgver-x86_64.tar.gz::https://github.com/ARahim3/DigUp/releases/download/v$pkgver/digup-$pkgver-x86_64.tar.gz") +sha256sums=('REPLACE_WITH_SHA256_FROM_SHA256SUMS') + +package() { + install -Dm755 usr/bin/digup "$pkgdir/usr/bin/digup" + install -Dm644 usr/share/doc/digup/README.md "$pkgdir/usr/share/doc/digup/README.md" + install -Dm644 usr/share/doc/digup/LICENSE "$pkgdir/usr/share/licenses/digup/LICENSE" +} diff --git a/packaging/digup.spec b/packaging/digup.spec new file mode 100644 index 00000000..fe1360fd --- /dev/null +++ b/packaging/digup.spec @@ -0,0 +1,30 @@ +Name: digup +Version: %{version} +Release: 1%{?dist} +Summary: Search your files by meaning +License: MIT +URL: https://github.com/ARahim3/DigUp +Source0: digup-%{version}-x86_64.tar.gz +BuildArch: x86_64 +Requires: sqlite-libs, ffmpeg, poppler-utils + +%description +DigUp indexes pictures, PDFs, documents, audio, video and code with +Google DeepMind's EmbeddingGemma 2 and finds them from a description +in your own words. Everything runs on your machine. + +%prep + +%build + +%install +install -Dm755 usr/bin/digup %{buildroot}%{_bindir}/digup +install -Dm644 usr/share/doc/digup/README.md %{buildroot}%{_docdir}/digup/README.md +install -Dm644 usr/share/doc/digup/LICENSE %{buildroot}%{_licensedir}/digup/LICENSE + +%files +%{_bindir}/digup +%{_docdir}/digup/README.md +%{_licensedir}/digup/LICENSE + +%changelog diff --git a/scripts/build-llama.sh b/scripts/build-llama.sh index 20c5f0f3..553c746b 100755 --- a/scripts/build-llama.sh +++ b/scripts/build-llama.sh @@ -1,8 +1,10 @@ #!/bin/bash -# Builds llama.cpp (MIT), the embedding runtime: the pinned build tag plus scripts/llama.cpp-no-logits.patch, as static -# libraries combined into Vendor/llama/lib/libllamacpp.a, with the headers in Vendor/llama/include (all out of git). -# Metal shaders are embedded and compiled on first use, so no Metal toolchain is needed. CMake comes from PATH, or from -# a venv in Vendor/ (needs uv). Does nothing when that build is already there; build.sh runs it every time. +# Builds llama.cpp (MIT), the embedding runtime: the pinned build tag plus scripts/llama.cpp-no-logits.patch. +# macOS combines the static libraries into Vendor/llama/lib/libllamacpp.a; Linux links the original archives +# directly (see Package.swift). Headers go in Vendor/llama/include (all out of git). +# macOS: Metal shaders are embedded and compiled on first use, so no Metal toolchain is needed. +# Linux (x86_64 CPU first): plain CPU build with GGML_NATIVE=OFF so -bin packages run on any x86_64. +# CMake comes from PATH, or from a venv in Vendor/ (needs uv). Does nothing when that build is already there. set -euo pipefail cd "$(dirname "$0")/.." TAG=b11461 # the first tags with EmbeddingGemma 2 (text + vision + audio) are b11452+ @@ -12,7 +14,13 @@ TAG=b11461 # the first tags with EmbeddingGemma 2 (text + vision + audio) are PATCH=scripts/llama.cpp-no-logits.patch SOURCE=Vendor/llama.cpp OUT=Vendor/llama -STAMP="$TAG $(shasum -a 256 "$PATCH" | cut -c1-12)" +UNAME="$(uname -s)" +if [[ "$UNAME" == "Darwin" ]]; then + STAMP="$TAG $(shasum -a 256 "$PATCH" | cut -c1-12)" +else + # r2: -fPIC static libs (SwiftPM links test bundles as .so). + STAMP="$TAG-Linux-r2 $(sha256sum "$PATCH" | cut -c1-12)" +fi if [[ -f "$OUT/lib/libllamacpp.a" && "$(cat "$OUT/BUILD" 2>/dev/null)" == "$STAMP" ]]; then exit 0 fi @@ -39,19 +47,39 @@ if ! command -v cmake > /dev/null; then export PATH="$PWD/Vendor/cmake-venv/bin:$PATH" fi GENERATOR=$(command -v ninja > /dev/null && echo Ninja || echo "Unix Makefiles") +CPUS=$(command -v nproc > /dev/null && nproc || sysctl -n hw.ncpu) # Only the libraries: no tools, no server, nothing downloaded at build time. -cmake -S "$SOURCE" -B Vendor/llama-build -G "$GENERATOR" -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \ - -DBUILD_SHARED_LIBS=OFF -DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON -DGGML_NATIVE=OFF -DLLAMA_BUILD_MTMD=ON \ - -DLLAMA_BUILD_COMMON=OFF -DLLAMA_BUILD_TOOLS=OFF -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_EXAMPLES=OFF \ - -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_APP=OFF -DLLAMA_BUILD_UI=OFF -DLLAMA_OPENSSL=OFF > /dev/null -cmake --build Vendor/llama-build --config Release -j "$(sysctl -n hw.ncpu)" --target llama mtmd > /dev/null +if [[ "$UNAME" == "Darwin" ]]; then + cmake -S "$SOURCE" -B Vendor/llama-build -G "$GENERATOR" -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \ + -DBUILD_SHARED_LIBS=OFF -DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON -DGGML_NATIVE=OFF -DLLAMA_BUILD_MTMD=ON \ + -DLLAMA_BUILD_COMMON=OFF -DLLAMA_BUILD_TOOLS=OFF -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_APP=OFF -DLLAMA_BUILD_UI=OFF -DLLAMA_OPENSSL=OFF > /dev/null +else + # Linux x86_64: CPU build (CUDA/Vulkan later). GGML_NATIVE=OFF keeps -bin packages portable + # (generic x86_64 baseline; no GGML_CPU_ALL_VARIANTS: it needs GGML_BACKEND_DL dynamic backends, + # which don't fit the static link model). PIC: SwiftPM links test bundles as shared objects. + cmake -S "$SOURCE" -B Vendor/llama-build -G "$GENERATOR" -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=OFF -DGGML_NATIVE=OFF -DGGML_CPU_ALL_VARIANTS=OFF -DLLAMA_BUILD_MTMD=ON \ + -DLLAMA_BUILD_COMMON=OFF -DLLAMA_BUILD_TOOLS=OFF -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_APP=OFF -DLLAMA_BUILD_UI=OFF -DLLAMA_OPENSSL=OFF > /dev/null +fi +cmake --build Vendor/llama-build --config Release -j "$CPUS" --target llama mtmd > /dev/null B=Vendor/llama-build mkdir -p "$OUT/lib" "$OUT/include" -xcrun libtool -static -no_warning_for_no_symbols -o "$OUT/lib/libllamacpp.a" \ - $B/src/libllama.a $B/tools/mtmd/libmtmd.a $B/ggml/src/libggml.a $B/ggml/src/libggml-base.a \ - $B/ggml/src/libggml-cpu.a $B/ggml/src/ggml-metal/libggml-metal.a $B/ggml/src/ggml-blas/libggml-blas.a \ - $B/vendor/hash/libvendor-hash.a +if [[ "$UNAME" == "Darwin" ]]; then + xcrun libtool -static -no_warning_for_no_symbols -o "$OUT/lib/libllamacpp.a" \ + $B/src/libllama.a $B/tools/mtmd/libmtmd.a $B/ggml/src/libggml.a $B/ggml/src/libggml-base.a \ + $B/ggml/src/libggml-cpu.a $B/ggml/src/ggml-metal/libggml-metal.a $B/ggml/src/ggml-blas/libggml-blas.a \ + $B/vendor/hash/libvendor-hash.a +else + # Linux links the original archives directly (--start-group in Package.swift): + # members share basenames across archives (quants.c.o, repack.cpp.o), so no + # merge can hold them all. Drop a stale merged library, so it shadows nothing. + rm -f "$OUT/lib/libllamacpp.a" + rm -rf "$OUT/combine-tmp" +fi cp "$SOURCE/include/llama.h" "$SOURCE"/ggml/include/*.h "$SOURCE/tools/mtmd/mtmd.h" "$SOURCE/tools/mtmd/mtmd-helper.h" \ "$OUT/include/" # The build, for logs and `digup status` (indexes record LlamaEmbedder.vectorVersion, not this). SwiftPM doesn't see a diff --git a/scripts/package-linux.sh b/scripts/package-linux.sh new file mode 100755 index 00000000..739d357e --- /dev/null +++ b/scripts/package-linux.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Linux binary packaging (x86_64 first): a portable tarball plus a .deb. +# .rpm and the AUR -bin package are built from the same tarball in CI +# (.github/workflows/linux-release.yml) and packaging/, so maintainers of +# other distros only need the tarball. +# +# scripts/package-linux.sh [--version X.Y.Z] [--no-build] +# +# Output in dist/: digup--x86_64.tar.gz, digup__amd64.deb, SHA256SUMS. +# Needs: a release Swift build already done (or it runs build-llama.sh + swift +# build itself unless --no-build). llama.cpp is built with GGML_NATIVE=OFF so +# the binary runs on any x86_64. +set -euo pipefail +cd "$(dirname "$0")/.." + +VERSION="" +NO_BUILD=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --no-build) NO_BUILD=1; shift ;; + *) echo "usage: $0 [--version X.Y.Z] [--no-build]"; exit 2 ;; + esac +done + +if [[ -z "$VERSION" ]]; then + if TAG="$(git describe --tags --exact-match 2>/dev/null)"; then + VERSION="${TAG#v}" + else + VERSION="$(grep -m1 CFBundleShortVersionString -A1 Resources/Info.plist | tail -1 | sed 's/.*\(.*\)<\/string>.*/\1/')" + fi +fi +[[ "$VERSION" =~ ^[0-9]+(\.[0-9]+)*$ ]] || { echo "bad version: $VERSION"; exit 1; } +ARCH="$(uname -m)" +[[ "$ARCH" == "x86_64" ]] || { echo "x86_64 first: refusing on $ARCH (add aarch64 as a second job)"; exit 1; } + +if [[ "$NO_BUILD" == 0 ]]; then + scripts/build-llama.sh + swift build -c release --product digup +fi +BIN="$(swift build -c release --show-bin-path)/digup" +[[ -x "$BIN" ]] || { echo "no binary at $BIN"; exit 1; } + +rm -rf dist +mkdir -p dist/stage/usr/bin dist/stage/usr/share/doc/digup +cp "$BIN" dist/stage/usr/bin/digup +cp README.md LICENSE dist/stage/usr/share/doc/digup/ +chmod 0755 dist/stage/usr/bin/digup + +# The -bin tarball (Arch digup-bin, manual installs): binary + docs at the root. +TARBALL="dist/digup-$VERSION-x86_64.tar.gz" +tar -czf "$TARBALL" -C dist/stage usr/bin/digup usr/share/doc/digup/README.md usr/share/doc/digup/LICENSE + +# The .deb (Debian/Ubuntu): needs dpkg-deb (CI has it; on Arch: yay -S dpkg). +if command -v dpkg-deb > /dev/null; then + mkdir -p dist/stage/DEBIAN + cat > dist/stage/DEBIAN/control < +Description: Search your files by meaning + DigUp indexes pictures, PDFs, documents, audio, video and code with + Google DeepMind's EmbeddingGemma 2 and finds them from a description + in your own words. Everything runs on your machine. +Homepage: https://github.com/ARahim3/DigUp +EOF + dpkg-deb --build dist/stage "dist/digup_${VERSION}_amd64.deb" + rm -rf dist/stage/DEBIAN +else + echo "dpkg-deb missing: skipping .deb (the tarball is done)" +fi + +(cd dist && sha256sum digup-*.tar.gz digup_*.deb 2>/dev/null > SHA256SUMS || true) +echo "Wrote:" +ls -la dist/ | tail -n +2