Linux CLI archive and legacy Word support on current main

.github/workflows/linux-cli.yml

@@ -0,0 +1,55 @@
+name: Linux CLI (x86_64)
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/linux-cli.yml'
+      - 'Package.swift'
+      - 'Sources/**'
+      - 'Tests/**'
+      - 'scripts/build-llama.sh'
+      - 'scripts/fetch-vendor.sh'
+      - 'scripts/package-linux.sh'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build-test-package:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: System tools
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq cmake ninja-build curl libcurl4-openssl-dev python3 ffmpeg libreoffice-writer
+      - name: Swift 6.2
+        uses: swift-actions/setup-swift@v2
+        with:
+          swift-version: '6.2'
+      - name: Pinned native libraries
+        run: |
+          scripts/fetch-vendor.sh
+          DIGUP_CPU_ONLY=1 scripts/build-llama.sh
+      - name: Build and test
+        run: |
+          swift build -c release --product digup --static-swift-stdlib
+          # Ubuntu 24.04's FFmpeg 6.1 cannot satisfy the existing HEIC and
+          # orientation-sensitive TIFF/WebP ImageTests (they require FFmpeg 8.1).
+          swift test --skip ImageTests
+      - name: Package CLI
+        run: |
+          version=$(python3 -c 'import plistlib; print(plistlib.load(open("Resources/Info.plist", "rb"))["CFBundleShortVersionString"])')
+          scripts/package-linux.sh --version "$version" --no-build
+          (cd dist && sha256sum -c SHA256SUMS)
+          folder=$(mktemp -d)
+          tar -xzf dist/digup-"$version"-linux-x86_64.tar.gz -C "$folder"
+          "$folder/digup-$version-linux-x86_64/digup" --help
+      - name: CLI archive
+        uses: actions/upload-artifact@v4
+        with:
+          name: digup-linux-cli-x86_64
+          path: |
+            dist/digup-*-linux-x86_64.tar.gz
+            dist/SHA256SUMS

.gitignore

@@ -2,6 +2,7 @@
 .build/
 .swiftpm/
 build.noindex/
+/dist/
 /Vendor/
 
 # Dev runs (scripts/dev-run.sh): their indexes, models and settings

README.md

@@ -145,7 +145,8 @@ holds the queries ranking changes are measured with.
 **How is this different from Spotlight?** Spotlight finds files by their names, text and metadata. DigUp also finds
 them by what they show or say, and takes you to the page or moment. The two work fine side by side.
 
-**Windows or Linux?** Both are on the way.
+**Windows or Linux?** Both are on the way. An experimental Linux CLI source build is described in
+[docs/linux.md](docs/linux.md); there is no downloadable Linux app yet.
 
 **Why is the download 865 MB?** That's the model: the text part (310 MB) and the image and audio encoders (555 MB).
 The app itself is about 20 MB.

Sources/DigUpKit/Platform/Portable/Docs+Portable.swift

@@ -1,5 +1,5 @@
 // Word-processor files without the Cocoa text system (Linux, Windows): docx and odt from the XML inside their zip
-// (miniz, Vendor/miniz), rtf stripped of its control words. Legacy .doc isn't read here.
+// (miniz, Vendor/miniz), rtf stripped of its control words. Linux can read legacy .doc through LibreOffice Writer.
 #if !canImport(AppKit)
 import CMiniz
 import Foundation
@@ -15,11 +15,49 @@ extension DocExtractor {
                                               breaks: ["text:line-break"], spaces: ["text:tab", "text:s"]))
         case "rtf":
             return RTF.text(try readText(url))
+        #if os(Linux)
+        case "doc":
+            return try legacyWordText(of: url)
+        #endif
         default:
             throw ExtractError.unreadable("legacy Word (.doc) files aren't read on this system")
         }
     }
 
+    #if os(Linux)
+    /// LibreOffice is optional. Keep each conversion in its own directory and profile: indexing can run in parallel,
+    /// and a user's open Writer must not own the headless process or receive the document.
+    private static func legacyWordText(of url: URL) throws -> String {
+        let work = FileManager.default.temporaryDirectory.appendingPathComponent("DigUp-doc-\(UUID().uuidString)", isDirectory: true)
+        try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true)
+        defer { try? FileManager.default.removeItem(at: work) }
+        let profile = work.appendingPathComponent("profile", isDirectory: true)
+        let output = work.appendingPathComponent(url.deletingPathExtension().lastPathComponent + ".txt")
+        let paths = (ProcessInfo.processInfo.environment["PATH"] ?? "").split(separator: ":")
+            .map { String($0) + "/soffice" }
+        guard let soffice = (["/usr/bin/soffice"] + paths).first(where: FileManager.default.isExecutableFile) else {
+            throw ExtractError.unreadable("legacy Word (.doc) files need LibreOffice Writer")
+        }
+        let process = Process()
+        process.executableURL = URL(fileURLWithPath: soffice)
+        process.arguments = ["-env:UserInstallation=\(profile.absoluteString)", "--headless",
+                             "--convert-to", "txt:Text (encoded):UTF8", "--outdir", work.path, url.path]
+        process.standardInput = FileHandle.nullDevice
+        process.standardOutput = FileHandle.nullDevice
+        process.standardError = FileHandle.nullDevice
+        do { try process.run() } catch {
+            throw ExtractError.unreadable("legacy Word (.doc) files need LibreOffice Writer")
+        }
+        process.waitUntilExit()
+        guard process.terminationStatus == 0, let data = try? Data(contentsOf: output),
+              let text = String(data: data, encoding: .utf8) else {
+            throw ExtractError.unreadable("LibreOffice Writer couldn't convert this .doc file")
+        }
+        // The UTF-8 text filter emits a BOM; it is not part of the document.
+        return text.hasPrefix("\u{FEFF}") ? String(text.dropFirst()) : text
+    }
+    #endif
+
     /// Word-processing XML as the little HTML `plainText` reads: paragraph ends become `</p>`, breaks `<br>`, tabs and
     /// spaces a space; every other tag is stripped there.
     static func asHTML(_ xml: String, paragraphs: [String], breaks: [String], spaces: [String]) -> String {

Tests/DigUpKitTests/LegacyWordTests.swift

@@ -0,0 +1,36 @@
+#if os(Linux)
+import Foundation
+import Testing
+@testable import DigUpKit
+
+@Suite struct LegacyWordTests {
+    @Test func libreOfficeReadsLegacyWordFile() throws {
+        // LibreOffice is an optional runtime dependency; Linux CI installs it.
+        guard FileManager.default.isExecutableFile(atPath: "/usr/bin/soffice") else { return }
+        let folder = FileManager.default.temporaryDirectory.appendingPathComponent("DigUp-doc-test-\(UUID().uuidString)")
+        try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
+        defer { try? FileManager.default.removeItem(at: folder) }
+
+        let rtf = folder.appendingPathComponent("sample.rtf")
+        try #"{\rtf1\ansi\uc1\pard Atlas nebula orchid. Caf\u233? and \u26085?\u26412?.\par}"#
+            .write(to: rtf, atomically: true, encoding: .utf8)
+        let converter = Process()
+        converter.executableURL = URL(fileURLWithPath: "/usr/bin/soffice")
+        let profile = folder.appendingPathComponent("profile", isDirectory: true)
+        converter.arguments = ["-env:UserInstallation=\(profile.absoluteString)", "--headless",
+                               "--convert-to", "doc", "--outdir", folder.path, rtf.path]
+        converter.standardInput = FileHandle.nullDevice
+        converter.standardOutput = FileHandle.nullDevice
+        converter.standardError = FileHandle.nullDevice
+        try converter.run()
+        converter.waitUntilExit()
+        #expect(converter.terminationStatus == 0)
+
+        let document = folder.appendingPathComponent("sample.doc")
+        let text = try DocExtractor.text(of: document)
+        #expect(text.contains("Atlas nebula orchid. Café and 日本."))
+        #expect(!text.hasPrefix("\u{FEFF}"))
+        #expect(DocExtractor.chunks(text, limit: 10).contains { $0.contains("nebula orchid") })
+    }
+}
+#endif

docs/linux.md

@@ -0,0 +1,54 @@
+# Linux CLI status
+
+DigUp's portable engine and `digup` CLI build on Linux. This is an experimental
+source build and CLI archive, not the released macOS menu bar app. The Linux
+desktop shell is still in development.
+
+The CLI keeps the same local EmbeddingGemma 2 model, SQLite index, and meaning
+plus keyword search. It reads text, code, PDF pages, common images, audio and
+video. PDFium renders scanned pages for image search, but Linux has no screenshot
+or scanned-page OCR for exact words yet. RAW images and presentation decks are
+not supported. HEIC and some orientation-sensitive TIFF/WebP images need FFmpeg
+8.1; Ubuntu 24.04's FFmpeg 6.1 fails the existing image fixture tests. Legacy
+`.doc` text needs LibreOffice Writer; `.docx`, `.odt`, and `.rtf` use the
+built-in readers. File-specific failures are reported while other files
+continue indexing.
+
+## Build on Ubuntu 24.04 x86_64
+
+Install Swift 6.2 or later and the build tools (`git`, `curl`, `python3`, `cmake`,
+`ninja-build`, and a C++ compiler). Install `ffmpeg` for media. LibreOffice
+Writer is optional for legacy `.doc` files.
+
+```sh
+scripts/fetch-vendor.sh
+DIGUP_CPU_ONLY=1 scripts/build-llama.sh
+swift build -c release --product digup
+.build/release/digup --help
+.build/release/digup download-model
+```
+
+`download-model` downloads and verifies the pinned model files (about 865 MB).
+The CLI keeps its model and index under `$XDG_DATA_HOME/digup` or
+`~/.local/share/digup`. Use `digup --help` for indexing and search commands.
+
+## CLI archive
+
+On Linux x86_64, make a CPU archive with:
+
+```sh
+scripts/package-linux.sh --version 0.6.1
+```
+
+The script builds the CLI with the Swift runtime linked statically, bundles
+the pinned PDFium and llama.cpp shared libraries and CPU backends beside the
+binary, and writes a tarball and `SHA256SUMS` under `dist/`. Extract it and run
+`./digup` inside its directory, or keep the libraries beside the binary when
+moving it. The model files are downloaded separately.
+
+The archive targets the Linux ABI of the build runner (Ubuntu 24.04 in CI). It
+still uses system glibc, C++/OpenMP libraries, and `ffmpeg` for media. It is
+not a `.deb` or `.rpm`, and no Linux desktop app is included. CI builds and
+tests the CLI and stores the tarball as an artifact. CI excludes the image
+fixture suite until its FFmpeg requirement is met; the other tests run.
+Publishing the archive as a release requires a separate decision.

scripts/build-llama.sh

@@ -7,7 +7,7 @@
 # time (GGML_BACKEND_DL): the CPU variants always, CUDA when nvcc is found (CUDACXX, or on PATH; CUDA_ARCHS picks the GPU
 # architectures, default native), else Vulkan when its SDK is. LLAMA_SOURCE / LLAMA_BUILD point at an existing clone and
 # build folder. On Windows it runs in Git for Windows' bash inside the MSVC environment (vcvars64) and
-# leaves DLLs with their import libraries.
+# leaves DLLs with their import libraries. DIGUP_CPU_ONLY=1 forces a portable CPU-only build for Linux archives.
 set -euo pipefail
 cd "$(dirname "$0")/.."
 TAG=b11461   # the first tags with EmbeddingGemma 2 (text + vision + audio) are b11452+
@@ -22,6 +22,8 @@ OUT=Vendor/llama
 OS=$(uname -s)
 if [[ $OS == Darwin ]]; then
   FLAVOR=metal
+elif [[ "${DIGUP_CPU_ONLY:-0}" == 1 ]]; then
+  FLAVOR=cpu
 elif command -v "${CUDACXX:-nvcc}" > /dev/null; then
   FLAVOR=cuda
 elif command -v glslc > /dev/null; then
@@ -83,9 +85,9 @@ else
     export CC=cl CXX=cl   # MSVC, not whatever else bash finds first
   fi
   case $FLAVOR in
-    cuda) GPU=(-DGGML_CUDA=ON "-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHS:-native}") ;;
-    vulkan) GPU=(-DGGML_VULKAN=ON) ;;
-    *) GPU=() ;;
+    cuda) GPU=(-DGGML_CUDA=ON -DGGML_VULKAN=OFF "-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHS:-native}") ;;
+    vulkan) GPU=(-DGGML_CUDA=OFF -DGGML_VULKAN=ON) ;;
+    *) GPU=(-DGGML_CUDA=OFF -DGGML_VULKAN=OFF) ;;
   esac
   # Shared libraries, the backends as modules (every CPU variant, so one build fits any x86-64 or arm64 machine).
   cmake -S "$SOURCE" -B "$BUILD" -G "$GENERATOR" -DCMAKE_BUILD_TYPE=Release \

scripts/package-linux.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Make a Linux x86_64 CLI archive with its pinned native libraries.
+# With --no-build, run the vendor and llama.cpp build scripts first.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+VERSION=""
+BUILD=1
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --version) [[ $# -ge 2 ]] || { echo "--version needs a value" >&2; exit 2; }; VERSION="$2"; shift 2 ;;
+    --no-build) BUILD=0; shift ;;
+    *) echo "usage: $0 --version X.Y.Z [--no-build]" >&2; exit 2 ;;
+  esac
+done
+[[ "$VERSION" =~ ^[0-9]+(\.[0-9]+)+$ ]] || { echo "use --version X.Y.Z" >&2; exit 2; }
+[[ "$(uname -s)-$(uname -m)" == "Linux-x86_64" ]] || { echo "Linux x86_64 only" >&2; exit 1; }
+
+if [[ "$BUILD" == 1 ]]; then
+  scripts/fetch-vendor.sh
+  DIGUP_CPU_ONLY=1 scripts/build-llama.sh
+  swift build -c release --product digup --static-swift-stdlib
+fi
+BIN="$(swift build -c release --show-bin-path)/digup"
+[[ -x "$BIN" ]] || { echo "build the release digup binary first" >&2; exit 1; }
+[[ "$(cat Vendor/llama/BUILD 2>/dev/null)" == *" cpu" ]] || {
+  echo "build the CPU runtime with DIGUP_CPU_ONLY=1 scripts/build-llama.sh" >&2
+  exit 1
+}
+if ldd "$BIN" | grep -Eq 'libswift|libFoundation|libdispatch|libBlocksRuntime'; then
+  echo "the CLI still needs shared Swift libraries; rebuild with --static-swift-stdlib" >&2
+  exit 1
+fi
+
+NAME="digup-$VERSION-linux-x86_64"
+STAGE="dist/$NAME"
+rm -rf dist
+mkdir -p "$STAGE/licenses/pdfium"
+cp "$BIN" "$STAGE/digup"
+cp -a Vendor/llama/lib/*.so* "$STAGE/"
+cp -a Vendor/pdfium/lib/libpdfium.so "$STAGE/"
+cp README.md LICENSE Resources/Acknowledgements.txt "$STAGE/"
+cp Vendor/pdfium/LICENSE "$STAGE/licenses/pdfium/LICENSE"
+cp -a Vendor/pdfium/licenses "$STAGE/licenses/pdfium/third-party"
+
+# The runtime and CPU backend modules live beside digup. Verify the staged
+# executable's native dependencies resolve before creating a distributable archive.
+for file in "$STAGE/digup" "$STAGE"/*.so*; do
+  [[ -L "$file" ]] && continue
+  if LD_LIBRARY_PATH="$PWD/$STAGE" ldd "$file" | grep -q 'not found'; then
+    echo "a native dependency is missing from the archive: $file" >&2
+    exit 1
+  fi
+done
+tar -czf "dist/$NAME.tar.gz" -C dist "$NAME"
+(cd dist && sha256sum "$NAME.tar.gz" > SHA256SUMS)
+echo "Wrote dist/$NAME.tar.gz"