From c8c75161444fd0a08f9f691f4ca38b79f930734d Mon Sep 17 00:00:00 2001 From: dowjames <371653+dowjames@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:05:39 -0400 Subject: [PATCH 1/7] Add CI: build + engine tests on macOS, shellcheck + ruff on Ubuntu Nothing ran swift build/swift test on PRs, so regressions merged unverified (CONTRIBUTING.md asks for passing tests). Vendor/llama is cached by the same stamp build-llama.sh records, so PR jobs reuse it instead of rebuilding llama.cpp; ReferenceVectorTests (needs the model) is skipped on PRs. --- .github/workflows/lint.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..1f8c3cdd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,36 @@ +# Lint the shell scripts and the Python worker on Ubuntu: shellcheck and ruff. The build/test CI (ci.yml) does not +# lint, so this runs alongside it on every pull request and push to main. +name: Lint + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + name: shellcheck + ruff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install shellcheck and ruff + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y -qq shellcheck + python3 -m pip install --quiet --user ruff + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: shellcheck + run: | + set -euo pipefail + shopt -s globstar nullglob + shellcheck -S warning build.sh scripts/**/*.sh + + - name: ruff + run: ruff check worker/ scripts/appcast.py From 580db80e536b61555375277bcdc8ad48dd5facea Mon Sep 17 00:00:00 2001 From: dowjames <371653+dowjames@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:05:39 -0400 Subject: [PATCH 2/7] Fail fast and loudly in the build and release scripts - build-llama.sh: a no-logits patch that does not apply (after a TAG bump) was silently skipped while BUILD still stamped it as patched, caching an unpatched build forever; now it is an error (already-applied is tolerated). - build.sh: locate Sparkle.framework by pattern with a clear error instead of one hardcoded slice path; treat a blank .sign-id as absent; build both products in one swift build invocation. - release.sh: check the Sparkle key before the multi-minute build instead of after notarization; drop deprecated --deep and verify nested code explicitly; keep notarytool logs as release evidence; require APPCAST or SKIP_APPCAST so a DMG is never silently left off the update feed. --- build.sh | 10 ++++++---- scripts/build-llama.sh | 2 ++ scripts/release.sh | 19 +++++++++++++++---- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index eeeef7e5..2bfe97d8 100755 --- a/build.sh +++ b/build.sh @@ -25,8 +25,7 @@ fi scripts/build-llama.sh # Both executables: the app, and the `digup` CLI, which the app runs as its indexing helper and query encoder. -swift build -c release --product DigUpApp -swift build -c release --product digup +swift build -c release --product DigUpApp --product digup BIN_DIR="$(swift build -c release --show-bin-path)" # The app icon ("One space", moss; Resources/AppIcon/README.md): macOS 26 draws the layered AppIcon.icon (compiled into @@ -51,7 +50,10 @@ cp Resources/AppIcon/tray.png Resources/AppIcon/tray-dim.png Resources/AppIcon/t # sandboxed apps and its headers for building against it: left out. FW="$APP/Contents/Frameworks/Sparkle.framework" mkdir -p "$APP/Contents/Frameworks" -ditto .build/artifacts/sparkle/Sparkle/Sparkle.xcframework/macos-arm64_x86_64/Sparkle.framework "$FW" +# Located by pattern, not one hardcoded slice path: Sparkle packaging changes the folder name between releases. +SPARKLE=$(find .build/artifacts/sparkle -type d -path '*/Sparkle.xcframework/*/Sparkle.framework' -not -path '*/Versions/*' | head -1) +[[ -n "$SPARKLE" ]] || { echo "Sparkle.framework not found under .build/artifacts/sparkle (does the Swift package resolve it?)" >&2; exit 1; } +ditto "$SPARKLE" "$FW" for part in XPCServices Headers PrivateHeaders Modules; do rm -rf "${FW:?}/$part" "${FW:?}/Versions/B/$part"; done PLIST="$APP/Contents/Info.plist" @@ -65,7 +67,7 @@ PLIST="$APP/Contents/Info.plist" # Nested code first, then the bundle. The helper needs no entitlements: llama.cpp compiles its Metal shaders at run # time through Metal's own compiler service, which the hardened runtime allows (checked 2026-10-08). The app's one # entitlement lets it ask QuickTime Player to open a recording at a moment. -IDENTITY="${SIGN_ID:-$( [[ -f .sign-id ]] && head -1 .sign-id || echo - )}" +IDENTITY="${SIGN_ID:-$( [[ -s .sign-id ]] && head -1 .sign-id || echo - )}" TIMESTAMP=$([[ "${RELEASE:-}" == 1 ]] && echo --timestamp || echo --timestamp=none) codesign --force --sign "$IDENTITY" --options runtime $TIMESTAMP "$APP/Contents/Helpers/digup" # Sparkle comes ad-hoc signed: its helpers, then the framework, with this identity (the app may only load code signed diff --git a/scripts/build-llama.sh b/scripts/build-llama.sh index a904cdbd..c69e1e89 100755 --- a/scripts/build-llama.sh +++ b/scripts/build-llama.sh @@ -52,6 +52,8 @@ if [[ "$(git -C "$SOURCE" rev-parse HEAD)" != "$COMMIT" ]]; then fi # Without the patch, every input token costs 1 MiB of logits that an embedding model never produces (262k vocabulary). # Its path is absolute: git -C reads a relative one from the source folder, which LLAMA_SOURCE can put anywhere. +# A build that silently skipped the patch would still be stamped as patched and cached forever, so a patch that +# does not apply (after a TAG bump) is an error, not a warning. if git -C "$SOURCE" apply --check "$PWD/$PATCH" 2>/dev/null; then git -C "$SOURCE" apply "$PWD/$PATCH" elif ! git -C "$SOURCE" apply --reverse --check "$PWD/$PATCH" 2>/dev/null; then diff --git a/scripts/release.sh b/scripts/release.sh index 2b94897c..f4795d7a 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -10,6 +10,11 @@ set -euo pipefail cd "$(dirname "$0")/.." : "${SIGN_ID:?set SIGN_ID to a Developer ID Application identity (security find-identity -v -p codesigning)}" : "${NOTARY_PROFILE:?set NOTARY_PROFILE to a notarytool keychain profile (see the top of this script)}" +# Checked now, not after a multi-minute build and notarization: Sparkle's key is needed to sign the DMG. +if [[ -n "${APPCAST:-}" && -z "${SPARKLE_KEY_FILE:-}" ]]; then + security find-generic-password -s "Sparkle EdDSA key" > /dev/null 2>&1 \ + || { echo "Sparkle EdDSA key not in the login keychain (run Sparkle's generate_keys once, or set SPARKLE_KEY_FILE=)." >&2; exit 1; } +fi VERSION=$(/usr/libexec/PlistBuddy -c "Print CFBundleShortVersionString" Resources/Info.plist) APP=build.noindex/DigUp.app OUT=build.noindex/release @@ -17,11 +22,12 @@ rm -rf "$OUT" mkdir -p "$OUT" RELEASE=1 ./build.sh -codesign --verify --strict --deep --verbose=2 "$APP" +# --deep is deprecated; build.sh signs each nested item explicitly, so verify them explicitly too. +codesign --verify --strict --verbose=2 "$APP/Contents/Helpers/digup" "$APP/Contents/Frameworks/Sparkle.framework" "$APP" # The app itself is notarized and stapled first, so a copy dragged out of the DMG passes Gatekeeper offline too. ditto -c -k --keepParent "$APP" "$OUT/DigUp.zip" -xcrun notarytool submit "$OUT/DigUp.zip" --keychain-profile "$NOTARY_PROFILE" --wait +xcrun notarytool submit "$OUT/DigUp.zip" --keychain-profile "$NOTARY_PROFILE" --wait --output "$OUT/notary-app.json" xcrun stapler staple "$APP" rm "$OUT/DigUp.zip" @@ -34,12 +40,17 @@ DMG="$OUT/DigUp-$VERSION.dmg" hdiutil create -quiet -volname "DigUp" -srcfolder "$STAGE" -fs HFS+ -format ULFO -ov "$DMG" rm -rf "$STAGE" codesign --force --sign "$SIGN_ID" --timestamp "$DMG" -xcrun notarytool submit "$DMG" --keychain-profile "$NOTARY_PROFILE" --wait +xcrun notarytool submit "$DMG" --keychain-profile "$NOTARY_PROFILE" --wait --output "$OUT/notary-dmg.json" xcrun stapler staple "$DMG" spctl --assess --type execute --verbose=2 "$APP" spctl --assess --type open --context context:primary-signature --verbose=2 "$DMG" if [[ -n "${APPCAST:-}" ]]; then - scripts/appcast.py "$DMG" "$APPCAST" ${NOTES:+--notes "$NOTES"} + scripts/appcast.py "$DMG" "$APPCAST" ${NOTES:+--notes "$NOTES"} ${SPARKLE_KEY_FILE:+--key-file "$SPARKLE_KEY_FILE"} +elif [[ "${SKIP_APPCAST:-}" == 1 ]]; then + echo "APPCAST skipped (SKIP_APPCAST=1): this DMG is not Sparkle-signed; in-app updates will not see it." +else + echo "Set APPCAST= to Sparkle-sign the DMG, or SKIP_APPCAST=1 to build without a feed." >&2 + exit 1 fi echo "Released $DMG ($(du -h "$DMG" | cut -f1))" From ece3386a7704ac8fe73363ca73098df0f1b07d64 Mon Sep 17 00:00:00 2001 From: dowjames <371653+dowjames@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:05:39 -0400 Subject: [PATCH 3/7] appcast.py: validate the feed, derive the download base from origin The f-string feed could contain unparseable XML (]]> in notes, raw ); it is now parsed before writing, ]]> is escaped, and DOWNLOAD_BASE defaults to the origin remote so forks get their own URLs. Read the feed with a context manager. --- scripts/appcast.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/appcast.py b/scripts/appcast.py index f83bec17..c334a7d6 100755 --- a/scripts/appcast.py +++ b/scripts/appcast.py @@ -17,6 +17,7 @@ import re import subprocess import sys +import xml.etree.ElementTree as ET ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SIGN_UPDATE = os.path.join(ROOT, ".build/artifacts/sparkle/Sparkle/bin/sign_update") @@ -29,6 +30,20 @@ """ +def download_base(root, version): + """The download URL for this version, from the origin remote so forks get their own URLs.""" + try: + remote = subprocess.run(["git", "-C", root, "remote", "get-url", "origin"], + check=True, capture_output=True, text=True).stdout.strip() + repo = re.sub(r"^(?:https?://|git@)", "", remote) + repo = re.sub(r"\.git$", "", repo).replace(":", "/", 1) + if repo.count("/") == 2 and repo.startswith("github.com"): + return f"https://{repo}/releases/download/v{version}/" + except Exception: + pass + return f"https://github.com/ARahim3/DigUp/releases/download/v{version}/" + + def main(): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("dmg") @@ -50,12 +65,13 @@ def main(): if not re.fullmatch(r'sparkle:edSignature="[^"]+" length="\d+"', signature): sys.exit(f"sign_update said: {signature}") - base = os.environ.get("DOWNLOAD_BASE", f"https://github.com/ARahim3/DigUp/releases/download/v{version}/") + base = os.environ.get("DOWNLOAD_BASE") or download_base(ROOT, version) url = base.rstrip("/") + "/" + os.path.basename(args.dmg) notes = "" if args.notes: with open(args.notes) as f: - notes = f"\n <description><![CDATA[{f.read().strip()}]]></description>" + body = f.read().strip().replace("]]>", "]]]]><![CDATA[>") # keep the CDATA section closed + notes = f"\n <description><![CDATA[{body}]]></description>" item = f""" <item> <title>{version} {email.utils.formatdate(usegmt=True)} @@ -66,12 +82,20 @@ def main(): """ - feed = open(args.appcast).read() if os.path.exists(args.appcast) else SKELETON + if os.path.exists(args.appcast): + with open(args.appcast) as f: + feed = f.read() + else: + feed = SKELETON # Running a release again replaces its item. feed = re.sub(r" \n(?:(?!).)*?" + re.escape(build) + r".*?\n", "", feed, flags=re.S) # Newest first: after the channel's title. feed = re.sub(r"(\n\s*[^<]*\n)", lambda m: m.group(1) + item, feed, count=1) + try: + ET.fromstring(feed) + except ET.ParseError as error: + sys.exit(f"the new appcast would not parse ({error}); {args.appcast} left unchanged") with open(args.appcast, "w") as f: f.write(feed) print(f"{args.appcast}: {version} ({build}) → {url}") From be0252aedd4306a4e062811ebbe4f458c844db5e Mon Sep 17 00:00:00 2001 From: dowjames <371653+dowjames@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:05:39 -0400 Subject: [PATCH 4/7] Testbed and dev-run: survive empty globs, missing voices, non-APFS An empty screenshots glob handed xattr a literal pattern and aborted the script (nullglob now); make_media.sh falls back from Samantha to any en_US say voice; cp -c (APFS clones) falls back to plain cp elsewhere. --- scripts/dev-run.sh | 2 +- scripts/make-testbed.sh | 1 + scripts/testbed/make_media.sh | 8 ++++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/dev-run.sh b/scripts/dev-run.sh index 0b82391b..e188f8e5 100755 --- a/scripts/dev-run.sh +++ b/scripts/dev-run.sh @@ -18,7 +18,7 @@ SNAPSHOT="$HOME/.cache/huggingface/hub/models--ggml-org--embeddinggemma-2-GGUF/s if [[ "$MODELS" == "$PWD/.dev/models" ]]; then mkdir -p "$MODELS" for file in embeddinggemma-2-Q8_0.gguf mmproj-embeddinggemma-2-Q8_0.gguf; do - [[ -f "$MODELS/$file" || ! -e "$SNAPSHOT/$file" ]] || cp -cL "$SNAPSHOT/$file" "$MODELS/$file" + [[ -f "$MODELS/$file" || ! -e "$SNAPSHOT/$file" ]] || cp -cL "$SNAPSHOT/$file" "$MODELS/$file" 2>/dev/null || cp -L "$SNAPSHOT/$file" "$MODELS/$file" # -c needs APFS; fall back elsewhere done fi diff --git a/scripts/make-testbed.sh b/scripts/make-testbed.sh index 00231fcf..20111b43 100755 --- a/scripts/make-testbed.sh +++ b/scripts/make-testbed.sh @@ -8,6 +8,7 @@ # scripts/make-testbed.sh [dir] default: ~/DigUpTestbed # Delete it any time: rm -rf ~/DigUpTestbed set -euo pipefail +shopt -s nullglob # the screenshots glob below is empty until make_screenshots.swift has run TB="${1:-$HOME/DigUpTestbed}" REPO="$(cd "$(dirname "$0")/.." && pwd)" export DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" diff --git a/scripts/testbed/make_media.sh b/scripts/testbed/make_media.sh index 7e0585a3..8d6a5b57 100755 --- a/scripts/testbed/make_media.sh +++ b/scripts/testbed/make_media.sh @@ -3,6 +3,7 @@ # TTS speech clips, system sounds, and short videos cut into 6 s "moments". # ./make_media.sh set -euo pipefail +shopt -s nullglob # a missing sample-picture folder must not hand sips a literal glob M="${1:?usage: make_media.sh }"; HERE="$(cd "$(dirname "$0")" && pwd)" mkdir -p "$M/img" "$M/shots" "$M/audio" "$M/audio_trunc" "$M/video" @@ -14,8 +15,11 @@ done DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" swift "$HERE/make_screenshots.swift" "$M/shots" -# The default system voice renders silence from a script; Samantha works. -say_wav() { say -v Samantha -o "$M/audio/tmp.aiff" "$2"; ffmpeg -loglevel error -y -i "$M/audio/tmp.aiff" -ac 1 -ar 16000 "$M/audio/$1.wav"; rm "$M/audio/tmp.aiff"; } +# The default system voice renders silence from a script; prefer Samantha, else any en_US voice. +VOICE=$(say -v '?' | awk '/Samantha/ {print $1; exit}') +VOICE=${VOICE:-$(say -v '?' | awk '/en_US/ {print $1; exit}')} +[[ -n "$VOICE" ]] || { echo "no en_US speech voice available (say -v '?')" >&2; exit 1; } +say_wav() { say -v "$VOICE" -o "$M/audio/tmp.aiff" "$2"; ffmpeg -loglevel error -y -i "$M/audio/tmp.aiff" -ac 1 -ar 16000 "$M/audio/$1.wav"; rm "$M/audio/tmp.aiff"; } say_wav speech_groceries "Don't forget to buy milk, eggs and bread on the way home tomorrow." say_wav speech_revenue "Our quarterly revenue grew by twenty percent, driven by strong sales in Europe." say_wav speech_flight_delay "Attention passengers. The flight to Tokyo has been delayed by two hours because of a storm." From b7142cacdc2b2027f4443b3663a7360104178e09 Mon Sep 17 00:00:00 2001 From: dowjames <371653+dowjames@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:05:39 -0400 Subject: [PATCH 5/7] homebrew.yml: commit only Casks/digup.rb git commit -am would sweep any unrelated dirty file in the tap into the release commit. --- .github/workflows/homebrew.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 34f2b0fe..dd4f3459 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -53,5 +53,6 @@ jobs: fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -am "digup $version" + git add Casks/digup.rb # only the cask: never commit unrelated files sitting in the tap + git commit -m "digup $version" git push From 754d312f6a48a0cd1a6fa0a86faf2785abd10fab Mon Sep 17 00:00:00 2001 From: dowjames <371653+dowjames@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:05:39 -0400 Subject: [PATCH 6/7] worker: per-item text errors, quiet exit on a broken pipe text() failed the whole request on one bad input, contradicting the protocol docstring; it now retries the batch item by item like media(). A dead app now ends the worker quietly instead of with a BrokenPipeError traceback. --- worker/digup_worker.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/worker/digup_worker.py b/worker/digup_worker.py index 30158839..d910a760 100644 --- a/worker/digup_worker.py +++ b/worker/digup_worker.py @@ -106,12 +106,27 @@ def _vectors(self, output): return np.array(output.text_embeds.astype(self.mx.float32)) def text(self, inputs): - rows = [] + rows, errors = [], [] for i in range(0, len(inputs), BATCH["text"]): batch = inputs[i : i + BATCH["text"]] - tokens = self.tokenizer(batch, padding=True, truncation=True, max_length=2048, return_tensors="np") - rows.append(self._vectors(self.model(**{k: self.mx.array(v) for k, v in tokens.items()}))) - return np.concatenate(rows), [None] * len(inputs) + try: + rows.append(self._text_batch(batch)) + errors += [None] * len(batch) + except Exception: + # One bad input must not sink its neighbours (the docstring promises per-item errors): + # retry the batch item by item, like media() does. + for text in batch: + try: + rows.append(self._text_batch([text])) + errors.append(None) + except Exception as item_error: + rows.append(np.zeros((1, DIM), np.float32)) + errors.append(f"{type(item_error).__name__}: {item_error}"[:300]) + return np.concatenate(rows), errors + + def _text_batch(self, batch): + tokens = self.tokenizer(batch, padding=True, truncation=True, max_length=2048, return_tensors="np") + return self._vectors(self.model(**{k: self.mx.array(v) for k, v in tokens.items()})) def media(self, kind, inputs, budget=None): if self.processor is None: @@ -146,8 +161,12 @@ def _media_batch(self, kind, paths): def reply(message): - sys.stdout.write(json.dumps(message, separators=(",", ":")) + "\n") - sys.stdout.flush() + try: + sys.stdout.write(json.dumps(message, separators=(",", ":")) + "\n") + sys.stdout.flush() + except BrokenPipeError: + # The app died mid-session: leave quietly instead of with a traceback. + raise SystemExit(0) def main(): From 730c08faa12bff7833ab53c4f55b52dfdad4afe9 Mon Sep 17 00:00:00 2001 From: dowjames <371653+dowjames@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:53:10 -0400 Subject: [PATCH 7/7] build.sh: keep the two product builds separate (swift build honours only one --product per run; the merged call built digup but not DigUpApp) --- build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 2bfe97d8..0b9d9ce0 100755 --- a/build.sh +++ b/build.sh @@ -25,7 +25,8 @@ fi scripts/build-llama.sh # Both executables: the app, and the `digup` CLI, which the app runs as its indexing helper and query encoder. -swift build -c release --product DigUpApp --product digup +swift build -c release --product DigUpApp +swift build -c release --product digup BIN_DIR="$(swift build -c release --show-bin-path)" # The app icon ("One space", moss; Resources/AppIcon/README.md): macOS 26 draws the layered AppIcon.icon (compiled into