Add Universe Google Drive backup setup

nests/mac/setup.json

@@ -18,7 +18,8 @@
     "rust",
     "claude-code",
     "codex",
-    "machines"
+    "machines",
+    "universe-gdrive-backup"
   ],
   "rc_scripts": [
     "common:fzf.sh",

setups/universe-gdrive-backup/.gitignore

@@ -0,0 +1 @@
+__pycache__/

setups/universe-gdrive-backup/README.md

@@ -0,0 +1,72 @@
+# Universe backup to Google Drive
+
+Mirrors `~/Universe/` (including `notes/`) into `My Drive/Universe Backup/`
+every hour and when the user LaunchAgent loads. Google Drive for desktop then
+uploads that directory. A successful rsync run confirms the local mirror, not
+completion of the cloud upload. **Deletions propagate to the mirror**; retain
+Git history or another versioned backup for recovery.
+
+## Setup
+
+Requires macOS, Python 3 (`/usr/bin/python3`, supplied with Apple developer
+tools), `~/Universe`, and Google Drive for desktop signed in with `My Drive`
+available under `~/Library/CloudStorage/GoogleDrive-*/`. The `mac` nest includes
+this setup. To install it independently:
+
+```sh
+owl setup universe-gdrive-backup all
+```
+
+The installer selects a single mounted Drive account and writes its explicit
+path into the LaunchAgent. With multiple accounts, select one yourself:
+
+```sh
+owl setup universe-gdrive-backup link
+~/owl/setups/universe-gdrive-backup/install.sh --drive-root "$HOME/Library/CloudStorage/GoogleDrive-YOUR_ACCOUNT/My Drive"
+```
+
+Preview the generated plist without installing or starting anything:
+
+```sh
+~/owl/setups/universe-gdrive-backup/install.sh --dry-run
+```
+
+The setup reuses `com.tylertracy.universe-gdrive-backup`, replacing the previous
+job rather than adding a second schedule. Existing plists are retained under
+`~/.cache/universe-backup/launchagent-backups/`. The plist points at this checkout;
+keep it available and rerun installation if you move it. No administrator access,
+Drive credentials, Syncthing configuration, or note-sync configuration is needed.
+
+## Running and troubleshooting
+
+```sh
+universe-gdrive-backup --dry-run  # changes are listed in backup.log
+universe-gdrive-backup           # mirror now
+launchctl list com.tylertracy.universe-gdrive-backup
+tail -n 20 ~/.cache/universe-backup/backup.log
+```
+
+With multiple mounted accounts, also pass `--drive-root` for manual runs.
+The generated schedule retains the account selected during installation.
+Configuration lives in this directory; the generated local plist is
+`~/Library/LaunchAgents/com.tylertracy.universe-gdrive-backup.plist`.
+Launchd output and errors are in `~/.cache/universe-backup/launchd.{out,err}.log`.
+Missing source/mount directories fail without creating a substitute mount.
+Concurrent runs skip while another run holds the backup lock.
+
+The original exclusions are retained: `.DS_Store`, `.stfolder`, `.stversions`,
+and filenames beginning `.sync-conflict-`. Other files, including notes and
+Git history, are copied. No files are read from Drive into Universe.
+
+To disable the job without removing either folder:
+
+```sh
+launchctl disable "gui/$(id -u)/com.tylertracy.universe-gdrive-backup"
+launchctl bootout "gui/$(id -u)/com.tylertracy.universe-gdrive-backup"
+```
+
+Run the installer again to enable it. Checks for this setup:
+
+```sh
+python3 -m unittest discover -s setups/universe-gdrive-backup -p 'test_*.py'
+```

setups/universe-gdrive-backup/backup.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Mirror Universe into Google Drive, or install its per-user launchd job."""
+import argparse
+import datetime
+import fcntl
+import os
+from pathlib import Path
+import plistlib
+import shutil
+import subprocess
+import sys
+
+LABEL = 'com.tylertracy.universe-gdrive-backup'
+EXCLUDES = ['.DS_Store', '.stfolder', '.stversions', '.sync-conflict-*']
+
+
+def drive_root(home, explicit=None):
+    if explicit is not None:
+        root = explicit.expanduser()
+        if not root.is_dir():
+            raise ValueError('Google Drive My Drive directory is not available: ' + str(root))
+        return root.resolve()
+    roots = sorted((home / 'Library/CloudStorage').glob('GoogleDrive-*/My Drive'))
+    roots = [root for root in roots if root.is_dir()]
+    if len(roots) != 1:
+        raise ValueError('Expected one mounted Google Drive account; pass --drive-root "/path/to/My Drive"')
+    return roots[0].resolve()
+
+
+def paths(source, root):
+    if not source.is_dir():
+        raise ValueError('Universe source directory is not available: ' + str(source))
+    source = source.resolve()
+    destination = root / 'Universe Backup'
+    if destination.is_symlink():
+        raise ValueError('Refusing a symlink at the backup destination')
+    destination = destination.resolve()
+    if source == destination or source in destination.parents or destination in source.parents:
+        raise ValueError('Source and backup destination must not overlap')
+    return source, destination
+
+
+def mirror(source, root, state, dry_run=False):
+    source, destination = paths(source, root)
+    state.mkdir(parents=True, exist_ok=True)
+    with (state / 'backup.lock').open('a') as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            print('Universe backup already running; skipped')
+            return 0
+        # Never create the Drive mount itself: an unavailable mount must fail.
+        if not root.is_dir():
+            raise ValueError('Google Drive mount disappeared before backup')
+        if not dry_run:
+            destination.mkdir(exist_ok=True)
+        command = ['/usr/bin/rsync', '-a', '--delete']
+        if dry_run:
+            command += ['--dry-run', '--itemize-changes']
+        for pattern in EXCLUDES:
+            command += ['--exclude', pattern]
+        command += [str(source) + '/', str(destination) + '/']
+        with (state / 'backup.log').open('a') as log:
+            log.write('===== ' + datetime.datetime.now().isoformat() + ' =====\n')
+            log.flush()
+            result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, pass_fds=(lock.fileno(),))
+            log.write('exit={} {}\n'.format(result.returncode, datetime.datetime.now().isoformat()))
+        print('Universe backup {} (details: {})'.format('completed' if result.returncode == 0 else 'failed', state / 'backup.log'))
+        return result.returncode
+
+
+def launch_agent(home, source, root, script, interpreter):
+    state = home / '.cache/universe-backup'
+    return {
+        'Label': LABEL,
+        'ProgramArguments': [str(interpreter), str(script), 'run', '--source', str(source), '--drive-root', str(root)],
+        'StartInterval': 3600,
+        'RunAtLoad': True,
+        'ProcessType': 'Background',
+        'StandardOutPath': str(state / 'launchd.out.log'),
+        'StandardErrorPath': str(state / 'launchd.err.log'),
+    }
+
+
+def install(home, source, root, dry_run=False):
+    source, _ = paths(source, root)
+    data = plistlib.dumps(launch_agent(home, source, root, Path(__file__).resolve(), Path(sys.executable)))
+    if dry_run:
+        sys.stdout.buffer.write(data)
+        return 0
+    agents = home / 'Library/LaunchAgents'
+    state = home / '.cache/universe-backup'
+    agents.mkdir(parents=True, exist_ok=True)
+    state.mkdir(parents=True, exist_ok=True)
+    target = agents / (LABEL + '.plist')
+    if target.exists():
+        backups = state / 'launchagent-backups'
+        backups.mkdir(exist_ok=True)
+        stamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S-%f')
+        shutil.copy2(target, backups / (stamp + '.plist'))
+    domain = 'gui/' + str(os.getuid())
+    service = domain + '/' + LABEL
+    present = subprocess.run(['/bin/launchctl', 'print', service], capture_output=True)
+    if present.returncode == 0:
+        subprocess.run(['/bin/launchctl', 'bootout', service], check=True)
+    temporary = target.with_suffix('.plist.tmp')
+    temporary.write_bytes(data)
+    temporary.replace(target)
+    subprocess.run(['/bin/launchctl', 'enable', service], check=True)
+    subprocess.run(['/bin/launchctl', 'bootstrap', domain, str(target)], check=True)
+    print('Installed hourly Universe backup; launchd has started the first run.')
+    return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('mode', choices=['run', 'install'], nargs='?', default='run')
+    parser.add_argument('--source', type=Path, default=Path.home() / 'Universe')
+    parser.add_argument('--drive-root', type=Path, help='Mounted Google Drive My Drive directory; required if multiple accounts exist')
+    parser.add_argument('--dry-run', action='store_true', help='Preview rsync changes or print the launchd plist without installing it')
+    args = parser.parse_args()
+    if sys.platform != 'darwin':
+        parser.error('This setup requires macOS and Google Drive for desktop')
+    try:
+        root = drive_root(Path.home(), args.drive_root)
+        if args.mode == 'install':
+            return install(Path.home(), args.source.expanduser(), root, args.dry_run)
+        return mirror(args.source.expanduser(), root, Path.home() / '.cache/universe-backup', args.dry_run)
+    except (ValueError, OSError, subprocess.CalledProcessError) as error:
+        print('Universe backup error: ' + str(error), file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

setups/universe-gdrive-backup/install.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -euo pipefail
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+exec /usr/bin/python3 "$script_dir/backup.py" install "$@"

setups/universe-gdrive-backup/setup.json

@@ -0,0 +1,9 @@
+{
+  "install": "local:install.sh",
+  "links": [
+    {
+      "source": "local:backup.py",
+      "target": "~/.local/bin/universe-gdrive-backup"
+    }
+  ]
+}

setups/universe-gdrive-backup/test_backup.py

@@ -0,0 +1,144 @@
+import importlib.util
+import io
+from pathlib import Path
+import plistlib
+import tempfile
+import unittest
+from unittest.mock import patch, Mock
+import fcntl
+import subprocess
+
+spec = importlib.util.spec_from_file_location('universe_backup', Path(__file__).with_name('backup.py'))
+backup = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(backup)
+
+
+class BackupTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory(prefix='owl-backup-test-')
+        self.addCleanup(self.temporary.cleanup)
+        self.root = Path(self.temporary.name).resolve()
+        self.source = self.root / 'Universe'
+        self.source.mkdir()
+        self.drive = self.root / 'Drive account & spaces' / 'My Drive'
+        self.drive.mkdir(parents=True)
+        self.state = self.root / 'cache'
+
+    def test_real_rsync_copies_notes_and_propagates_only_destination_deletions(self):
+        notes = self.source / 'notes'
+        notes.mkdir()
+        (notes / 'daily note.md').write_text('precious writing')
+        (self.source / '.DS_Store').write_text('ignored')
+        (self.source / '.stversions').mkdir()
+        (self.source / '.stversions/old.md').write_text('ignored')
+        (notes / '.git').mkdir()
+        (notes / '.git/HEAD').write_text('ref: refs/heads/main')
+        destination = self.drive / 'Universe Backup'
+        destination.mkdir()
+        (destination / 'obsolete.md').write_text('old mirror content')
+        self.assertEqual(backup.mirror(self.source, self.drive, self.state), 0)
+        self.assertEqual((destination / 'notes/daily note.md').read_text(), 'precious writing')
+        self.assertTrue((destination / 'notes/.git/HEAD').is_file())
+        self.assertFalse((destination / 'obsolete.md').exists())
+        self.assertFalse((destination / '.DS_Store').exists())
+        self.assertFalse((destination / '.stversions').exists())
+        self.assertEqual((notes / 'daily note.md').read_text(), 'precious writing')
+        (notes / 'daily note.md').unlink()
+        self.assertEqual(backup.mirror(self.source, self.drive, self.state), 0)
+        self.assertFalse((destination / 'notes/daily note.md').exists())
+
+    def test_dry_run_does_not_create_or_change_destination(self):
+        (self.source / 'note.md').write_text('new')
+        self.assertEqual(backup.mirror(self.source, self.drive, self.state, True), 0)
+        destination = self.drive / 'Universe Backup'
+        self.assertFalse(destination.exists())
+        destination.mkdir()
+        (destination / 'keep.md').write_text('keep')
+        self.assertEqual(backup.mirror(self.source, self.drive, self.state, True), 0)
+        self.assertEqual((destination / 'keep.md').read_text(), 'keep')
+        self.assertFalse((destination / 'note.md').exists())
+
+    def test_missing_source_or_mount_never_touches_backup(self):
+        destination = self.drive / 'Universe Backup'
+        destination.mkdir()
+        (destination / 'keep.md').write_text('keep')
+        with self.assertRaises(ValueError):
+            backup.mirror(self.root / 'missing', self.drive, self.state)
+        self.assertEqual((destination / 'keep.md').read_text(), 'keep')
+        with self.assertRaises(ValueError):
+            backup.drive_root(self.root, self.root / 'missing-drive')
+        with self.assertRaises(ValueError):
+            backup.mirror(self.source, self.root / 'missing-drive', self.state)
+        self.assertFalse((self.root / 'missing-drive').exists())
+
+    def test_account_discovery_rejects_zero_and_multiple_accounts(self):
+        with self.assertRaises(ValueError):
+            backup.drive_root(self.root)
+        first = self.root / 'Library/CloudStorage/GoogleDrive-first/My Drive'
+        first.mkdir(parents=True)
+        self.assertEqual(backup.drive_root(self.root), first)
+        second = first.parent.parent / 'GoogleDrive-second/My Drive'
+        second.mkdir(parents=True)
+        with self.assertRaises(ValueError):
+            backup.drive_root(self.root)
+        self.assertEqual(backup.drive_root(self.root, second), second)
+
+    def test_overlapping_paths_and_destination_symlink_are_refused(self):
+        with self.assertRaises(ValueError):
+            backup.paths(self.root, self.drive)
+        nested = self.drive / 'Universe Backup/nested'
+        nested.mkdir(parents=True)
+        with self.assertRaises(ValueError):
+            backup.paths(nested, self.drive)
+        elsewhere = self.root / 'elsewhere'
+        elsewhere.mkdir()
+        other_drive = self.root / 'other-drive'
+        other_drive.mkdir()
+        (other_drive / 'Universe Backup').symlink_to(elsewhere, target_is_directory=True)
+        with self.assertRaises(ValueError):
+            backup.paths(self.source, other_drive)
+
+    def test_concurrent_run_does_not_invoke_rsync(self):
+        self.state.mkdir()
+        with (self.state / 'backup.lock').open('a') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            with patch.object(backup.subprocess, 'run') as run:
+                self.assertEqual(backup.mirror(self.source, self.drive, self.state), 0)
+                run.assert_not_called()
+
+    def test_rsync_failure_is_logged_and_propagated(self):
+        with patch.object(backup.subprocess, 'run', return_value=subprocess.CompletedProcess([], 23)):
+            self.assertEqual(backup.mirror(self.source, self.drive, self.state), 23)
+        self.assertIn('exit=23', (self.state / 'backup.log').read_text())
+
+    def test_install_retains_previous_plist_and_replaces_same_user_job(self):
+        agents = self.root / 'Library/LaunchAgents'
+        agents.mkdir(parents=True)
+        target = agents / (backup.LABEL + '.plist')
+        target.write_bytes(b'previous configuration')
+        with patch.object(backup.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0)) as run:
+            self.assertEqual(backup.install(self.root, self.source, self.drive), 0)
+        plist = plistlib.loads(target.read_bytes())
+        self.assertEqual(plist['Label'], backup.LABEL)
+        self.assertEqual(plist['StartInterval'], 3600)
+        self.assertTrue(plist['RunAtLoad'])
+        self.assertEqual(plist['ProgramArguments'][-1], str(self.drive))
+        self.assertEqual(plist['ProgramArguments'][4], str(self.source))
+        self.assertTrue(any('bootout' in call.args[0] for call in run.call_args_list))
+        self.assertTrue(any('bootstrap' in call.args[0] for call in run.call_args_list))
+        copies = list((self.root / '.cache/universe-backup/launchagent-backups').iterdir())
+        self.assertEqual(len(copies), 1)
+        self.assertEqual(copies[0].read_bytes(), b'previous configuration')
+
+    def test_install_preview_does_not_write_files_or_launch_job(self):
+        output = Mock(buffer=io.BytesIO())
+        with patch.object(backup.sys, 'stdout', output), patch.object(backup.subprocess, 'run') as run:
+            self.assertEqual(backup.install(self.root, self.source, self.drive, True), 0)
+            run.assert_not_called()
+        self.assertEqual(plistlib.loads(output.buffer.getvalue())['StartInterval'], 3600)
+        self.assertFalse((self.root / 'Library/LaunchAgents').exists())
+        self.assertFalse((self.root / '.cache').exists())
+
+
+if __name__ == '__main__':
+    unittest.main()