From 092161a2048ae717e8e60d9d274c3f62ef6de7a4 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 12:08:05 +0100 Subject: [PATCH 1/6] feat(vm): Time Machine over SMB to a Lima machine (stage 1, next to the old layout) macOS 27 refuses a disk image as a new Time Machine destination (tmutil setdestination -> error 45), so the attached-image layout cannot be set up from scratch there. This adds the replacement next to it; the old code stays until the first full backup through the new layout has finished. - vm/: Lima template cmsmb.yaml with idempotent provisioning (packages, rclone v1.75.1 checked against its published SHA256, user tm, directories, systemd units, Samba, FUSE, punchfs build); the units and smb.conf as files. Machine-specific values (Drive folder, image name) come from /etc/cloudmachine/*.env written by the agent - no names in the repo. Descriptor limits from the EMFILE stop of 27.10 (LimitNOFILE=524288 for punchfs and smbd, max open files = 500000). - vm/punchfs: libfuse passthrough_ll.c with hole punch emulated by zeros and xattrs kept in side files (GPLv2, that file only), plus an in-machine test. - cloudmachine-agent vm-setup, vm-prepare-bundle, vm-guard, vm-status. - launchd/vm/: vm-start and vm-guard agents, installed only by vm-setup --install-agents (install-launchd does not pick them up). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 1 + .gitignore | 4 + README.md | 23 +- ...acode.cloudmachine.vm-guard.plist.template | 31 + ...acode.cloudmachine.vm-start.plist.template | 32 + .../CloudMachineAgent/BuildAppCommand.swift | 5 + .../CloudMachineAgent/CloudMachineAgent.swift | 5 + .../CloudMachineAgent/VMCommands.swift | 109 ++ mac-app/Sources/CloudMachineCore/L10n.swift | 1 + .../CloudMachineCore/L10nPolish+VM.swift | 164 +++ .../CloudMachineCore/LaunchdInstaller.swift | 38 + .../CloudMachineCore/VMBundlePrep.swift | 388 +++++ .../Sources/CloudMachineCore/VMGuard.swift | 143 ++ .../Sources/CloudMachineCore/VMLayout.swift | 234 +++ .../Sources/CloudMachineCore/VMSetup.swift | 218 +++ .../Sources/CloudMachineCore/VMStatus.swift | 126 ++ .../VMBundlePrepTests.swift | 127 ++ .../VMGuardDecisionTests.swift | 206 +++ .../CloudMachineAppTests/VMLayoutTests.swift | 195 +++ vm/README.md | 43 + vm/cmsmb.yaml | 113 ++ vm/fuse.conf | 17 + vm/punchfs/README.md | 41 + vm/punchfs/punchfs.c | 1280 +++++++++++++++++ vm/punchfs/test-punchfs.sh | 79 + vm/samba/smb.conf | 27 + vm/systemd/punchfs.service | 21 + vm/systemd/rclone-tm.service | 27 + vm/systemd/smbd.service.d/cloudmachine.conf | 6 + vm/systemd/smbd.service.d/limits.conf | 3 + vm/systemd/tm-bands-bind.service | 20 + 31 files changed, 3726 insertions(+), 1 deletion(-) create mode 100644 launchd/vm/com.renacode.cloudmachine.vm-guard.plist.template create mode 100644 launchd/vm/com.renacode.cloudmachine.vm-start.plist.template create mode 100644 mac-app/Sources/CloudMachineAgent/VMCommands.swift create mode 100644 mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift create mode 100644 mac-app/Sources/CloudMachineCore/VMBundlePrep.swift create mode 100644 mac-app/Sources/CloudMachineCore/VMGuard.swift create mode 100644 mac-app/Sources/CloudMachineCore/VMLayout.swift create mode 100644 mac-app/Sources/CloudMachineCore/VMSetup.swift create mode 100644 mac-app/Sources/CloudMachineCore/VMStatus.swift create mode 100644 mac-app/Tests/CloudMachineAppTests/VMBundlePrepTests.swift create mode 100644 mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift create mode 100644 mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift create mode 100644 vm/README.md create mode 100644 vm/cmsmb.yaml create mode 100644 vm/fuse.conf create mode 100644 vm/punchfs/README.md create mode 100644 vm/punchfs/punchfs.c create mode 100755 vm/punchfs/test-punchfs.sh create mode 100644 vm/samba/smb.conf create mode 100644 vm/systemd/punchfs.service create mode 100644 vm/systemd/rclone-tm.service create mode 100644 vm/systemd/smbd.service.d/cloudmachine.conf create mode 100644 vm/systemd/smbd.service.d/limits.conf create mode 100644 vm/systemd/tm-bands-bind.service diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6283bb6..85ef31b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,6 +47,7 @@ on: - "mac-app/Sources/**" - "mac-app/Resources/**" - "launchd/**" + - "vm/**" - "config/**" - "packaging/homebrew/**" workflow_dispatch: diff --git a/.gitignore b/.gitignore index 830f63c..c75721e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ logs/ # rclone config (contains OAuth tokens) - never commit rclone.conf *.conf +# ...except the machine's templates (Samba, FUSE, systemd drop-ins) - no +# secrets in them; rclone.conf stays ignored there too. +!vm/**/*.conf +vm/**/rclone.conf # Generated launchd files (contain the user's local paths) launchd/*.plist diff --git a/README.md b/README.md index bb9e122..8a2533c 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,25 @@ so they read the same whoever sends them to you. with an Apple Developer ID. The Homebrew cask clears the quarantine flag; a DMG downloaded by hand gets Gatekeeper's "unidentified developer" warning. +### macOS 27: Time Machine over SMB to a local machine (preview) + +macOS 27 no longer accepts a disk image as a new Time Machine destination +(`tmutil setdestination` answers error 45), so the layout above cannot be set +up from scratch there. The replacement, being rolled out next to it: a small +Linux machine on this Mac serves Time Machine an SMB share, and the backup's +bands go from it to Google Drive (details in [`vm/README.md`](vm/README.md)). + +- Requires [Lima](https://lima-vm.io) from Homebrew (`brew install lima`) and + a connected Google Drive (`configure-remote`). +- `cloudmachine-agent vm-setup` creates and starts the machine and prints + a `sudo tmutil setdestination smb://...` command - the one step that needs + an administrator, once. +- When Time Machine has created its empty image, stop the backup and run + `cloudmachine-agent vm-prepare-bundle` (32 MB bands, moved to Drive). +- `cloudmachine-agent vm-status` shows the machine; `vm-setup --install-agents` + adds the agents that start it at login and pause Time Machine while the + upload buffer is full. + --- ## Getting started @@ -176,4 +195,6 @@ repository (`renacode-infra: docs/projekty/cloudmachine/`): ## Licence -MIT. FUSE-T and rclone keep their own licences. +MIT. FUSE-T and rclone keep their own licences. One file is GPLv2: +`vm/punchfs/punchfs.c`, based on libfuse's passthrough example (see +`vm/punchfs/README.md`). diff --git a/launchd/vm/com.renacode.cloudmachine.vm-guard.plist.template b/launchd/vm/com.renacode.cloudmachine.vm-guard.plist.template new file mode 100644 index 0000000..cf92f6e --- /dev/null +++ b/launchd/vm/com.renacode.cloudmachine.vm-guard.plist.template @@ -0,0 +1,31 @@ + + + + + Label + com.renacode.cloudmachine.vm-guard + + ProgramArguments + + __CM_AGENT_BIN__ + vm-guard + + StartInterval + 60 + RunAtLoad + + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin + + StandardOutPath + __CM_LOG_DIR__/launchd-vm-guard.out.log + StandardErrorPath + __CM_LOG_DIR__/launchd-vm-guard.err.log + + diff --git a/launchd/vm/com.renacode.cloudmachine.vm-start.plist.template b/launchd/vm/com.renacode.cloudmachine.vm-start.plist.template new file mode 100644 index 0000000..e990144 --- /dev/null +++ b/launchd/vm/com.renacode.cloudmachine.vm-start.plist.template @@ -0,0 +1,32 @@ + + + + + Label + com.renacode.cloudmachine.vm-start + + ProgramArguments + + /usr/bin/env + limactl + start + --tty=false + cmsmb + + RunAtLoad + + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin + + StandardOutPath + __CM_LOG_DIR__/launchd-vm-start.out.log + StandardErrorPath + __CM_LOG_DIR__/launchd-vm-start.err.log + + diff --git a/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift b/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift index a0bb6fa..70c2f2f 100644 --- a/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift +++ b/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift @@ -130,6 +130,11 @@ struct BuildApp: AsyncParsableCommand { try fm.copyItem( at: projectRoot.appendingPathComponent("launchd"), to: resourcesDir.appendingPathComponent("launchd")) + // The machine template (`vm-setup`): cmsmb.yaml reads its files relative + // to itself, so the directory goes in whole. + try fm.copyItem( + at: projectRoot.appendingPathComponent("vm"), + to: resourcesDir.appendingPathComponent("vm")) let configDir = resourcesDir.appendingPathComponent("config") try fm.createDirectory(at: configDir, withIntermediateDirectories: true) try fm.copyItem( diff --git a/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift b/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift index 40d73b1..6462080 100644 --- a/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift +++ b/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift @@ -32,6 +32,11 @@ struct CloudMachineAgent: AsyncParsableCommand { PrepareShutdown.self, RestartMount.self, DriveStatus.self, + // The machine layout (Time Machine over SMB) - see vm/README.md. + VMSetupCommand.self, + VMPrepareBundle.self, + VMGuardCommand.self, + VMStatusCommand.self, ] ) } diff --git a/mac-app/Sources/CloudMachineAgent/VMCommands.swift b/mac-app/Sources/CloudMachineAgent/VMCommands.swift new file mode 100644 index 0000000..7ebc42f --- /dev/null +++ b/mac-app/Sources/CloudMachineAgent/VMCommands.swift @@ -0,0 +1,109 @@ +import ArgumentParser +import CloudMachineCore +import Foundation + +// The machine layout (Time Machine over SMB to a Lima VM) - see vm/README.md. +// It runs beside the attached-image commands; none of these touch the image. + +struct VMSetupCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "vm-setup", + abstract: L10n.tr( + "Creates or starts the CloudMachine machine (Lima), gives it rclone's configuration and an SMB password, and prints the Time Machine command." + )) + + @Flag( + name: .long, + help: ArgumentHelp( + L10n.tr("Copy rclone.conf into the machine even if it already has one."))) + var replaceRcloneConfig = false + + @Flag( + name: .long, + help: ArgumentHelp(L10n.tr("Also install the vm-start and vm-guard launchd agents."))) + var installAgents = false + + func run() async throws { + let result = await VMSetup.run(replaceRcloneConfig: replaceRcloneConfig) + for line in result.messages { print(line) } + guard result.succeeded else { throw ExitCode.failure } + if installAgents { + let agents = await LaunchdInstaller.installVMAgents() + print(agents.message) + if !agents.succeeded { throw ExitCode.failure } + } + if let command = result.destinationCommand { + // Printed, never logged: it holds the password. + print("") + print( + L10n.tr("Point Time Machine at the machine (once, needs an administrator password):")) + print(" \(command)") + print( + L10n.tr( + "When Time Machine has created its empty image, stop the backup (tmutil stopbackup) and run: cloudmachine-agent vm-prepare-bundle" + )) + } + } +} + +struct VMPrepareBundle: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "vm-prepare-bundle", + abstract: L10n.tr( + "Converts Time Machine's new, empty image in the machine to 32 MB bands and moves the bands to Google Drive." + )) + + @Option( + name: .long, + help: ArgumentHelp( + L10n.tr("Image name in the machine (default: from the computer name)."))) + var bundle: String? + + func run() async throws { + let outcome = await VMBundlePrep.run(bundleOverride: bundle) + print(outcome.message) + if !outcome.succeeded { throw ExitCode.failure } + } +} + +struct VMGuardCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "vm-guard", + abstract: L10n.tr( + "One check of the machine's upload buffer: pauses Time Machine when it fills, resumes what it paused." + )) + + @Option(name: .long, help: ArgumentHelp(L10n.tr("Pause above this many GiB in the buffer."))) + var highGiB: Int = VMGuard.Thresholds().highGiB + + @Option( + name: .long, + help: ArgumentHelp(L10n.tr("Resume below this many GiB (only a backup the guard paused)."))) + var lowGiB: Int = VMGuard.Thresholds().lowGiB + + @Option( + name: .long, + help: ArgumentHelp(L10n.tr("Pause when the machine has less free than this, in GiB.")) + ) + var minFreeGiB: Int = VMGuard.Thresholds().minFreeGiB + + func run() async throws { + let guardian = VMGuard( + thresholds: .init(highGiB: highGiB, lowGiB: lowGiB, minFreeGiB: minFreeGiB)) + await guardian.step() + } +} + +struct VMStatusCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "vm-status", + abstract: L10n.tr( + "Shows the machine: address, services, upload buffer, free space and the Time Machine destination." + )) + + func run() async throws { + let (lines, healthy) = await VMStatus.lines() + for line in lines { print(line) } + if !healthy { throw ExitCode.failure } + } +} diff --git a/mac-app/Sources/CloudMachineCore/L10n.swift b/mac-app/Sources/CloudMachineCore/L10n.swift index 2b77ee6..a6cf616 100644 --- a/mac-app/Sources/CloudMachineCore/L10n.swift +++ b/mac-app/Sources/CloudMachineCore/L10n.swift @@ -61,6 +61,7 @@ public enum L10n { L10nPolish.app, L10nPolish.agent, L10nPolish.safety, + L10nPolish.vm, ] } diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift new file mode 100644 index 0000000..5dab27b --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift @@ -0,0 +1,164 @@ +// Polish translations, keyed by the English text passed to `L10n.tr`. +// The machine layout: vm-setup, vm-prepare-bundle, vm-guard, vm-status. +extension L10nPolish { + static let vm: [String: String] = [ + " WARNING: close to the limit - new bands will fail with EMFILE": + " UWAGA: blisko limitu - nowe pasma skończą się błędem EMFILE", + "%@/bands on Google Drive is not empty - it may hold another backup. NOT overwriting it; check it by hand.": + "%@/bands na Google Drive nie jest pusty - może tam być inna kopia. NIE nadpisuję; sprawdź ręcznie.", + "Address: %@": + "Adres: %@", + "Already prepared (%@ exists) - nothing to do.": + "Już przygotowany (istnieje %@) - nie ma nic do zrobienia.", + "Also install the vm-start and vm-guard launchd agents.": + "Zainstaluj też agentów launchd vm-start i vm-guard.", + "Converts Time Machine's new, empty image in the machine to 32 MB bands and moves the bands to Google Drive.": + "Przerabia nowy, pusty obraz Time Machine w maszynie na pasma 32 MB i przenosi pasma na Google Drive.", + "Copy rclone.conf into the machine even if it already has one.": + "Skopiuj rclone.conf do maszyny, nawet jeśli już go ma.", + "Copying the converted image into the machine failed: %@": + "Kopiowanie przerobionego obrazu do maszyny nie powiodło się: %@", + "Copying the image to the Mac failed: %@": + "Kopiowanie obrazu na Maca nie powiodło się: %@", + "Could not ask Lima about the machine: %@": + "Nie udało się zapytać Limy o maszynę: %@", + "Could not carry the Time Machine files over: %@": + "Nie udało się przenieść plików Time Machine: %@", + "Could not check the image in the machine.": + "Nie udało się sprawdzić obrazu w maszynie.", + "Could not copy the rclone configuration into the machine.": + "Nie udało się skopiować konfiguracji rclone do maszyny.", + "Could not create %@ on Google Drive: %@": + "Nie udało się założyć %@ na Google Drive: %@", + "Could not create %@: %@": + "Nie udało się utworzyć %@: %@", + "Could not find the vm/ directory with the machine template.": + "Nie znaleziono katalogu vm/ z szablonem maszyny.", + "Could not read the image size.": + "Nie udało się odczytać rozmiaru obrazu.", + "Could not set the SMB password of user %@ in the machine.": + "Nie udało się ustawić hasła SMB użytkownika %@ w maszynie.", + "Could not stop smbd in the machine.": + "Nie udało się zatrzymać smbd w maszynie.", + "Could not write %@ in the machine.": + "Nie udało się zapisać %@ w maszynie.", + "Could not write the password file %@.": + "Nie udało się zapisać pliku z hasłem %@.", + "Creates or starts the CloudMachine machine (Lima), gives it rclone's configuration and an SMB password, and prints the Time Machine command.": + "Tworzy albo uruchamia maszynę CloudMachine (Lima), przekazuje jej konfigurację rclone i hasło SMB, i wypisuje polecenie dla Time Machine.", + "Creating the machine %@ (first start downloads Ubuntu and installs packages - several minutes)...": + "Tworzę maszynę %@ (pierwszy start pobiera Ubuntu i instaluje pakiety - kilka minut)...", + "Done: %@ has %@ MB bands on Google Drive; the original is kept in %@ in the machine. Start the backup: tmutil startbackup": + "Gotowe: %@ ma pasma %@ MB na Google Drive; oryginał leży w %@ w maszynie. Uruchom kopię: tmutil startbackup", + "exit code %@, no message": + "kod wyjścia %@, bez komunikatu", + "Free in machine: %@ GiB": + "Wolne w maszynie: %@ GiB", + "fsck_apfs -n found problems: %@": + "fsck_apfs -n znalazł problemy: %@", + "Google Drive folder: %@": + "Folder na Google Drive: %@", + "hdiutil attach of the converted image failed: %@": + "hdiutil attach przerobionego obrazu nie powiódł się: %@", + "hdiutil convert failed: %@": + "hdiutil convert nie powiódł się: %@", + "Image name in the machine (default: from the computer name).": + "Nazwa obrazu w maszynie (domyślnie: z nazwy komputera).", + "Installing the converted image FAILED: %@\nsmbd is left STOPPED so Time Machine does not open a half-installed image. The original is in %@ in the machine.": + "Instalacja przerobionego obrazu NIE POWIODŁA SIĘ: %@\nsmbd zostaje ZATRZYMANY, żeby Time Machine nie otworzył w połowie zainstalowanego obrazu. Oryginał leży w %@ w maszynie.", + "limactl create failed: %@": + "limactl create nie powiódł się: %@", + "limactl list failed: %@": + "limactl list nie powiódł się: %@", + "limactl not found - install Lima: brew install lima": + "Nie znaleziono limactl - zainstaluj Limę: brew install lima", + "limactl start failed: %@\nThe machine's own log: %@": + "limactl start nie powiódł się: %@\nWłasny dziennik maszyny: %@", + "Machine %@ is running.": + "Maszyna %@ działa.", + "Machine %@ was already running.": + "Maszyna %@ już działała.", + "Machine: %@ (%@)": + "Maszyna: %@ (%@)", + "Machine: %@": + "Maszyna: %@", + "Machine: not created - run: cloudmachine-agent vm-setup": + "Maszyna: nie utworzona - uruchom: cloudmachine-agent vm-setup", + "no answer within the time limit": + "brak odpowiedzi w limicie czasu", + "No image %@ in the machine.": + "W maszynie nie ma obrazu %@.", + "No image in the machine yet - point Time Machine at the share and let it create one first.": + "W maszynie nie ma jeszcze obrazu - najpierw wskaż Time Machine udział i pozwól mu go utworzyć.", + "No rclone configuration at %@ - connect Google Drive first (configure-remote).": + "Brak konfiguracji rclone w %@ - najpierw połącz Google Drive (configure-remote).", + "no SMB destination": + "brak celu SMB", + "One check of the machine's upload buffer: pauses Time Machine when it fills, resumes what it paused.": + "Jedno sprawdzenie bufora wysyłki maszyny: wstrzymuje Time Machine, gdy bufor się zapełnia, i wznawia to, co sam wstrzymał.", + "Packing the converted image failed: %@": + "Pakowanie przerobionego obrazu nie powiodło się: %@", + "Packing the image failed: %@": + "Pakowanie obrazu nie powiodło się: %@", + "Pause above this many GiB in the buffer.": + "Wstrzymaj powyżej tylu GiB w buforze.", + "Pause when the machine has less free than this, in GiB.": + "Wstrzymaj, gdy maszyna ma mniej wolnego miejsca niż tyle GiB.", + "Point Time Machine at the machine (once, needs an administrator password):": + "Wskaż Time Machine maszynę (raz, wymaga hasła administratora):", + "punchfs files: %@ open, limit %@": + "Pliki punchfs: %@ otwartych, limit %@", + "rclone configuration already in the machine - kept (--replace-rclone-config replaces it).": + "Konfiguracja rclone już jest w maszynie - zostawiona (--replace-rclone-config ją podmienia).", + "rclone configuration copied into the machine.": + "Konfiguracja rclone skopiowana do maszyny.", + "rclone-tm or punchfs is not running in the machine - see cloudmachine-agent vm-status.": + "rclone-tm albo punchfs nie działa w maszynie - patrz cloudmachine-agent vm-status.", + "Resume below this many GiB (only a backup the guard paused).": + "Wznów poniżej tylu GiB (tylko kopię wstrzymaną przez strażnika).", + "Service %@: %@": + "Usługa %@: %@", + "Several images in the machine (%@) - choose one with --bundle.": + "W maszynie jest kilka obrazów (%@) - wybierz jeden przez --bundle.", + "Shows the machine: address, services, upload buffer, free space and the Time Machine destination.": + "Pokazuje maszynę: adres, usługi, bufor wysyłki, wolne miejsce i cel Time Machine.", + "SMB password stored in %@ (mode 600).": + "Hasło SMB zapisane w %@ (uprawnienia 600).", + "The converted image does not have %@-byte bands.": + "Przerobiony obraz nie ma pasm po %@ bajtów.", + "The converted image has no APFS partition.": + "Przerobiony obraz nie ma partycji APFS.", + "The image already takes %@ MB - this command converts only a new, empty image (under %@ MB).": + "Obraz zajmuje już %@ MB - to polecenie przerabia tylko nowy, pusty obraz (poniżej %@ MB).", + "The image is encrypted - converting encrypted images is not supported.": + "Obraz jest zaszyfrowany - przerabianie zaszyfrowanych obrazów nie jest obsługiwane.", + "The image was not changed; smbd started again.": + "Obraz nie został zmieniony; smbd uruchomiony ponownie.", + "The machine does not answer - is it running? cloudmachine-agent vm-status": + "Maszyna nie odpowiada - czy działa? cloudmachine-agent vm-status", + "The machine has no address on the vzNAT network.": + "Maszyna nie ma adresu w sieci vzNAT.", + "The services in the machine did not start - check: cloudmachine-agent vm-status": + "Usługi w maszynie nie wystartowały - sprawdź: cloudmachine-agent vm-status", + "Time Machine is backing up - stop it first: tmutil stopbackup": + "Time Machine robi kopię - najpierw ją zatrzymaj: tmutil stopbackup", + "Time Machine: %@": + "Time Machine: %@", + "Time Machine: NOT pointing at the machine (%@)": + "Time Machine: NIE wskazuje maszyny (%@)", + "Time Machine: tmutil did not answer": + "Time Machine: tmutil nie odpowiedział", + "tmutil did not answer - not touching the image while its state is unknown.": + "tmutil nie odpowiedział - nie ruszam obrazu, dopóki jego stan jest nieznany.", + "unknown": + "nieznany", + "Unpacking the image failed: %@": + "Rozpakowanie obrazu nie powiodło się: %@", + "Upload buffer: %@ GiB, queued %@, uploading %@": + "Bufor wysyłki: %@ GiB, w kolejce %@, wysyłane %@", + "Upload buffer: no answer from rclone (rc %@)": + "Bufor wysyłki: brak odpowiedzi rclone (rc %@)", + "When Time Machine has created its empty image, stop the backup (tmutil stopbackup) and run: cloudmachine-agent vm-prepare-bundle": + "Gdy Time Machine utworzy pusty obraz, zatrzymaj kopię (tmutil stopbackup) i uruchom: cloudmachine-agent vm-prepare-bundle", + ] +} diff --git a/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift b/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift index 7b4af52..61a780e 100644 --- a/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift +++ b/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift @@ -156,6 +156,44 @@ public enum LaunchdInstaller { templates: templates, into: launchAgentsDir, agentBin: agentBin, logDir: CMPaths.logDir)) } + /// The machine's agents (`launchd/vm/`): `vm-start` and `vm-guard`. + /// + /// A subdirectory on purpose: `install()` installs every template at the + /// top of `launchd/`, and a Mac on the attached-image layout must not get + /// an agent that starts a machine it does not have. These go in only when + /// asked for (`vm-setup --install-agents`). No image detach here: reloading + /// them does not touch the Drive mount. + public static func installVMAgents() async -> CMActionResult { + guard let templatesDir = CMPaths.launchdTemplatesDir?.appendingPathComponent("vm") else { + return CMActionResult( + succeeded: false, message: L10n.tr("Could not find the launchd/ directory with templates.")) + } + guard let resolvedAgentBin = CMPaths.agentBinaryPath else { + return CMActionResult( + succeeded: false, + message: L10n.tr("Could not find the compiled cloudmachine-agent binary.")) + } + let agentBin: URL + do { + agentBin = try stableAgentBinaryPath(resolvedFrom: resolvedAgentBin) + } catch { + return CMActionResult( + succeeded: false, message: L10n.tr("ABORTED: %@", error.localizedDescription)) + } + guard + let templates = try? FileManager.default.contentsOfDirectory( + at: templatesDir, includingPropertiesForKeys: nil) + else { + return CMActionResult( + succeeded: false, + message: L10n.tr("Could not list the templates in %@.", templatesDir.path)) + } + try? FileManager.default.createDirectory(at: launchAgentsDir, withIntermediateDirectories: true) + return installVerdict( + await installAgents( + templates: templates, into: launchAgentsDir, agentBin: agentBin, logDir: CMPaths.logDir)) + } + /// What went in and what did NOT - with a reason, one per agent. /// /// Until 25.09.2026 we collected only `installedLabels`, and failures left no diff --git a/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift b/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift new file mode 100644 index 0000000..8a81674 --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift @@ -0,0 +1,388 @@ +import Foundation + +/// `vm-prepare-bundle`: after Time Machine has created its first, EMPTY image +/// on the machine's local disk, convert it to 32 MB bands and move the bands +/// to Google Drive. +/// +/// Why: Time Machine creates bands of 875 MB (band-size 917504000). Every +/// touched band is uploaded whole, and with bands that large a day of +/// ordinary changes passes Google's 750 GB daily upload limit. 32 MB bands +/// were chosen from measurements of this project. The image's small files +/// (Info.plist, token, the Time Machine plists) stay on ext4, because Samba on +/// rclone fails on renames; only `bands/` becomes a bind of the Drive layer. +/// +/// The order of steps is the safety: nothing in the machine changes until the +/// converted copy has passed `fsck_apfs -n` and the band-size check on the +/// Mac. A failure before the install restarts Samba with the image untouched; +/// a failure during it leaves Samba STOPPED (Time Machine must not see a half +/// installed image) and says where the original was set aside. +public enum VMBundlePrep { + /// 32 MiB in 512-byte sectors, as `hdiutil convert -imagekey` wants it. + public static let bandSectors = 65536 + public static let bandBytes = bandSectors * 512 + /// An empty image is a few hundred MB; anything near this already holds a + /// backup, and converting that by hand is not what this command is for. + public static let maxImageBytes: Int64 = 2 << 30 + + public struct Outcome: Sendable { + public var succeeded: Bool + public var message: String + } + + public static func run(bundleOverride: String?) async -> Outcome { + func fail(_ message: String) -> Outcome { + CMLogger.log("vm-prepare-bundle: \(message.replacingOccurrences(of: "\n", with: " "))") + return Outcome(succeeded: false, message: message) + } + func step(_ text: String) { + CMLogger.log("vm-prepare-bundle: \(text)") + } + + // 1. Time Machine must not be writing. + switch await TimeMachineStatus.runningState() { + case .none: + return fail( + L10n.tr("tmutil did not answer - not touching the image while its state is unknown.")) + case .some(true): + return fail(L10n.tr("Time Machine is backing up - stop it first: tmutil stopbackup")) + case .some(false): + break + } + + // 2. Which image. + guard + let listing = try? await VMLayout.rootShell( + "cd \(VMLayout.Guest.localRoot) && for d in *.sparsebundle; do [ -d \"$d\" ] && printf '%s\\n' \"$d\"; done; true", + timeout: 60), listing.succeeded + else { + return fail( + L10n.tr("The machine does not answer - is it running? cloudmachine-agent vm-status")) + } + let found = listing.stdout.split(separator: "\n").map(String.init) + let computerName = await currentComputerName() + let name: String + switch chooseBundle(found: found, computerName: computerName, override: bundleOverride) { + case .success(let chosen): name = chosen + case .failure(let problem): return fail(problem.message) + } + let bundle = "\(VMLayout.Guest.localRoot)/\(name)" + let qBundle = VMLayout.shellQuoted(bundle) + step("image \(name)") + + // 3. Already done? + if (try? await VMLayout.rootShell("test -s \(VMLayout.Guest.bundleEnv)", timeout: 60))? + .succeeded == true + { + return Outcome( + succeeded: true, + message: L10n.tr( + "Already prepared (%@ exists) - nothing to do.", VMLayout.Guest.bundleEnv)) + } + + // 4. Preconditions in the machine: small image, Drive layers up, no bands on Drive yet. + let checks = + "set -e; du -s -B1 \(qBundle) | cut -f1; " + + "systemctl is-active --quiet rclone-tm punchfs && echo LAYERS-UP || echo LAYERS-DOWN; " + + "if [ -n \"$(ls -A \(VMLayout.Guest.driveRoot)/bands 2>/dev/null)\" ]; then echo BANDS-ON-DRIVE; fi" + guard let checked = try? await VMLayout.rootShell(checks, timeout: 120), checked.succeeded + else { return fail(L10n.tr("Could not check the image in the machine.")) } + let lines = checked.stdout.split(separator: "\n").map(String.init) + guard let used = lines.first.flatMap({ Int64($0) }) else { + return fail(L10n.tr("Could not read the image size.")) + } + if used >= maxImageBytes { + return fail( + L10n.tr( + "The image already takes %@ MB - this command converts only a new, empty image (under %@ MB).", + "\(used >> 20)", "\(maxImageBytes >> 20)")) + } + if !lines.contains("LAYERS-UP") { + return fail( + L10n.tr( + "rclone-tm or punchfs is not running in the machine - see cloudmachine-agent vm-status.")) + } + if lines.contains("BANDS-ON-DRIVE") { + return fail( + L10n.tr( + "%@/bands on Google Drive is not empty - it may hold another backup. NOT overwriting it; check it by hand.", + VMLayout.remote)) + } + + // 5. A working directory on the Mac. + let stamp = Self.stamp() + let work = VMLayout.hostDir.appendingPathComponent("prepare-\(stamp)") + let source = work.appendingPathComponent("source") + let converted = work.appendingPathComponent("converted") + do { + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: converted, withIntermediateDirectories: true) + } catch { + return fail(L10n.tr("Could not create %@: %@", work.path, error.localizedDescription)) + } + defer { try? FileManager.default.removeItem(at: work) } + let staging = "/tmp/cm-prepare-\(stamp)" + + // 6. Samba off, so Time Machine cannot open the image while we copy it. + step("stopping smbd") + guard (try? await VMLayout.rootShell("systemctl stop smbd", timeout: 120))?.succeeded == true + else { return fail(L10n.tr("Could not stop smbd in the machine.")) } + func failRestoringSamba(_ message: String) async -> Outcome { + _ = try? await VMLayout.rootShell("rm -rf \(staging); systemctl start smbd", timeout: 120) + return fail(message + "\n" + L10n.tr("The image was not changed; smbd started again.")) + } + + // 7. Copy to the Mac. + step("copying the image to the Mac") + let pack = + "set -e; install -d -m 700 -o \"$SUDO_USER\" \(staging); " + + "tar -C \(VMLayout.Guest.localRoot) --sparse -cf \(staging)/source.tar \(VMLayout.shellQuoted(name)); " + + "chown \"$SUDO_USER\" \(staging)/source.tar" + let packed = try? await VMLayout.rootShell(pack, timeout: 900) + guard packed?.succeeded == true else { + return await failRestoringSamba( + L10n.tr("Packing the image failed: %@", VMLayout.errorTail(packed))) + } + let sourceTar = work.appendingPathComponent("source.tar") + let fetched = try? await VMLayout.limactl( + ["copy", "\(VMLayout.instanceName):\(staging)/source.tar", sourceTar.path], timeout: 1800) + guard fetched?.succeeded == true else { + return await failRestoringSamba( + L10n.tr("Copying the image to the Mac failed: %@", VMLayout.errorTail(fetched))) + } + let unpacked = try? await ProcessRunner.run( + "/usr/bin/tar", ["-C", source.path, "-xf", sourceTar.path], timeout: 900) + try? FileManager.default.removeItem(at: sourceTar) + guard unpacked?.succeeded == true else { + return await failRestoringSamba( + L10n.tr("Unpacking the image failed: %@", VMLayout.errorTail(unpacked))) + } + let src = source.appendingPathComponent(name) + let dst = converted.appendingPathComponent(name) + + // 8. Convert. + let encrypted = try? await ProcessRunner.run( + "/usr/bin/hdiutil", ["isencrypted", src.path], timeout: 120) + if encrypted?.stdout.contains("encrypted: YES") == true { + return await failRestoringSamba( + L10n.tr("The image is encrypted - converting encrypted images is not supported.")) + } + step("hdiutil convert to \(bandSectors)-sector bands") + let convert = try? await ProcessRunner.run( + "/usr/bin/hdiutil", + [ + "convert", src.path, "-format", "UDSB", "-imagekey", "sparse-band-size=\(bandSectors)", + "-o", dst.path, + ], timeout: 3600) + guard convert?.succeeded == true else { + return await failRestoringSamba( + L10n.tr("hdiutil convert failed: %@", VMLayout.errorTail(convert))) + } + do { + let entries = try FileManager.default.contentsOfDirectory(atPath: src.path) + for entry in entries where entry.hasPrefix("com.apple.TimeMachine.") { + let target = dst.appendingPathComponent(entry) + try? FileManager.default.removeItem(at: target) + try FileManager.default.copyItem( + at: src.appendingPathComponent(entry), to: target) + } + try FileManager.default.createDirectory( + at: dst.appendingPathComponent("mapped"), withIntermediateDirectories: true) + } catch { + return await failRestoringSamba( + L10n.tr("Could not carry the Time Machine files over: %@", error.localizedDescription)) + } + + // 9. Verify - read only. + guard + let info = try? Data(contentsOf: dst.appendingPathComponent("Info.plist")), + bandSize(infoPlist: info) == bandBytes + else { + return await failRestoringSamba( + L10n.tr("The converted image does not have %@-byte bands.", "\(bandBytes)")) + } + switch await fsckReadOnly(dst) { + case .success: step("fsck_apfs -n passed") + case .failure(let problem): return await failRestoringSamba(problem.message) + } + + // 10. Install. + step("installing the converted image") + let convertedTar = work.appendingPathComponent("converted.tar") + let repacked = try? await ProcessRunner.run( + "/usr/bin/tar", ["-C", converted.path, "-cf", convertedTar.path, name], + env: ["COPYFILE_DISABLE": "1"], timeout: 900) + guard repacked?.succeeded == true else { + return await failRestoringSamba( + L10n.tr("Packing the converted image failed: %@", VMLayout.errorTail(repacked))) + } + let pushed = try? await VMLayout.limactl( + ["copy", convertedTar.path, "\(VMLayout.instanceName):\(staging)/converted.tar"], + timeout: 1800) + guard pushed?.succeeded == true else { + return await failRestoringSamba( + L10n.tr( + "Copying the converted image into the machine failed: %@", VMLayout.errorTail(pushed))) + } + + let aside = "\(VMLayout.Guest.backupDir)/\(stamp)" + let install = installScript( + name: name, staging: staging, aside: aside, + envLine: VMLayout.envLine("CM_BUNDLE", name)) + let installed = try? await VMLayout.rootShell(install, timeout: 3600) + guard installed?.succeeded == true else { + return fail( + L10n.tr( + "Installing the converted image FAILED: %@\nsmbd is left STOPPED so Time Machine does not open a half-installed image. The original is in %@ in the machine.", + VMLayout.errorTail(installed), aside)) + } + return Outcome( + succeeded: true, + message: L10n.tr( + "Done: %@ has %@ MB bands on Google Drive; the original is kept in %@ in the machine. Start the backup: tmutil startbackup", + name, "\(bandBytes >> 20)", aside)) + } + + /// The machine-side install, one script so it stops at the first error + /// (`set -e`). Samba is still stopped while it runs. + static func installScript(name: String, staging: String, aside: String, envLine: String) -> String + { + let local = VMLayout.Guest.localRoot + let bundle = VMLayout.shellQuoted("\(local)/\(name)") + let drive = "\(VMLayout.Guest.driveRoot)/bands" + return [ + "set -euo pipefail", + "B=\(bundle)", + // Checked again here: between the first check and now someone may have written there. + "if [ -n \"$(ls -A \(drive) 2>/dev/null)\" ]; then echo 'bands on Drive appeared meanwhile' >&2; exit 3; fi", + "install -d -m 700 \(VMLayout.Guest.backupDir)", + "mv \"$B\" \(VMLayout.shellQuoted(aside))", + "tar -C \(local) -xf \(staging)/converted.tar", + "find \"$B\" -name '._*' -delete", + "count=$(find \"$B/bands\" -type f | wc -l)", + "mkdir -p \(drive)", + "cp -r \"$B/bands/.\" \(drive)/", + "test \"$(find \(drive) -type f | wc -l)\" -eq \"$count\"", + "rm -rf \"$B/bands\" && mkdir \"$B/bands\"", + "chown -R tm:tm \"$B\"", + // punchfs keeps a handle to the directory it saw before mkdir; without a + // restart the bind would show the old, empty view. + "systemctl restart punchfs", + "printf '%s\\n' \(VMLayout.shellQuoted(envLine)) > \(VMLayout.Guest.bundleEnv)", + "systemctl restart tm-bands-bind", + "mountpoint -q \"$B/bands\"", + "test \"$(find \"$B/bands/\" -type f | wc -l)\" -eq \"$count\"", + "rm -rf \(staging)", + "systemctl start smbd", + ].joined(separator: "\n") + } + + // MARK: - Pure parts + + struct Problem: Error, Equatable { + var message: String + } + + /// The image Time Machine named after the computer. Another single image + /// is accepted (a renamed Mac); several, without a match, are a question for + /// the person (`--bundle`), not a guess. + static func chooseBundle(found: [String], computerName: String?, override: String?) -> Result< + String, Problem + > { + if let override { + let wanted = override.hasSuffix(".sparsebundle") ? override : override + ".sparsebundle" + return found.contains(wanted) + ? .success(wanted) + : .failure(Problem(message: L10n.tr("No image %@ in the machine.", wanted))) + } + if let computerName, found.contains(computerName + ".sparsebundle") { + return .success(computerName + ".sparsebundle") + } + if found.count == 1 { return .success(found[0]) } + if found.isEmpty { + return .failure( + Problem( + message: L10n.tr( + "No image in the machine yet - point Time Machine at the share and let it create one first." + ))) + } + return .failure( + Problem( + message: L10n.tr( + "Several images in the machine (%@) - choose one with --bundle.", + found.joined(separator: ", ")))) + } + + static func bandSize(infoPlist: Data) -> Int? { + guard + let plist = try? PropertyListSerialization.propertyList(from: infoPlist, format: nil) + as? [String: Any] + else { return nil } + return (plist["band-size"] as? NSNumber)?.intValue + } + + /// From `hdiutil attach -plist`: the APFS partition to check and the whole + /// disk to detach. + static func attachedDevices(attachPlist: Data) -> (apfs: String?, whole: String?) { + guard + let plist = try? PropertyListSerialization.propertyList(from: attachPlist, format: nil) + as? [String: Any], + let entities = plist["system-entities"] as? [[String: Any]] + else { return (nil, nil) } + var apfs: String? + var whole: String? + for entity in entities { + guard let device = entity["dev-entry"] as? String else { continue } + let hint = entity["content-hint"] as? String + if hint == "Apple_APFS", apfs == nil { apfs = device } + if device.range(of: #"^/dev/disk\d+$"#, options: .regularExpression) != nil, whole == nil { + whole = device + } + } + return (apfs, whole) + } + + static func fsckReadOnly(_ image: URL) async -> Result { + let attach = try? await ProcessRunner.run( + "/usr/bin/hdiutil", + ["attach", "-nomount", "-readonly", "-noverify", "-noautofsck", "-plist", image.path], + timeout: 300) + guard let attach, attach.succeeded else { + return .failure( + Problem( + message: L10n.tr( + "hdiutil attach of the converted image failed: %@", VMLayout.errorTail(attach)))) + } + let devices = attachedDevices(attachPlist: Data(attach.stdout.utf8)) + var fsck: ProcessResult? + if let apfs = devices.apfs { + fsck = try? await ProcessRunner.run("/sbin/fsck_apfs", ["-n", apfs], timeout: 1800) + } + if let whole = devices.whole { + _ = try? await ProcessRunner.run("/usr/bin/hdiutil", ["detach", whole], timeout: 120) + } + guard devices.apfs != nil else { + return .failure(Problem(message: L10n.tr("The converted image has no APFS partition."))) + } + guard fsck?.succeeded == true else { + return .failure( + Problem(message: L10n.tr("fsck_apfs -n found problems: %@", VMLayout.errorTail(fsck)))) + } + return .success(()) + } + + static func currentComputerName() async -> String? { + guard + let result = try? await ProcessRunner.run( + "/usr/sbin/scutil", ["--get", "ComputerName"], timeout: 30), result.succeeded + else { return nil } + let name = result.stdout.trimmingCharacters(in: .whitespacesAndNewlines) + return name.isEmpty ? nil : name + } + + static func stamp() -> String { + let formatter = DateFormatter() + formatter.dateFormat = "yyyyMMdd-HHmmss" + formatter.locale = Locale(identifier: "en_US_POSIX") + return formatter.string(from: Date()) + } +} diff --git a/mac-app/Sources/CloudMachineCore/VMGuard.swift b/mac-app/Sources/CloudMachineCore/VMGuard.swift new file mode 100644 index 0000000..593db16 --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/VMGuard.swift @@ -0,0 +1,143 @@ +import Foundation + +/// Guard of the machine's upload buffer: Time Machine writes to Samba faster +/// than rclone uploads to Google Drive (750 GB a day at most), and the buffer +/// is the machine's 200 GB disk. One `step()` per launchd run (every 60 s). +/// +/// Pause when the buffer holds more than `highGiB` or the disk has less than +/// `minFreeGiB` free. Resume below `lowGiB` - and ONLY if this guard paused +/// the backup (the marker file); a backup the person stopped stays stopped. +/// No answer from the machine or from tmutil means: do nothing. +public struct VMGuard: Sendable { + public struct Thresholds: Equatable, Sendable { + public var highGiB: Int + public var lowGiB: Int + public var minFreeGiB: Int + + /// The values the first full backup ran with (27.10.2026): the buffer is + /// capped at 120 GB (`--vfs-cache-max-size`), so 110 leaves room for what + /// is already in flight. + public init(highGiB: Int = 110, lowGiB: Int = 40, minFreeGiB: Int = 40) { + self.highGiB = highGiB + self.lowGiB = lowGiB + self.minFreeGiB = minFreeGiB + } + } + + public enum Action: Equatable, Sendable { + case pause + case resume + case nothing + } + + /// The decision. Pure, so every rule has a test. + /// + /// - `reading == nil`: the machine did not answer. Nothing - pausing on a + /// guess stops backups for no reason, resuming on one floods a full disk. + /// - over a threshold: pause, but only a backup that is RUNNING (`true`); + /// `nil` from tmutil is not "running". + /// - under `lowGiB` with the marker: resume. Without the marker the guard + /// did not pause it, so it is not the guard's to resume. + public static func decide( + reading: VMLayout.Reading?, backupRunning: Bool?, pausedByGuard: Bool, + thresholds: Thresholds + ) -> Action { + guard let reading else { return .nothing } + let usedGiB = reading.buffer.bytesUsed / gib + let freeGiB = reading.freeBytes / gib + if usedGiB > Int64(thresholds.highGiB) || freeGiB < Int64(thresholds.minFreeGiB) { + return backupRunning == true ? .pause : .nothing + } + if pausedByGuard && usedGiB < Int64(thresholds.lowGiB) { + return .resume + } + return .nothing + } + + static let gib: Int64 = 1 << 30 + + /// What the guard reads and does - replaceable, so the tests walk the whole + /// step (marker included) without a machine or tmutil. + public struct Probes: Sendable { + public var reading: @Sendable () async -> VMLayout.Reading? + public var backupRunning: @Sendable () async -> Bool? + public var markerExists: @Sendable () -> Bool + public var setMarker: @Sendable (Bool) -> Void + public var tmutil: @Sendable (String) async -> Bool + public var log: @Sendable (String) -> Void + public var minute: @Sendable () -> Int + + public static let live = Probes( + reading: { await VMLayout.currentReading() }, + backupRunning: { await TimeMachineStatus.runningState() }, + markerExists: { FileManager.default.fileExists(atPath: VMLayout.guardMarker.path) }, + setMarker: { on in + if on { + try? FileManager.default.createDirectory( + at: VMLayout.hostDir, withIntermediateDirectories: true) + FileManager.default.createFile(atPath: VMLayout.guardMarker.path, contents: Data()) + } else { + try? FileManager.default.removeItem(at: VMLayout.guardMarker) + } + }, + tmutil: { await BufferGuardService.tmutil($0) }, + log: { CMLogger.log($0) }, + minute: { Calendar.current.component(.minute, from: Date()) }) + } + + public var thresholds: Thresholds + var probes: Probes + + public init(thresholds: Thresholds = Thresholds()) { + self.init(thresholds: thresholds, probes: .live) + } + + init(thresholds: Thresholds, probes: Probes) { + self.thresholds = thresholds + self.probes = probes + } + + /// One check. Returns what it decided (for the tests and the CLI). + @discardableResult + public func step() async -> Action { + let reading = await probes.reading() + guard let reading else { + probes.log("VM guard: NO ANSWER from the machine (rclone rc) - neither pausing nor resuming") + return .nothing + } + let running = await probes.backupRunning() + let paused = probes.markerExists() + let action = Self.decide( + reading: reading, backupRunning: running, pausedByGuard: paused, thresholds: thresholds) + let used = reading.buffer.bytesUsed / Self.gib + let free = reading.freeBytes / Self.gib + let queued = reading.buffer.uploadsQueued + switch action { + case .pause: + if await probes.tmutil("stopbackup") { + probes.setMarker(true) + probes.log("VM guard: PAUSE - buffer \(used) GiB, queue \(queued), free \(free) GiB") + } else { + probes.log("VM guard: tmutil stopbackup FAILED - buffer \(used) GiB, free \(free) GiB") + } + case .resume: + // The marker goes only after a successful start. The shell version + // removed it first, so one failed `startbackup` left the backup paused + // for good with nobody left to resume it. + if await probes.tmutil("startbackup") { + probes.setMarker(false) + probes.log("VM guard: RESUME - buffer \(used) GiB, free \(free) GiB") + } else { + probes.log("VM guard: tmutil startbackup FAILED - will try again on the next run") + } + case .nothing: + break + } + if probes.minute() == 0 { + let state = running.map { $0 ? "1" : "0" } ?? "unknown" + probes.log( + "VM guard ok: buffer \(used) GiB, queue \(queued), free \(free) GiB, TM running=\(state)") + } + return action + } +} diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift new file mode 100644 index 0000000..0f5c2d1 --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift @@ -0,0 +1,234 @@ +import Foundation + +/// The CloudMachine machine: a Lima VM that serves Time Machine over SMB and +/// keeps the bands on Google Drive (see `vm/README.md`). +/// +/// Why a machine at all: macOS 27 refuses a disk image as a NEW Time Machine +/// destination (`tmutil setdestination` -> error 45, even for a local +/// sparsebundle), and NetFS refuses SMB to the Mac's own address. A share +/// served from a VM on the Mac is what Time Machine still accepts. +/// +/// This is the new path, next to the attached-image one; neither touches the +/// other. +public enum VMLayout { + /// Lima instance name. Fixed: the launchd agent and the guard look it up. + public static let instanceName = "cmsmb" + /// SMB share name, as in `vm/samba/smb.conf`. + public static let shareName = "CloudMachine" + /// SMB user created by the provisioning script. + public static let smbUser = "tm" + + /// Guest paths (`vm/cmsmb.yaml`, `vm/systemd/*`). + public enum Guest { + public static let localRoot = "/srv/tm-local" + public static let driveRoot = "/srv/tm" + public static let punchRoot = "/srv/tm-punch" + public static let rcloneConfig = "/etc/rclone/rclone.conf" + public static let vmEnv = "/etc/cloudmachine/vm.env" + public static let bundleEnv = "/etc/cloudmachine/bundle.env" + public static let cacheDir = "/var/cache/rclone" + public static let backupDir = "/var/lib/cloudmachine/prepare-backup" + public static let rcAddress = "127.0.0.1:5572" + } + + /// The systemd units the share depends on, in start order. + public static let services = ["rclone-tm", "punchfs", "tm-bands-bind", "smbd"] + + /// Host-side state: `~/.cloudmachine/vm`. + public static var hostDir: URL { + DriveBufferService.root.appendingPathComponent("vm") + } + /// The SMB password of `tm`, mode 600. Never logged. + public static var passwordFile: URL { hostDir.appendingPathComponent("tm-smb.pass") } + /// Present while the guard holds Time Machine paused; it resumes only then. + public static var guardMarker: URL { hostDir.appendingPathComponent("guard-paused") } + + /// `gdrive:CloudMachine/-tm`: a folder of its own, next to the + /// attached image's folder, so the two layouts never share bands. + public static var remote: String { + "\(DriveBufferService.remoteName):\(DriveBufferService.remoteRoot)/\(DriveFolder.name)-tm" + } + + /// `vm/` among the resources (the repo root in a checkout, Resources in the app). + public static var templateDir: URL? { + guard let root = CMPaths.resourcesRoot else { return nil } + let dir = root.appendingPathComponent("vm") + return FileManager.default.fileExists(atPath: dir.appendingPathComponent("cmsmb.yaml").path) + ? dir : nil + } + + /// Lima from Homebrew. `nil` = not installed. + public static var limactlPath: String? { + ["/opt/homebrew/bin/limactl", "/usr/local/bin/limactl"].first { + FileManager.default.isExecutableFile(atPath: $0) + } + } + + /// rclone's configuration on the Mac - the machine gets a copy of it. + public static var hostRcloneConfig: URL { + if let custom = ProcessInfo.processInfo.environment["RCLONE_CONFIG"], !custom.isEmpty { + return URL(fileURLWithPath: custom) + } + return FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".config/rclone/rclone.conf") + } + + // MARK: - Running things in the machine + + /// `limactl` with the given arguments. Throws only when it cannot start; + /// a non-zero exit is in the result. + public static func limactl(_ args: [String], timeout: TimeInterval) async throws + -> ProcessResult + { + guard let lima = limactlPath else { + throw ProcessRunnerError.launchFailed( + L10n.tr("limactl not found - install Lima: brew install lima")) + } + return try await ProcessRunner.run(lima, args, timeout: timeout) + } + + /// A root shell script in the machine. `--workdir /`, because Lima + /// otherwise tries to enter the Mac's current directory in the guest. + public static func rootShell(_ script: String, timeout: TimeInterval) async throws + -> ProcessResult + { + try await limactl( + ["shell", "--workdir", "/", instanceName, "sudo", "bash", "-c", script], timeout: timeout) + } + + /// The last lines of what a failed command said, for an error message. + public static func errorTail(_ result: ProcessResult?, lines: Int = 5) -> String { + guard let result else { return L10n.tr("no answer within the time limit") } + let text = (result.stderr + "\n" + result.stdout) + .split(separator: "\n").map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty } + return text.isEmpty + ? L10n.tr("exit code %@, no message", "\(result.exitCode)") + : text.suffix(lines).joined(separator: "\n") + } + + /// Quotes a value for a bash script (single quotes). + static func shellQuoted(_ value: String) -> String { + "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'" + } + + /// A line of a systemd `EnvironmentFile`: double quotes, with `\` and `"` + /// escaped (systemd unquotes them). + static func envLine(_ key: String, _ value: String) -> String { + let escaped = value.replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + return "\(key)=\"\(escaped)\"" + } + + // MARK: - Parsing (pure, tested) + + /// The instance's status from `limactl list --json` (one JSON object per + /// line). `nil` = no such instance. + static func instanceStatus(listJSON: String, name: String = instanceName) -> String? { + for line in listJSON.split(separator: "\n") { + guard let data = line.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + object["name"] as? String == name + else { continue } + return object["status"] as? String ?? "Unknown" + } + return nil + } + + /// The address the Mac reaches the machine at, from + /// `ip -4 -o addr show scope global`. + /// + /// Lima's own user-mode network (192.168.5.0/24) is reachable only through + /// Lima's port forwarding, not by Time Machine; the vzNAT address is the + /// other one (192.168.64.x on this Mac's measurements, but taken from the + /// machine rather than assumed). + static func vmAddress(ipAddrOutput: String) -> String? { + for line in ipAddrOutput.split(separator: "\n") { + let fields = line.split(separator: " ", omittingEmptySubsequences: true) + guard let index = fields.firstIndex(of: "inet"), index + 1 < fields.count else { continue } + let address = String(fields[index + 1].split(separator: "/")[0]) + if address.hasPrefix("192.168.5.") || address.hasPrefix("127.") { continue } + return address + } + return nil + } + + /// What rclone's `vfs/stats` says about the upload buffer. + public struct BufferStats: Equatable, Sendable { + public var bytesUsed: Int64 + public var uploadsQueued: Int + public var uploadsInProgress: Int + } + + static func bufferStats(vfsStatsJSON: String) -> BufferStats? { + guard let data = vfsStatsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let cache = object["diskCache"] as? [String: Any], + let used = (cache["bytesUsed"] as? NSNumber)?.int64Value + else { return nil } + return BufferStats( + bytesUsed: used, + uploadsQueued: (cache["uploadsQueued"] as? NSNumber)?.intValue ?? 0, + uploadsInProgress: (cache["uploadsInProgress"] as? NSNumber)?.intValue ?? 0) + } + + /// One reading of the machine for the guard and the status: the rclone + /// buffer and the free space on the buffer's disk. + public struct Reading: Equatable, Sendable { + public var buffer: BufferStats + public var freeBytes: Int64 + } + + /// The script prints `vfs/stats` JSON on one line, then `CMFREE `. + static let readingScript = + "curl -s -m 20 -X POST http://\(Guest.rcAddress)/vfs/stats | tr -d '\\n'; echo; " + + "echo CMFREE $(df -B1 --output=avail \(Guest.cacheDir) | tail -1 | tr -dc 0-9)" + + /// `nil` unless BOTH parts are there: half an answer is no answer. + static func reading(scriptOutput: String) -> Reading? { + var stats: BufferStats? + var free: Int64? + for line in scriptOutput.split(separator: "\n") { + if line.hasPrefix("CMFREE ") { + free = Int64(line.dropFirst("CMFREE ".count).trimmingCharacters(in: .whitespaces)) + } else if line.hasPrefix("{") { + stats = bufferStats(vfsStatsJSON: String(line)) + } + } + guard let stats, let free else { return nil } + return Reading(buffer: stats, freeBytes: free) + } + + /// Reads the machine. `nil` = no answer (machine down, rclone not up, time + /// limit) - which callers must treat as "unknown", never as zero. + public static func currentReading() async -> Reading? { + guard let result = try? await rootShell(readingScript, timeout: 40), result.succeeded + else { return nil } + return reading(scriptOutput: result.stdout) + } + + /// The machine's SMB address, or `nil` when it does not answer. + public static func currentAddress() async -> String? { + guard + let result = try? await limactl( + [ + "shell", "--workdir", "/", instanceName, "ip", "-4", "-o", "addr", "show", "scope", + "global", + ], timeout: 30), + result.succeeded + else { return nil } + return vmAddress(ipAddrOutput: result.stdout) + } + + /// `Running`, `Stopped`, ... or `nil` when there is no instance. Throws + /// when `limactl` itself does not answer. + public static func instanceStatus() async throws -> String? { + let result = try await limactl(["list", "--json"], timeout: 60) + guard result.succeeded else { + throw ProcessRunnerError.launchFailed( + L10n.tr( + "limactl list failed: %@", result.stderr.trimmingCharacters(in: .whitespacesAndNewlines))) + } + return instanceStatus(listJSON: result.stdout) + } +} diff --git a/mac-app/Sources/CloudMachineCore/VMSetup.swift b/mac-app/Sources/CloudMachineCore/VMSetup.swift new file mode 100644 index 0000000..348f7f1 --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/VMSetup.swift @@ -0,0 +1,218 @@ +import Foundation +import Security + +/// `vm-setup`: creates or starts the machine from `vm/cmsmb.yaml`, gives it +/// rclone's configuration and the Drive folder, sets the SMB password, and +/// prints the one `tmutil` command only an administrator can run. +/// +/// Every step checks first, so running it again is safe: an existing machine +/// is started, not recreated; an existing password is kept; an existing +/// rclone configuration in the machine is kept unless asked otherwise (rclone +/// in the machine refreshes its own token and writes it back there). +public enum VMSetup { + public struct Result: Sendable { + public var succeeded: Bool + public var messages: [String] + /// The `tmutil setdestination` command - holds the password, so it is + /// printed for the person and never logged. + public var destinationCommand: String? + } + + public static func run(replaceRcloneConfig: Bool) async -> Result { + var messages: [String] = [] + func fail(_ message: String) -> Result { + CMLogger.log("vm-setup: \(message)") + return Result(succeeded: false, messages: messages + [message], destinationCommand: nil) + } + + guard VMLayout.limactlPath != nil else { + return fail(L10n.tr("limactl not found - install Lima: brew install lima")) + } + guard let templates = VMLayout.templateDir else { + return fail(L10n.tr("Could not find the vm/ directory with the machine template.")) + } + guard FileManager.default.fileExists(atPath: VMLayout.hostRcloneConfig.path) else { + return fail( + L10n.tr( + "No rclone configuration at %@ - connect Google Drive first (configure-remote).", + VMLayout.hostRcloneConfig.path)) + } + + // 1. The machine. + let status: String? + do { + status = try await VMLayout.instanceStatus() + } catch { + return fail(L10n.tr("Could not ask Lima about the machine: %@", error.localizedDescription)) + } + if status == nil { + messages.append( + L10n.tr( + "Creating the machine %@ (first start downloads Ubuntu and installs packages - several minutes)...", + VMLayout.instanceName)) + CMLogger.log("vm-setup: creating \(VMLayout.instanceName)") + let template = templates.appendingPathComponent("cmsmb.yaml").path + let created = try? await VMLayout.limactl( + ["create", "--tty=false", "--name", VMLayout.instanceName, template], timeout: 1800) + guard created?.succeeded == true else { + return fail(L10n.tr("limactl create failed: %@", VMLayout.errorTail(created))) + } + } + if status != "Running" { + CMLogger.log("vm-setup: starting \(VMLayout.instanceName)") + let started = try? await VMLayout.limactl( + ["start", "--tty=false", VMLayout.instanceName], timeout: 3600) + guard started?.succeeded == true else { + return fail( + L10n.tr( + "limactl start failed: %@\nThe machine's own log: %@", VMLayout.errorTail(started), + "~/.lima/\(VMLayout.instanceName)/ha.stderr.log")) + } + messages.append(L10n.tr("Machine %@ is running.", VMLayout.instanceName)) + } else { + messages.append(L10n.tr("Machine %@ was already running.", VMLayout.instanceName)) + } + + // 2. The Drive folder. + let env = VMLayout.envLine("CM_REMOTE", VMLayout.remote) + guard + let wroteEnv = try? await VMLayout.rootShell( + "mkdir -p /etc/cloudmachine && printf '%s\\n' \(VMLayout.shellQuoted(env)) > \(VMLayout.Guest.vmEnv)", + timeout: 60), wroteEnv.succeeded + else { return fail(L10n.tr("Could not write %@ in the machine.", VMLayout.Guest.vmEnv)) } + messages.append(L10n.tr("Google Drive folder: %@", VMLayout.remote)) + + // 3. rclone's configuration. + let hasConfig = + (try? await VMLayout.rootShell("test -s \(VMLayout.Guest.rcloneConfig)", timeout: 60))? + .succeeded == true + if !hasConfig || replaceRcloneConfig { + guard await copyIn(VMLayout.hostRcloneConfig, to: VMLayout.Guest.rcloneConfig, mode: "600") + else { return fail(L10n.tr("Could not copy the rclone configuration into the machine.")) } + messages.append(L10n.tr("rclone configuration copied into the machine.")) + } else { + messages.append( + L10n.tr( + "rclone configuration already in the machine - kept (--replace-rclone-config replaces it)." + )) + } + let made = try? await VMLayout.rootShell( + "/usr/local/bin/rclone --config \(VMLayout.Guest.rcloneConfig) mkdir \(VMLayout.shellQuoted(VMLayout.remote))", + timeout: 300) + guard made?.succeeded == true else { + return fail( + L10n.tr( + "Could not create %@ on Google Drive: %@", VMLayout.remote, VMLayout.errorTail(made))) + } + + // 4. The SMB password. + let password: String + switch loadOrCreatePassword() { + case .some(let value): password = value + case .none: + return fail(L10n.tr("Could not write the password file %@.", VMLayout.passwordFile.path)) + } + guard await setSmbPassword() else { + return fail( + L10n.tr("Could not set the SMB password of user %@ in the machine.", VMLayout.smbUser)) + } + messages.append(L10n.tr("SMB password stored in %@ (mode 600).", VMLayout.passwordFile.path)) + + // 5. Services. + let units = VMLayout.services.map { "\($0).service" }.joined(separator: " ") + guard + let started = try? await VMLayout.rootShell( + "systemctl daemon-reload && systemctl start \(units)", timeout: 300), + started.succeeded + else { + return fail( + L10n.tr("The services in the machine did not start - check: cloudmachine-agent vm-status")) + } + + // 6. The address and the command. + guard let address = await VMLayout.currentAddress() else { + return fail(L10n.tr("The machine has no address on the vzNAT network.")) + } + let command = destinationCommand(password: password, address: address) + CMLogger.log("vm-setup: done, machine at \(address)") + return Result(succeeded: true, messages: messages, destinationCommand: command) + } + + /// `sudo tmutil setdestination smb://tm:@
/CloudMachine`. + static func destinationCommand(password: String, address: String) -> String { + let allowed = CharacterSet.alphanumerics + let user = VMLayout.smbUser + let encoded = password.addingPercentEncoding(withAllowedCharacters: allowed) ?? password + return "sudo tmutil setdestination 'smb://\(user):\(encoded)@\(address)/\(VMLayout.shareName)'" + } + + /// Copies a Mac file into the machine as root with the given mode. + /// `limactl copy` writes as the Lima user, so the file goes to a private + /// temporary directory first and is moved with `install`. + static func copyIn(_ source: URL, to destination: String, mode: String) async -> Bool { + let staging = "/tmp/cm-copy-\(UUID().uuidString)" + guard + let made = try? await VMLayout.limactl( + ["shell", "--workdir", "/", VMLayout.instanceName, "install", "-d", "-m", "700", staging], + timeout: 60), made.succeeded, + let copied = try? await VMLayout.limactl( + ["copy", source.path, "\(VMLayout.instanceName):\(staging)/file"], timeout: 300), + copied.succeeded + else { return false } + let moved = try? await VMLayout.rootShell( + "install -D -m \(mode) -o root -g root \(staging)/file \(VMLayout.shellQuoted(destination)); " + + "rc=$?; rm -rf \(staging); exit $rc", timeout: 60) + return moved?.succeeded == true + } + + /// The password from the file, or a new one written with mode 600 from the + /// start (never readable by others, not even for a moment). + static func loadOrCreatePassword(file: URL = VMLayout.passwordFile) -> String? { + if let existing = try? String(contentsOf: file, encoding: .utf8) { + let trimmed = existing.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { return trimmed } + } + try? FileManager.default.createDirectory( + at: file.deletingLastPathComponent(), withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700]) + guard let password = randomPassword() else { return nil } + try? FileManager.default.removeItem(at: file) + guard + FileManager.default.createFile( + atPath: file.path, contents: Data((password + "\n").utf8), + attributes: [.posixPermissions: 0o600]) + else { return nil } + return password + } + + /// 24 characters from [A-Za-z0-9] - safe in a URL, a shell and smbpasswd. + static func randomPassword(length: Int = 24) -> String? { + let alphabet = Array("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") + var result = "" + while result.count < length { + var byte: UInt8 = 0 + guard SecRandomCopyBytes(kSecRandomDefault, 1, &byte) == errSecSuccess else { return nil } + // Rejection sampling: 248 = 4 * 62, so every character is equally likely. + if byte < 248 { result.append(alphabet[Int(byte) % alphabet.count]) } + } + return result + } + + /// `smbpasswd` reads the password from the copied file, never from the + /// command line (`ps` shows that) and never through the log. + static func setSmbPassword() async -> Bool { + let staging = "/tmp/cm-copy-\(UUID().uuidString)" + guard + let made = try? await VMLayout.limactl( + ["shell", "--workdir", "/", VMLayout.instanceName, "install", "-d", "-m", "700", staging], + timeout: 60), made.succeeded, + let copied = try? await VMLayout.limactl( + ["copy", VMLayout.passwordFile.path, "\(VMLayout.instanceName):\(staging)/p"], timeout: 60), + copied.succeeded + else { return false } + let script = + "p=$(head -1 \(staging)/p); rm -rf \(staging); " + + "printf '%s\\n%s\\n' \"$p\" \"$p\" | smbpasswd -s -a \(VMLayout.smbUser) >/dev/null" + return (try? await VMLayout.rootShell(script, timeout: 60))?.succeeded == true + } +} diff --git a/mac-app/Sources/CloudMachineCore/VMStatus.swift b/mac-app/Sources/CloudMachineCore/VMStatus.swift new file mode 100644 index 0000000..108a960 --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/VMStatus.swift @@ -0,0 +1,126 @@ +import Foundation + +/// `vm-status`: one look at the machine - address, services, upload buffer, +/// free space, punchfs descriptors and where Time Machine points. +public enum VMStatus { + public struct Report: Equatable, Sendable { + public var services: [String: String] = [:] + /// Open descriptors of punchfs and its limit. passthrough_ll keeps one + /// per inode it has seen, so a full backup climbs towards the number of + /// bands; at the limit every new band fails with EMFILE (27.10.2026: 910 + /// of 1024, the first copy stopped). + public var punchfsOpenFiles: Int? + public var punchfsFileLimit: Int? + public var reading: VMLayout.Reading? + } + + static let script = + VMLayout.services.map { "echo SVC \($0) $(systemctl is-active \($0))" }.joined(separator: "; ") + + "; pid=$(systemctl show -p MainPID --value punchfs); " + + "if [ \"${pid:-0}\" -gt 0 ]; then echo FD $(ls /proc/$pid/fd | wc -l) " + + "$(awk '/Max open files/{print $4}' /proc/$pid/limits); fi; " + + VMLayout.readingScript + + static func report(scriptOutput: String) -> Report { + var report = Report() + for line in scriptOutput.split(separator: "\n") { + let fields = line.split(separator: " ").map(String.init) + if fields.first == "SVC", fields.count >= 3 { + report.services[fields[1]] = fields[2] + } else if fields.first == "FD", fields.count >= 2 { + report.punchfsOpenFiles = Int(fields[1]) + if fields.count >= 3 { report.punchfsFileLimit = Int(fields[2]) } + } + } + report.reading = VMLayout.reading(scriptOutput: scriptOutput) + return report + } + + /// `URL : smb://...` lines of `tmutil destinationinfo`. + static func destinationURLs(destinationInfo: String) -> [String] { + destinationInfo.split(separator: "\n").compactMap { raw in + let line = raw.trimmingCharacters(in: .whitespaces) + guard line.hasPrefix("URL") else { return nil } + let parts = line.split(separator: ":", maxSplits: 1) + return parts.count == 2 ? parts[1].trimmingCharacters(in: .whitespaces) : nil + } + } + + /// Above this share of the limit the status warns about descriptors. + static let fileWarningRatio = 0.8 + + public static func lines() async -> (lines: [String], healthy: Bool) { + var out: [String] = [] + var healthy = true + let status: String? + do { + status = try await VMLayout.instanceStatus() + } catch { + return ([L10n.tr("Machine: %@", error.localizedDescription)], false) + } + guard let status else { + return ( + [L10n.tr("Machine: not created - run: cloudmachine-agent vm-setup")], false + ) + } + out.append(L10n.tr("Machine: %@ (%@)", VMLayout.instanceName, status)) + guard status == "Running" else { return (out, false) } + + let address = await VMLayout.currentAddress() + out.append(L10n.tr("Address: %@", address ?? L10n.tr("unknown"))) + if address == nil { healthy = false } + + let result = try? await VMLayout.rootShell(script, timeout: 60) + let report = report(scriptOutput: result?.stdout ?? "") + for service in VMLayout.services { + let state = report.services[service] ?? L10n.tr("unknown") + out.append( + L10n.tr("Service %@: %@", service.padding(toLength: 14, withPad: " ", startingAt: 0), state) + ) + // tm-bands-bind is legitimately inactive until vm-prepare-bundle. + if state != "active" && service != "tm-bands-bind" { healthy = false } + } + if let reading = report.reading { + out.append( + L10n.tr( + "Upload buffer: %@ GiB, queued %@, uploading %@", + "\(reading.buffer.bytesUsed / VMGuard.gib)", "\(reading.buffer.uploadsQueued)", + "\(reading.buffer.uploadsInProgress)")) + out.append(L10n.tr("Free in machine: %@ GiB", "\(reading.freeBytes / VMGuard.gib)")) + } else { + out.append( + L10n.tr("Upload buffer: no answer from rclone (rc %@)", VMLayout.Guest.rcAddress)) + healthy = false + } + if let open = report.punchfsOpenFiles { + let limit = report.punchfsFileLimit.map(String.init) ?? "?" + out.append(L10n.tr("punchfs files: %@ open, limit %@", "\(open)", limit)) + if let max = report.punchfsFileLimit, Double(open) > Double(max) * fileWarningRatio { + out.append( + L10n.tr(" WARNING: close to the limit - new bands will fail with EMFILE") + ) + healthy = false + } + } + + let destination = try? await ProcessRunner.run( + "/usr/bin/tmutil", ["destinationinfo"], timeout: TimeMachineStatus.commandTimeout) + if let destination, destination.succeeded || destination.stdout.contains("No destinations") { + let urls = destinationURLs(destinationInfo: destination.stdout) + let ours = urls.first { url in address.map { url.contains("\($0)/") } ?? false } + if let ours { + out.append(L10n.tr("Time Machine: %@", ours)) + } else { + out.append( + L10n.tr( + "Time Machine: NOT pointing at the machine (%@)", + urls.isEmpty ? L10n.tr("no SMB destination") : urls.joined(separator: ", "))) + healthy = false + } + } else { + out.append(L10n.tr("Time Machine: tmutil did not answer")) + healthy = false + } + return (out, healthy) + } +} diff --git a/mac-app/Tests/CloudMachineAppTests/VMBundlePrepTests.swift b/mac-app/Tests/CloudMachineAppTests/VMBundlePrepTests.swift new file mode 100644 index 0000000..8ce3dfb --- /dev/null +++ b/mac-app/Tests/CloudMachineAppTests/VMBundlePrepTests.swift @@ -0,0 +1,127 @@ +import XCTest + +@testable import CloudMachineCore + +/// `vm-prepare-bundle`: choosing the image, reading what hdiutil says, and +/// the order of the machine-side install. +final class VMBundlePrepTests: XCTestCase { + + func testBandSizeIs32MiB() { + XCTAssertEqual(VMBundlePrep.bandBytes, 33_554_432) + } + + // MARK: - Which image + + func testPrefersTheImageNamedAfterTheComputer() { + XCTAssertEqual( + try? VMBundlePrep.chooseBundle( + found: ["Old.sparsebundle", "Studio (2).sparsebundle"], computerName: "Studio (2)", + override: nil + ).get(), "Studio (2).sparsebundle") + } + + func testASingleImageIsTakenEvenUnderAnotherName() { + XCTAssertEqual( + try? VMBundlePrep.chooseBundle( + found: ["Renamed.sparsebundle"], computerName: "Studio", override: nil + ).get(), "Renamed.sparsebundle") + } + + func testSeveralImagesWithoutAMatchAreAQuestion() { + guard + case .failure(let problem) = VMBundlePrep.chooseBundle( + found: ["A.sparsebundle", "B.sparsebundle"], computerName: "C", override: nil) + else { return XCTFail("must not guess between two images") } + XCTAssertTrue(problem.message.contains("--bundle"), problem.message) + } + + func testNoImageYet() { + guard + case .failure = VMBundlePrep.chooseBundle(found: [], computerName: "C", override: nil) + else { return XCTFail("nothing to prepare") } + } + + func testOverrideMustExist() { + XCTAssertEqual( + try? VMBundlePrep.chooseBundle( + found: ["A.sparsebundle", "B.sparsebundle"], computerName: nil, override: "B" + ).get(), "B.sparsebundle") + guard + case .failure = VMBundlePrep.chooseBundle( + found: ["A.sparsebundle"], computerName: nil, override: "Z.sparsebundle") + else { return XCTFail("an override that is not there must fail") } + } + + // MARK: - hdiutil + + func testBandSizeFromInfoPlist() throws { + let plist: [String: Any] = [ + "band-size": 33_554_432, "bundle-backingstore-version": 2, + "diskimage-bundle-type": "com.apple.diskimage.sparsebundle", + ] + let data = try PropertyListSerialization.data( + fromPropertyList: plist, format: .xml, options: 0) + XCTAssertEqual(VMBundlePrep.bandSize(infoPlist: data), 33_554_432) + XCTAssertNil(VMBundlePrep.bandSize(infoPlist: Data("not a plist".utf8))) + } + + func testAttachedDevices() throws { + let plist: [String: Any] = [ + "system-entities": [ + ["content-hint": "GUID_partition_scheme", "dev-entry": "/dev/disk7"], + ["content-hint": "EFI", "dev-entry": "/dev/disk7s1"], + ["content-hint": "Apple_APFS", "dev-entry": "/dev/disk7s2"], + ["dev-entry": "/dev/disk8"], + ["content-hint": "41504653-0000-11AA-AA11-00306543ECAC", "dev-entry": "/dev/disk8s1"], + ] + ] + let data = try PropertyListSerialization.data( + fromPropertyList: plist, format: .xml, options: 0) + let devices = VMBundlePrep.attachedDevices(attachPlist: data) + XCTAssertEqual(devices.apfs, "/dev/disk7s2") + XCTAssertEqual(devices.whole, "/dev/disk7") + } + + // MARK: - The install script + + private func script() -> [String] { + VMBundlePrep.installScript( + name: "Some Mac (3).sparsebundle", staging: "/tmp/cm-prepare-x", aside: "/var/lib/x/1", + envLine: VMLayout.envLine("CM_BUNDLE", "Some Mac (3).sparsebundle") + ).components(separatedBy: "\n") + } + + private func index(_ needle: String, in lines: [String]) -> Int? { + lines.firstIndex { $0.contains(needle) } + } + + func testInstallStopsAtTheFirstError() { + XCTAssertEqual(script().first, "set -euo pipefail") + } + + /// punchfs keeps a handle to the directory it saw before `mkdir`; the bind + /// made before its restart shows an empty view. Samba comes back last. + func testInstallOrder() throws { + let lines = script() + let check = try XCTUnwrap(index("bands on Drive appeared meanwhile", in: lines)) + let aside = try XCTUnwrap(index("mv \"$B\"", in: lines)) + let copy = try XCTUnwrap(index("cp -r", in: lines)) + let restart = try XCTUnwrap(index("systemctl restart punchfs", in: lines)) + let env = try XCTUnwrap(index("/etc/cloudmachine/bundle.env", in: lines)) + let bind = try XCTUnwrap(index("systemctl restart tm-bands-bind", in: lines)) + let mounted = try XCTUnwrap(index("mountpoint -q", in: lines)) + let samba = try XCTUnwrap(index("systemctl start smbd", in: lines)) + XCTAssertLessThan(check, aside, "nothing is moved before Drive is checked again") + XCTAssertLessThan(aside, copy) + XCTAssertLessThan(copy, restart) + XCTAssertLessThan(restart, bind) + XCTAssertLessThan(env, bind) + XCTAssertLessThan(bind, mounted) + XCTAssertEqual(samba, lines.count - 1, "smbd starts only after everything else passed") + XCTAssertNotNil(index("._*", in: lines), "AppleDouble files from the Mac are removed") + } + + func testImageNameIsQuoted() { + XCTAssertTrue(script().contains("B='/srv/tm-local/Some Mac (3).sparsebundle'"), "\(script())") + } +} diff --git a/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift b/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift new file mode 100644 index 0000000..33e8adc --- /dev/null +++ b/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift @@ -0,0 +1,206 @@ +import XCTest + +@testable import CloudMachineCore + +/// The machine's buffer guard (`vm-guard`, port of the shell `tm-guard.sh`). +/// +/// The rules the shell version had, each pinned: pause over a threshold only +/// a RUNNING backup, resume only what the guard itself paused, and do nothing +/// at all when the machine or tmutil does not answer. Walked through +/// `step()` too, because the marker is what ties two runs together - the same +/// reason `BufferGuardDecisionTests` walks whole sequences. +final class VMGuardDecisionTests: XCTestCase { + private let thresholds = VMGuard.Thresholds() + private let gib: Int64 = 1 << 30 + + private func reading(usedGiB: Int64, freeGiB: Int64 = 150, queued: Int = 3) -> VMLayout.Reading { + VMLayout.Reading( + buffer: .init(bytesUsed: usedGiB * gib, uploadsQueued: queued, uploadsInProgress: 1), + freeBytes: freeGiB * gib) + } + + // MARK: - The decision + + func testDefaultsAreTheOnesTheFirstBackupRanWith() { + XCTAssertEqual(thresholds, .init(highGiB: 110, lowGiB: 40, minFreeGiB: 40)) + } + + func testNoAnswerFromTheMachineDoesNothing() { + for paused in [false, true] { + for running in [Bool?.none, true, false] { + XCTAssertEqual( + VMGuard.decide( + reading: nil, backupRunning: running, pausedByGuard: paused, thresholds: thresholds), + .nothing, "no reading must never pause or resume (running=\(String(describing: running)))" + ) + } + } + } + + func testFullBufferPausesARunningBackup() { + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 111), backupRunning: true, pausedByGuard: false, + thresholds: thresholds), .pause) + } + + func testExactlyAtTheHighThresholdIsNotOver() { + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 110), backupRunning: true, pausedByGuard: false, + thresholds: thresholds), .nothing) + } + + func testLowFreeSpacePausesEvenWithASmallBuffer() { + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 5, freeGiB: 39), backupRunning: true, pausedByGuard: false, + thresholds: thresholds), .pause) + } + + func testOverThresholdWithoutARunningBackupDoesNothing() { + for running in [Bool?.none, false] { + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 115), backupRunning: running, pausedByGuard: false, + thresholds: thresholds), .nothing, + "tmutil not answering is not 'running' (running=\(String(describing: running)))") + } + } + + func testResumesOnlyWhatItPaused() { + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 39), backupRunning: false, pausedByGuard: true, + thresholds: thresholds), .resume) + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 0), backupRunning: false, pausedByGuard: false, + thresholds: thresholds), .nothing, + "a backup the person stopped is not the guard's to start") + } + + func testBetweenThresholdsKeepsThePause() { + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 40), backupRunning: false, pausedByGuard: true, + thresholds: thresholds), .nothing, "hysteresis: resume only BELOW the low threshold") + } + + func testDoesNotResumeIntoADiskWithoutRoom() { + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 10, freeGiB: 30), backupRunning: false, pausedByGuard: true, + thresholds: thresholds), .nothing) + } + + // MARK: - Whole steps + + private final class Fake: @unchecked Sendable { + private let lock = NSLock() + private var _reading: VMLayout.Reading? + private var _running: Bool? = true + private var _marker = false + private var _tmutilSucceeds = true + private var _calls: [String] = [] + private var _log: [String] = [] + + private func locked(_ body: () -> T) -> T { + lock.lock() + defer { lock.unlock() } + return body() + } + var reading: VMLayout.Reading? { + get { locked { _reading } } + set { locked { _reading = newValue } } + } + var running: Bool? { + get { locked { _running } } + set { locked { _running = newValue } } + } + var marker: Bool { + get { locked { _marker } } + set { locked { _marker = newValue } } + } + var tmutilSucceeds: Bool { + get { locked { _tmutilSucceeds } } + set { locked { _tmutilSucceeds = newValue } } + } + var calls: [String] { locked { _calls } } + var log: [String] { locked { _log } } + func call(_ command: String) -> Bool { + locked { + _calls.append(command) + return _tmutilSucceeds + } + } + func record(_ line: String) { locked { _log.append(line) } } + + var probes: VMGuard.Probes { + VMGuard.Probes( + reading: { self.reading }, backupRunning: { self.running }, + markerExists: { self.marker }, setMarker: { self.marker = $0 }, + tmutil: { self.call($0) }, log: { self.record($0) }, minute: { 17 }) + } + } + + func testPauseThenResumeAcrossRuns() async { + let fake = Fake() + let guardian = VMGuard(thresholds: thresholds, probes: fake.probes) + + fake.reading = reading(usedGiB: 120) + let first = await guardian.step() + XCTAssertEqual(first, .pause) + XCTAssertEqual(fake.calls, ["stopbackup"]) + XCTAssertTrue(fake.marker, "the marker is what lets the guard resume later") + + fake.running = false + fake.reading = reading(usedGiB: 60) + let second = await guardian.step() + XCTAssertEqual(second, .nothing) + + fake.reading = reading(usedGiB: 20) + let third = await guardian.step() + XCTAssertEqual(third, .resume) + XCTAssertEqual(fake.calls, ["stopbackup", "startbackup"]) + XCTAssertFalse(fake.marker) + } + + func testFailedStopDoesNotClaimThePause() async { + let fake = Fake() + fake.tmutilSucceeds = false + fake.reading = reading(usedGiB: 120) + await VMGuard(thresholds: thresholds, probes: fake.probes).step() + XCTAssertFalse( + fake.marker, + "with no marker after a failed stop the guard will not later start a backup it never paused") + } + + /// The shell version removed the marker BEFORE `startbackup`; one failure + /// left the backup paused with nobody to resume it. + func testFailedResumeKeepsTheMarkerForTheNextRun() async { + let fake = Fake() + fake.marker = true + fake.running = false + fake.tmutilSucceeds = false + fake.reading = reading(usedGiB: 10) + let guardian = VMGuard(thresholds: thresholds, probes: fake.probes) + await guardian.step() + XCTAssertTrue(fake.marker) + + fake.tmutilSucceeds = true + let retry = await guardian.step() + XCTAssertEqual(retry, .resume) + XCTAssertFalse(fake.marker) + } + + func testNoAnswerLeavesTheMarkerAndCallsNothing() async { + let fake = Fake() + fake.marker = true + fake.reading = nil + await VMGuard(thresholds: thresholds, probes: fake.probes).step() + XCTAssertTrue(fake.calls.isEmpty) + XCTAssertTrue(fake.marker) + XCTAssertTrue(fake.log.contains { $0.contains("NO ANSWER") }, "\(fake.log)") + } +} diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift new file mode 100644 index 0000000..f0409b4 --- /dev/null +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -0,0 +1,195 @@ +import XCTest + +@testable import CloudMachineCore + +/// The pure parts of the machine layout: what is read from Lima, the +/// machine and macOS, and what is written into the machine's files. +final class VMLayoutTests: XCTestCase { + + // MARK: - Lima and the network + + func testInstanceStatusFromLimaList() { + let list = """ + {"name":"other","status":"Running"} + {"name":"cmsmb","status":"Stopped","vmType":"vz"} + """ + XCTAssertEqual(VMLayout.instanceStatus(listJSON: list), "Stopped") + XCTAssertNil(VMLayout.instanceStatus(listJSON: #"{"name":"other","status":"Running"}"#)) + XCTAssertNil(VMLayout.instanceStatus(listJSON: "")) + } + + /// Lima's user-mode network is not reachable by Time Machine; the vzNAT + /// address is. + func testAddressSkipsLimaUserNetwork() { + let output = """ + 2: eth0 inet 192.168.5.15/24 metric 200 brd 192.168.5.255 scope global dynamic eth0 + 3: lima0 inet 192.168.64.10/24 metric 100 brd 192.168.64.255 scope global dynamic lima0 + """ + XCTAssertEqual(VMLayout.vmAddress(ipAddrOutput: output), "192.168.64.10") + XCTAssertNil( + VMLayout.vmAddress( + ipAddrOutput: "2: eth0 inet 192.168.5.15/24 scope global dynamic eth0")) + } + + // MARK: - rclone rc + + func testReadingNeedsBothParts() { + let json = + #"{"diskCache":{"bytesUsed":53687091200,"uploadsQueued":12,"uploadsInProgress":8},"metadataCache":{}}"# + let full = VMLayout.reading(scriptOutput: json + "\nCMFREE 107374182400\n") + XCTAssertEqual(full?.buffer.bytesUsed, 53_687_091_200) + XCTAssertEqual(full?.buffer.uploadsQueued, 12) + XCTAssertEqual(full?.buffer.uploadsInProgress, 8) + XCTAssertEqual(full?.freeBytes, 107_374_182_400) + + XCTAssertNil(VMLayout.reading(scriptOutput: json + "\n"), "no free space is no reading") + XCTAssertNil( + VMLayout.reading(scriptOutput: "\nCMFREE 107374182400\n"), + "curl printing nothing (rclone down) is no reading, not an empty buffer") + XCTAssertNil(VMLayout.reading(scriptOutput: "{\"error\":\"x\"}\nCMFREE 1\n")) + XCTAssertNil(VMLayout.reading(scriptOutput: json + "\nCMFREE \n"), "empty df is no reading") + } + + // MARK: - Writing into the machine + + func testShellQuoting() { + XCTAssertEqual(VMLayout.shellQuoted("a b"), "'a b'") + XCTAssertEqual(VMLayout.shellQuoted("it's"), #"'it'\''s'"#) + } + + func testEnvLineSurvivesSpacesAndQuotes() { + XCTAssertEqual( + VMLayout.envLine("CM_BUNDLE", "Some Mac (3).sparsebundle"), + #"CM_BUNDLE="Some Mac (3).sparsebundle""#) + XCTAssertEqual(VMLayout.envLine("K", #"a"b\c"#), #"K="a\"b\\c""#) + } + + // MARK: - Setup + + func testDestinationCommandShape() { + XCTAssertEqual( + VMSetup.destinationCommand(password: "Abc123", address: "192.168.64.10"), + "sudo tmutil setdestination 'smb://tm:Abc123@192.168.64.10/CloudMachine'") + } + + func testRandomPasswordIsLongAndPlain() throws { + let password = try XCTUnwrap(VMSetup.randomPassword()) + XCTAssertEqual(password.count, 24) + XCTAssertTrue(password.allSatisfy { $0.isASCII && ($0.isLetter || $0.isNumber) }) + XCTAssertNotEqual(password, VMSetup.randomPassword()) + } + + func testPasswordFileIsPrivateAndKept() throws { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: dir) } + let file = dir.appendingPathComponent("tm-smb.pass") + + let first = try XCTUnwrap(VMSetup.loadOrCreatePassword(file: file)) + let mode = try FileManager.default.attributesOfItem(atPath: file.path)[.posixPermissions] + XCTAssertEqual((mode as? NSNumber)?.intValue, 0o600) + XCTAssertEqual(VMSetup.loadOrCreatePassword(file: file), first, "running again keeps it") + } + + // MARK: - Status + + func testStatusReport() { + let output = """ + SVC rclone-tm active + SVC punchfs active + SVC tm-bands-bind inactive + SVC smbd active + FD 910 1024 + {"diskCache":{"bytesUsed":1073741824,"uploadsQueued":1,"uploadsInProgress":0}} + CMFREE 2147483648 + """ + let report = VMStatus.report(scriptOutput: output) + XCTAssertEqual(report.services["punchfs"], "active") + XCTAssertEqual(report.services["tm-bands-bind"], "inactive") + XCTAssertEqual(report.punchfsOpenFiles, 910) + XCTAssertEqual(report.punchfsFileLimit, 1024) + XCTAssertEqual(report.reading?.freeBytes, 2_147_483_648) + } + + func testDestinationURLs() { + let info = """ + ==================================================== + Name : CloudMachine + Kind : Network + URL : smb://tm@192.168.64.10/CloudMachine + ID : 1234 + """ + XCTAssertEqual( + VMStatus.destinationURLs(destinationInfo: info), ["smb://tm@192.168.64.10/CloudMachine"]) + } + + // MARK: - The template ships with the repo + + private static let repoRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() + .deletingLastPathComponent() + + /// Every file the Lima template copies in must exist - Lima reads them + /// only when the machine is created, so a missing one shows up there. + func testEveryFileTheTemplateNamesExists() throws { + let vm = Self.repoRoot.appendingPathComponent("vm") + let yaml = try String(contentsOf: vm.appendingPathComponent("cmsmb.yaml"), encoding: .utf8) + let files = yaml.split(separator: "\n").compactMap { line -> String? in + let trimmed = line.trimmingCharacters(in: .whitespaces) + guard trimmed.hasPrefix("file: ") else { return nil } + return String(trimmed.dropFirst("file: ".count)) + } + XCTAssertGreaterThanOrEqual(files.count, 9) + for file in files { + XCTAssertTrue( + FileManager.default.fileExists(atPath: vm.appendingPathComponent(file).path), + "vm/\(file) is named in cmsmb.yaml but missing") + // Lima evaluates data files as Go templates. + let text = try String(contentsOf: vm.appendingPathComponent(file), encoding: .utf8) + XCTAssertFalse(text.contains("{{"), "vm/\(file) would be mangled by Lima's templating") + } + } + + /// The rclone download is pinned to a version AND a checksum, checked + /// before anything is installed. + func testRcloneIsPinnedWithAChecksum() throws { + let yaml = try String( + contentsOf: Self.repoRoot.appendingPathComponent("vm/cmsmb.yaml"), encoding: .utf8) + XCTAssertTrue(yaml.contains("RCLONE_VERSION=v1.75.1")) + XCTAssertNotNil( + yaml.range(of: #"RCLONE_SHA256=[0-9a-f]{64}\n"#, options: .regularExpression)) + XCTAssertTrue(yaml.contains("sha256sum -c -")) + } + + /// The descriptor limits that stopped the first full backup (EMFILE at + /// 910 of 1024) must stay in the templates. + func testDescriptorLimitsAreInTheTemplates() throws { + let vm = Self.repoRoot.appendingPathComponent("vm") + let punchfs = try String( + contentsOf: vm.appendingPathComponent("systemd/punchfs.service"), encoding: .utf8) + let smbd = try String( + contentsOf: vm.appendingPathComponent("systemd/smbd.service.d/limits.conf"), encoding: .utf8) + let smbConf = try String( + contentsOf: vm.appendingPathComponent("samba/smb.conf"), encoding: .utf8) + XCTAssertTrue(punchfs.contains("LimitNOFILE=524288")) + XCTAssertTrue(smbd.contains("LimitNOFILE=524288")) + XCTAssertTrue(smbConf.contains("max open files = 500000")) + } + + /// `install-launchd` installs every template at the TOP of `launchd/`. The + /// machine's agents must not be there: a Mac on the attached-image layout + /// would get an agent starting a machine it does not have. + func testMachineAgentsAreNotInstalledByInstallLaunchd() throws { + let launchd = Self.repoRoot.appendingPathComponent("launchd") + let top = try FileManager.default.contentsOfDirectory(atPath: launchd.path) + XCTAssertFalse(top.contains { $0.contains("vm-") }, "\(top)") + let vm = try FileManager.default.contentsOfDirectory( + atPath: launchd.appendingPathComponent("vm").path + ).sorted() + XCTAssertEqual( + vm, + [ + "com.renacode.cloudmachine.vm-guard.plist.template", + "com.renacode.cloudmachine.vm-start.plist.template", + ]) + } +} diff --git a/vm/README.md b/vm/README.md new file mode 100644 index 0000000..42f7e5f --- /dev/null +++ b/vm/README.md @@ -0,0 +1,43 @@ +# The CloudMachine machine (Time Machine over SMB) + +macOS 27 does not accept a disk image as a new Time Machine destination, so +CloudMachine gives Time Machine what it does accept: an SMB share. The share is +served by a small Linux machine on this Mac (Lima, Apple Virtualization), +and the backup's bands live on Google Drive. + +``` +Time Machine --SMB--> Samba /srv/tm-local/.sparsebundle (ext4: image metadata) + bands/ = bind of /srv/tm-punch/bands + punchfs /srv/tm-punch (hole punch, xattrs) + rclone /srv/tm = gdrive:CloudMachine/-tm +``` + +Each layer is there for a reason: + +- Samba straight on rclone fails with `NT_STATUS_SHARING_VIOLATION`: rclone + derives inodes from a hash of the path, and they change under a rename. + So the image's small files stay on local ext4 and only `bands/` goes to Drive. +- macOS frees bands with a hole punch and keeps Samba's sparse flag and Apple + streams in extended attributes; rclone supports neither. `punchfs` fills + both gaps (see `punchfs/README.md`). +- Time Machine creates bands of 875 MB. `vm-prepare-bundle` converts the new, + empty image to 32 MB bands, so a day of changes fits Google's upload limit. + +## Files + +- `cmsmb.yaml` - the Lima template; its provisioning installs everything and + can run again on every boot. +- `samba/smb.conf`, `fuse.conf`, `systemd/` - copied into the machine as they are. + Machine-specific values come from files `cloudmachine-agent` writes: + `/etc/cloudmachine/vm.env` (the Drive folder) and + `/etc/cloudmachine/bundle.env` (the image name, from the computer name). +- `punchfs/` - the FUSE layer and its test. + +## Commands + + cloudmachine-agent vm-setup # create/start the machine, print the tmutil command + sudo tmutil setdestination smb://tm:@
/CloudMachine + # ...Time Machine creates its empty image; stop the backup... + cloudmachine-agent vm-prepare-bundle # 32 MB bands, bands moved to Drive + cloudmachine-agent vm-status + cloudmachine-agent vm-guard # one check; the launchd agent runs it every minute diff --git a/vm/cmsmb.yaml b/vm/cmsmb.yaml new file mode 100644 index 0000000..7e643f0 --- /dev/null +++ b/vm/cmsmb.yaml @@ -0,0 +1,113 @@ +# CloudMachine: the Linux machine that serves Time Machine over SMB and keeps +# the backup on Google Drive. Created by `cloudmachine-agent vm-setup` +# (`limactl create --name cmsmb vm/cmsmb.yaml`); see vm/README.md. +# +# Every provisioning step below runs again on every boot, so each one checks +# before it acts. The `data` files are rewritten on every boot (Lima's +# default), which is how a changed template reaches an existing machine. +# Files are read relative to this template when the machine is created. +vmType: vz +arch: aarch64 +cpus: 2 +memory: 2GiB +# The disk is also rclone's upload buffer (--vfs-cache-max-size 120G). +disk: 200GiB +images: +- location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img" + arch: aarch64 +# Nothing from the Mac is shared into the machine. +mounts: [] +networks: +# vzNAT gives the machine an address the Mac can reach (192.168.64.x); SMB to +# the Mac's own address is refused by NetFS. +- vzNAT: true +containerd: + system: false + user: false +provision: +- mode: data + path: /etc/samba/smb.conf + file: samba/smb.conf +- mode: data + path: /etc/fuse.conf + file: fuse.conf +- mode: data + path: /etc/systemd/system/rclone-tm.service + file: systemd/rclone-tm.service +- mode: data + path: /etc/systemd/system/punchfs.service + file: systemd/punchfs.service +- mode: data + path: /etc/systemd/system/tm-bands-bind.service + file: systemd/tm-bands-bind.service +- mode: data + path: /etc/systemd/system/smbd.service.d/cloudmachine.conf + file: systemd/smbd.service.d/cloudmachine.conf +- mode: data + path: /etc/systemd/system/smbd.service.d/limits.conf + file: systemd/smbd.service.d/limits.conf +- mode: data + path: /opt/cloudmachine/punchfs/punchfs.c + file: punchfs/punchfs.c +- mode: data + path: /opt/cloudmachine/punchfs/test-punchfs.sh + file: punchfs/test-punchfs.sh + permissions: 755 +- mode: system + script: | + #!/bin/bash + set -eux -o pipefail + export DEBIAN_FRONTEND=noninteractive + + # 1. Packages. + PACKAGES="samba fuse3 unzip curl libfuse3-dev pkg-config build-essential attr" + if ! dpkg -s $PACKAGES >/dev/null 2>&1; then + apt-get update + apt-get install -y $PACKAGES + fi + + # 2. rclone, pinned. Ubuntu's package is years behind; the release zip is + # checked against the sum published in + # https://downloads.rclone.org/v1.75.1/SHA256SUMS before it is installed. + RCLONE_VERSION=v1.75.1 + RCLONE_SHA256=03f2504174034b6d004152ed7369251c9a9ec1f7e0836eda420f5c7a5ec0dff9 + if [ "$(/usr/local/bin/rclone version 2>/dev/null | head -1)" != "rclone $RCLONE_VERSION" ]; then + tmp=$(mktemp -d) + zip="rclone-$RCLONE_VERSION-linux-arm64" + curl -fsSL -o "$tmp/$zip.zip" "https://downloads.rclone.org/$RCLONE_VERSION/$zip.zip" + echo "$RCLONE_SHA256 $tmp/$zip.zip" | sha256sum -c - + unzip -q "$tmp/$zip.zip" -d "$tmp" + install -m 755 "$tmp/$zip/rclone" /usr/local/bin/rclone + rm -rf "$tmp" + fi + + # 3. The SMB user. Its password is set by `cloudmachine-agent vm-setup`. + id tm >/dev/null 2>&1 || useradd -M -s /usr/sbin/nologin tm + + # 4. Directories. + mkdir -p /srv/tm /srv/tm-local /srv/tm-punch /srv/tm-xattr \ + /etc/rclone /etc/cloudmachine /var/cache/rclone /var/lib/cloudmachine + chmod 700 /srv/tm-xattr /etc/rclone /var/lib/cloudmachine + chown tm:tm /srv/tm-local + printf 'TM_UID=%s\nTM_GID=%s\n' "$(id -u tm)" "$(id -g tm)" > /etc/cloudmachine/tm-ids.env + + # 5. punchfs, rebuilt only when its source changed. + src=/opt/cloudmachine/punchfs/punchfs.c + built=/var/lib/cloudmachine/punchfs.built.c + if [ ! -x /usr/local/bin/punchfs ] || ! cmp -s "$src" "$built"; then + gcc -Wall -O2 -D_FILE_OFFSET_BITS=64 -DHAVE_FALLOCATE "$src" \ + $(pkg-config fuse3 --cflags --libs) -o /usr/local/bin/punchfs.new + mv /usr/local/bin/punchfs.new /usr/local/bin/punchfs + cp "$src" "$built" + # A running punchfs keeps the old binary until it restarts. + if systemctl is-active --quiet punchfs; then echo "punchfs rebuilt: restart it when idle"; fi + fi + + # 6. Services. They wait for /etc/rclone/rclone.conf (ConditionPathExists), + # which `vm-setup` copies in, and the bands bind waits for + # /etc/cloudmachine/bundle.env, which `vm-prepare-bundle` writes. + systemctl daemon-reload + systemctl enable rclone-tm.service punchfs.service tm-bands-bind.service smbd.service + if [ -s /etc/rclone/rclone.conf ] && [ -s /etc/cloudmachine/vm.env ]; then + systemctl start rclone-tm.service punchfs.service tm-bands-bind.service smbd.service + fi diff --git a/vm/fuse.conf b/vm/fuse.conf new file mode 100644 index 0000000..c292f74 --- /dev/null +++ b/vm/fuse.conf @@ -0,0 +1,17 @@ +# The file /etc/fuse.conf allows for the following parameters: +# +# user_allow_other - Using the allow_other mount option works fine as root, in +# order to have it work as user you need user_allow_other in /etc/fuse.conf as +# well. (This option allows users to use the allow_other option.) You need +# allow_other if you want users other than the owner to access a mounted fuse. +# This option must appear on a line by itself. There is no value, just the +# presence of the option. + +user_allow_other + + +# mount_max = n - this option sets the maximum number of mounts. +# Currently (2014) it must be typed exactly as shown +# (with a single space before and after the equals sign). + +#mount_max = 1000 diff --git a/vm/punchfs/README.md b/vm/punchfs/README.md new file mode 100644 index 0000000..c2acc6d --- /dev/null +++ b/vm/punchfs/README.md @@ -0,0 +1,41 @@ +# punchfs + +`punchfs.c` is libfuse's `example/passthrough_ll.c` (libfuse 3.14, the version +in Ubuntu 24.04) with two changes. It runs inside the CloudMachine machine, +between rclone's Google Drive mount (`/srv/tm`) and Samba (`/srv/tm-punch`). + +## What changed and why + +- **`fallocate` (hole punch, zero range).** macOS (DiskImages2) frees a band + of a sparse bundle by punching a hole in it over SMB. rclone's FUSE mount + has no `fallocate`, answers `EOPNOTSUPP`, and the refusal makes macOS drop + the whole image. punchfs passes the call on and, when the layer below + refuses it, writes zeros over the range instead - never past the end of + the file for a punch or a keep-size zero range; a plain allocation extends + the file. +- **Extended attributes.** Samba keeps the sparse flag (`user.DOSATTRIB`) + and the Apple metadata streams (`streams_xattr`) in extended attributes, + and rclone has none. Without them macOS treats the band files as not + sparse and refuses to erase a band. punchfs stores each attribute as a + small file `/srv/tm-xattr//` on the machine's local disk. + rclone's inode is a hash of the path, so it stays the same for a band. + +Everything else is upstream code, unchanged. + +## Building and testing + +The machine's provisioning builds it (`vm/cmsmb.yaml`): + + gcc -Wall -O2 -D_FILE_OFFSET_BITS=64 -DHAVE_FALLOCATE punchfs.c \ + $(pkg-config fuse3 --cflags --libs) -o /usr/local/bin/punchfs + +`test-punchfs.sh` checks both changes on a throwaway rclone mount of a local +directory, without touching Google Drive: + + limactl shell cmsmb sudo /opt/cloudmachine/punchfs/test-punchfs.sh + +## Licence + +`punchfs.c` is distributed under the terms of the **GNU GPLv2**, like the +libfuse example it is based on (see its header). This applies to that one +file only; the rest of the CloudMachine repository is MIT. diff --git a/vm/punchfs/punchfs.c b/vm/punchfs/punchfs.c new file mode 100644 index 0000000..420de19 --- /dev/null +++ b/vm/punchfs/punchfs.c @@ -0,0 +1,1280 @@ +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2007 Miklos Szeredi + + This program can be distributed under the terms of the GNU GPLv2. + See the file COPYING. +*/ + +/** @file + * + * This file system mirrors the existing file system hierarchy of the + * system, starting at the root file system. This is implemented by + * just "passing through" all requests to the corresponding user-space + * libc functions. In contrast to passthrough.c and passthrough_fh.c, + * this implementation uses the low-level API. Its performance should + * be the least bad among the three, but many operations are not + * implemented. In particular, it is not possible to remove files (or + * directories) because the code necessary to defer actual removal + * until the file is not opened anymore would make the example much + * more complicated. + * + * When writeback caching is enabled (-o writeback mount option), it + * is only possible to write to files for which the mounting user has + * read permissions. This is because the writeback cache requires the + * kernel to be able to issue read requests for all files (which the + * passthrough filesystem cannot satisfy if it can't read the file in + * the underlying filesystem). + * + * Compile with: + * + * gcc -Wall passthrough_ll.c `pkg-config fuse3 --cflags --libs` -o passthrough_ll + * + * ## Source code ## + * \include passthrough_ll.c + */ + +#define _GNU_SOURCE +#define FUSE_USE_VERSION 34 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "passthrough_helpers.h" + +/* We are re-using pointers to our `struct lo_inode` and `struct + lo_dirp` elements as inodes. This means that we must be able to + store uintptr_t values in a fuse_ino_t variable. The following + incantation checks this condition at compile time. */ +#if defined(__GNUC__) && (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 6) && !defined __cplusplus +_Static_assert(sizeof(fuse_ino_t) >= sizeof(uintptr_t), + "fuse_ino_t too small to hold uintptr_t values!"); +#else +struct _uintptr_to_must_hold_fuse_ino_t_dummy_struct \ + { unsigned _uintptr_to_must_hold_fuse_ino_t: + ((sizeof(fuse_ino_t) >= sizeof(uintptr_t)) ? 1 : -1); }; +#endif + +struct lo_inode { + struct lo_inode *next; /* protected by lo->mutex */ + struct lo_inode *prev; /* protected by lo->mutex */ + int fd; + ino_t ino; + dev_t dev; + uint64_t refcount; /* protected by lo->mutex */ +}; + +enum { + CACHE_NEVER, + CACHE_NORMAL, + CACHE_ALWAYS, +}; + +struct lo_data { + pthread_mutex_t mutex; + int debug; + int writeback; + int flock; + int xattr; + const char *source; + double timeout; + int cache; + int timeout_set; + struct lo_inode root; /* protected by lo->mutex */ +}; + +static const struct fuse_opt lo_opts[] = { + { "writeback", + offsetof(struct lo_data, writeback), 1 }, + { "no_writeback", + offsetof(struct lo_data, writeback), 0 }, + { "source=%s", + offsetof(struct lo_data, source), 0 }, + { "flock", + offsetof(struct lo_data, flock), 1 }, + { "no_flock", + offsetof(struct lo_data, flock), 0 }, + { "xattr", + offsetof(struct lo_data, xattr), 1 }, + { "no_xattr", + offsetof(struct lo_data, xattr), 0 }, + { "timeout=%lf", + offsetof(struct lo_data, timeout), 0 }, + { "timeout=", + offsetof(struct lo_data, timeout_set), 1 }, + { "cache=never", + offsetof(struct lo_data, cache), CACHE_NEVER }, + { "cache=auto", + offsetof(struct lo_data, cache), CACHE_NORMAL }, + { "cache=always", + offsetof(struct lo_data, cache), CACHE_ALWAYS }, + + FUSE_OPT_END +}; + +static void passthrough_ll_help(void) +{ + printf( +" -o writeback Enable writeback\n" +" -o no_writeback Disable write back\n" +" -o source=/home/dir Source directory to be mounted\n" +" -o flock Enable flock\n" +" -o no_flock Disable flock\n" +" -o xattr Enable xattr\n" +" -o no_xattr Disable xattr\n" +" -o timeout=1.0 Caching timeout\n" +" -o timeout=0/1 Timeout is set\n" +" -o cache=never Disable cache\n" +" -o cache=auto Auto enable cache\n" +" -o cache=always Cache always\n"); +} + +static struct lo_data *lo_data(fuse_req_t req) +{ + return (struct lo_data *) fuse_req_userdata(req); +} + +static struct lo_inode *lo_inode(fuse_req_t req, fuse_ino_t ino) +{ + if (ino == FUSE_ROOT_ID) + return &lo_data(req)->root; + else + return (struct lo_inode *) (uintptr_t) ino; +} + +static int lo_fd(fuse_req_t req, fuse_ino_t ino) +{ + return lo_inode(req, ino)->fd; +} + +static bool lo_debug(fuse_req_t req) +{ + return lo_data(req)->debug != 0; +} + +static void lo_init(void *userdata, + struct fuse_conn_info *conn) +{ + struct lo_data *lo = (struct lo_data*) userdata; + + if(conn->capable & FUSE_CAP_EXPORT_SUPPORT) + conn->want |= FUSE_CAP_EXPORT_SUPPORT; + + if (lo->writeback && + conn->capable & FUSE_CAP_WRITEBACK_CACHE) { + if (lo->debug) + fuse_log(FUSE_LOG_DEBUG, "lo_init: activating writeback\n"); + conn->want |= FUSE_CAP_WRITEBACK_CACHE; + } + if (lo->flock && conn->capable & FUSE_CAP_FLOCK_LOCKS) { + if (lo->debug) + fuse_log(FUSE_LOG_DEBUG, "lo_init: activating flock locks\n"); + conn->want |= FUSE_CAP_FLOCK_LOCKS; + } +} + +static void lo_destroy(void *userdata) +{ + struct lo_data *lo = (struct lo_data*) userdata; + + while (lo->root.next != &lo->root) { + struct lo_inode* next = lo->root.next; + lo->root.next = next->next; + free(next); + } +} + +static void lo_getattr(fuse_req_t req, fuse_ino_t ino, + struct fuse_file_info *fi) +{ + int res; + struct stat buf; + struct lo_data *lo = lo_data(req); + + (void) fi; + + res = fstatat(lo_fd(req, ino), "", &buf, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); + if (res == -1) + return (void) fuse_reply_err(req, errno); + + fuse_reply_attr(req, &buf, lo->timeout); +} + +static void lo_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr, + int valid, struct fuse_file_info *fi) +{ + int saverr; + char procname[64]; + struct lo_inode *inode = lo_inode(req, ino); + int ifd = inode->fd; + int res; + + if (valid & FUSE_SET_ATTR_MODE) { + if (fi) { + res = fchmod(fi->fh, attr->st_mode); + } else { + sprintf(procname, "/proc/self/fd/%i", ifd); + res = chmod(procname, attr->st_mode); + } + if (res == -1) + goto out_err; + } + if (valid & (FUSE_SET_ATTR_UID | FUSE_SET_ATTR_GID)) { + uid_t uid = (valid & FUSE_SET_ATTR_UID) ? + attr->st_uid : (uid_t) -1; + gid_t gid = (valid & FUSE_SET_ATTR_GID) ? + attr->st_gid : (gid_t) -1; + + res = fchownat(ifd, "", uid, gid, + AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); + if (res == -1) + goto out_err; + } + if (valid & FUSE_SET_ATTR_SIZE) { + if (fi) { + res = ftruncate(fi->fh, attr->st_size); + } else { + sprintf(procname, "/proc/self/fd/%i", ifd); + res = truncate(procname, attr->st_size); + } + if (res == -1) + goto out_err; + } + if (valid & (FUSE_SET_ATTR_ATIME | FUSE_SET_ATTR_MTIME)) { + struct timespec tv[2]; + + tv[0].tv_sec = 0; + tv[1].tv_sec = 0; + tv[0].tv_nsec = UTIME_OMIT; + tv[1].tv_nsec = UTIME_OMIT; + + if (valid & FUSE_SET_ATTR_ATIME_NOW) + tv[0].tv_nsec = UTIME_NOW; + else if (valid & FUSE_SET_ATTR_ATIME) + tv[0] = attr->st_atim; + + if (valid & FUSE_SET_ATTR_MTIME_NOW) + tv[1].tv_nsec = UTIME_NOW; + else if (valid & FUSE_SET_ATTR_MTIME) + tv[1] = attr->st_mtim; + + if (fi) + res = futimens(fi->fh, tv); + else { + sprintf(procname, "/proc/self/fd/%i", ifd); + res = utimensat(AT_FDCWD, procname, tv, 0); + } + if (res == -1) + goto out_err; + } + + return lo_getattr(req, ino, fi); + +out_err: + saverr = errno; + fuse_reply_err(req, saverr); +} + +static struct lo_inode *lo_find(struct lo_data *lo, struct stat *st) +{ + struct lo_inode *p; + struct lo_inode *ret = NULL; + + pthread_mutex_lock(&lo->mutex); + for (p = lo->root.next; p != &lo->root; p = p->next) { + if (p->ino == st->st_ino && p->dev == st->st_dev) { + assert(p->refcount > 0); + ret = p; + ret->refcount++; + break; + } + } + pthread_mutex_unlock(&lo->mutex); + return ret; +} + +static int lo_do_lookup(fuse_req_t req, fuse_ino_t parent, const char *name, + struct fuse_entry_param *e) +{ + int newfd; + int res; + int saverr; + struct lo_data *lo = lo_data(req); + struct lo_inode *inode; + + memset(e, 0, sizeof(*e)); + e->attr_timeout = lo->timeout; + e->entry_timeout = lo->timeout; + + newfd = openat(lo_fd(req, parent), name, O_PATH | O_NOFOLLOW); + if (newfd == -1) + goto out_err; + + res = fstatat(newfd, "", &e->attr, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); + if (res == -1) + goto out_err; + + inode = lo_find(lo_data(req), &e->attr); + if (inode) { + close(newfd); + newfd = -1; + } else { + struct lo_inode *prev, *next; + + saverr = ENOMEM; + inode = calloc(1, sizeof(struct lo_inode)); + if (!inode) + goto out_err; + + inode->refcount = 1; + inode->fd = newfd; + inode->ino = e->attr.st_ino; + inode->dev = e->attr.st_dev; + + pthread_mutex_lock(&lo->mutex); + prev = &lo->root; + next = prev->next; + next->prev = inode; + inode->next = next; + inode->prev = prev; + prev->next = inode; + pthread_mutex_unlock(&lo->mutex); + } + e->ino = (uintptr_t) inode; + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, " %lli/%s -> %lli\n", + (unsigned long long) parent, name, (unsigned long long) e->ino); + + return 0; + +out_err: + saverr = errno; + if (newfd != -1) + close(newfd); + return saverr; +} + +static void lo_lookup(fuse_req_t req, fuse_ino_t parent, const char *name) +{ + struct fuse_entry_param e; + int err; + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, "lo_lookup(parent=%" PRIu64 ", name=%s)\n", + parent, name); + + err = lo_do_lookup(req, parent, name, &e); + if (err) + fuse_reply_err(req, err); + else + fuse_reply_entry(req, &e); +} + +static void lo_mknod_symlink(fuse_req_t req, fuse_ino_t parent, + const char *name, mode_t mode, dev_t rdev, + const char *link) +{ + int res; + int saverr; + struct lo_inode *dir = lo_inode(req, parent); + struct fuse_entry_param e; + + res = mknod_wrapper(dir->fd, name, link, mode, rdev); + + saverr = errno; + if (res == -1) + goto out; + + saverr = lo_do_lookup(req, parent, name, &e); + if (saverr) + goto out; + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, " %lli/%s -> %lli\n", + (unsigned long long) parent, name, (unsigned long long) e.ino); + + fuse_reply_entry(req, &e); + return; + +out: + fuse_reply_err(req, saverr); +} + +static void lo_mknod(fuse_req_t req, fuse_ino_t parent, + const char *name, mode_t mode, dev_t rdev) +{ + lo_mknod_symlink(req, parent, name, mode, rdev, NULL); +} + +static void lo_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name, + mode_t mode) +{ + lo_mknod_symlink(req, parent, name, S_IFDIR | mode, 0, NULL); +} + +static void lo_symlink(fuse_req_t req, const char *link, + fuse_ino_t parent, const char *name) +{ + lo_mknod_symlink(req, parent, name, S_IFLNK, 0, link); +} + +static void lo_link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t parent, + const char *name) +{ + int res; + struct lo_data *lo = lo_data(req); + struct lo_inode *inode = lo_inode(req, ino); + struct fuse_entry_param e; + char procname[64]; + int saverr; + + memset(&e, 0, sizeof(struct fuse_entry_param)); + e.attr_timeout = lo->timeout; + e.entry_timeout = lo->timeout; + + sprintf(procname, "/proc/self/fd/%i", inode->fd); + res = linkat(AT_FDCWD, procname, lo_fd(req, parent), name, + AT_SYMLINK_FOLLOW); + if (res == -1) + goto out_err; + + res = fstatat(inode->fd, "", &e.attr, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); + if (res == -1) + goto out_err; + + pthread_mutex_lock(&lo->mutex); + inode->refcount++; + pthread_mutex_unlock(&lo->mutex); + e.ino = (uintptr_t) inode; + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, " %lli/%s -> %lli\n", + (unsigned long long) parent, name, + (unsigned long long) e.ino); + + fuse_reply_entry(req, &e); + return; + +out_err: + saverr = errno; + fuse_reply_err(req, saverr); +} + +static void lo_rmdir(fuse_req_t req, fuse_ino_t parent, const char *name) +{ + int res; + + res = unlinkat(lo_fd(req, parent), name, AT_REMOVEDIR); + + fuse_reply_err(req, res == -1 ? errno : 0); +} + +static void lo_rename(fuse_req_t req, fuse_ino_t parent, const char *name, + fuse_ino_t newparent, const char *newname, + unsigned int flags) +{ + int res; + + if (flags) { + fuse_reply_err(req, EINVAL); + return; + } + + res = renameat(lo_fd(req, parent), name, + lo_fd(req, newparent), newname); + + fuse_reply_err(req, res == -1 ? errno : 0); +} + +static void lo_unlink(fuse_req_t req, fuse_ino_t parent, const char *name) +{ + int res; + + res = unlinkat(lo_fd(req, parent), name, 0); + + fuse_reply_err(req, res == -1 ? errno : 0); +} + +static void unref_inode(struct lo_data *lo, struct lo_inode *inode, uint64_t n) +{ + if (!inode) + return; + + pthread_mutex_lock(&lo->mutex); + assert(inode->refcount >= n); + inode->refcount -= n; + if (!inode->refcount) { + struct lo_inode *prev, *next; + + prev = inode->prev; + next = inode->next; + next->prev = prev; + prev->next = next; + + pthread_mutex_unlock(&lo->mutex); + close(inode->fd); + free(inode); + + } else { + pthread_mutex_unlock(&lo->mutex); + } +} + +static void lo_forget_one(fuse_req_t req, fuse_ino_t ino, uint64_t nlookup) +{ + struct lo_data *lo = lo_data(req); + struct lo_inode *inode = lo_inode(req, ino); + + if (lo_debug(req)) { + fuse_log(FUSE_LOG_DEBUG, " forget %lli %lli -%lli\n", + (unsigned long long) ino, + (unsigned long long) inode->refcount, + (unsigned long long) nlookup); + } + + unref_inode(lo, inode, nlookup); +} + +static void lo_forget(fuse_req_t req, fuse_ino_t ino, uint64_t nlookup) +{ + lo_forget_one(req, ino, nlookup); + fuse_reply_none(req); +} + +static void lo_forget_multi(fuse_req_t req, size_t count, + struct fuse_forget_data *forgets) +{ + int i; + + for (i = 0; i < count; i++) + lo_forget_one(req, forgets[i].ino, forgets[i].nlookup); + fuse_reply_none(req); +} + +static void lo_readlink(fuse_req_t req, fuse_ino_t ino) +{ + char buf[PATH_MAX + 1]; + int res; + + res = readlinkat(lo_fd(req, ino), "", buf, sizeof(buf)); + if (res == -1) + return (void) fuse_reply_err(req, errno); + + if (res == sizeof(buf)) + return (void) fuse_reply_err(req, ENAMETOOLONG); + + buf[res] = '\0'; + + fuse_reply_readlink(req, buf); +} + +struct lo_dirp { + DIR *dp; + struct dirent *entry; + off_t offset; +}; + +static struct lo_dirp *lo_dirp(struct fuse_file_info *fi) +{ + return (struct lo_dirp *) (uintptr_t) fi->fh; +} + +static void lo_opendir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) +{ + int error = ENOMEM; + struct lo_data *lo = lo_data(req); + struct lo_dirp *d; + int fd; + + d = calloc(1, sizeof(struct lo_dirp)); + if (d == NULL) + goto out_err; + + fd = openat(lo_fd(req, ino), ".", O_RDONLY); + if (fd == -1) + goto out_errno; + + d->dp = fdopendir(fd); + if (d->dp == NULL) + goto out_errno; + + d->offset = 0; + d->entry = NULL; + + fi->fh = (uintptr_t) d; + if (lo->cache == CACHE_ALWAYS) + fi->cache_readdir = 1; + fuse_reply_open(req, fi); + return; + +out_errno: + error = errno; +out_err: + if (d) { + if (fd != -1) + close(fd); + free(d); + } + fuse_reply_err(req, error); +} + +static int is_dot_or_dotdot(const char *name) +{ + return name[0] == '.' && (name[1] == '\0' || + (name[1] == '.' && name[2] == '\0')); +} + +static void lo_do_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, + off_t offset, struct fuse_file_info *fi, int plus) +{ + struct lo_dirp *d = lo_dirp(fi); + char *buf; + char *p; + size_t rem = size; + int err; + + (void) ino; + + buf = calloc(1, size); + if (!buf) { + err = ENOMEM; + goto error; + } + p = buf; + + if (offset != d->offset) { + seekdir(d->dp, offset); + d->entry = NULL; + d->offset = offset; + } + while (1) { + size_t entsize; + off_t nextoff; + const char *name; + + if (!d->entry) { + errno = 0; + d->entry = readdir(d->dp); + if (!d->entry) { + if (errno) { // Error + err = errno; + goto error; + } else { // End of stream + break; + } + } + } + nextoff = d->entry->d_off; + name = d->entry->d_name; + fuse_ino_t entry_ino = 0; + if (plus) { + struct fuse_entry_param e; + if (is_dot_or_dotdot(name)) { + e = (struct fuse_entry_param) { + .attr.st_ino = d->entry->d_ino, + .attr.st_mode = d->entry->d_type << 12, + }; + } else { + err = lo_do_lookup(req, ino, name, &e); + if (err) + goto error; + entry_ino = e.ino; + } + + entsize = fuse_add_direntry_plus(req, p, rem, name, + &e, nextoff); + } else { + struct stat st = { + .st_ino = d->entry->d_ino, + .st_mode = d->entry->d_type << 12, + }; + entsize = fuse_add_direntry(req, p, rem, name, + &st, nextoff); + } + if (entsize > rem) { + if (entry_ino != 0) + lo_forget_one(req, entry_ino, 1); + break; + } + + p += entsize; + rem -= entsize; + + d->entry = NULL; + d->offset = nextoff; + } + + err = 0; +error: + // If there's an error, we can only signal it if we haven't stored + // any entries yet - otherwise we'd end up with wrong lookup + // counts for the entries that are already in the buffer. So we + // return what we've collected until that point. + if (err && rem == size) + fuse_reply_err(req, err); + else + fuse_reply_buf(req, buf, size - rem); + free(buf); +} + +static void lo_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, + off_t offset, struct fuse_file_info *fi) +{ + lo_do_readdir(req, ino, size, offset, fi, 0); +} + +static void lo_readdirplus(fuse_req_t req, fuse_ino_t ino, size_t size, + off_t offset, struct fuse_file_info *fi) +{ + lo_do_readdir(req, ino, size, offset, fi, 1); +} + +static void lo_releasedir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) +{ + struct lo_dirp *d = lo_dirp(fi); + (void) ino; + closedir(d->dp); + free(d); + fuse_reply_err(req, 0); +} + +static void lo_create(fuse_req_t req, fuse_ino_t parent, const char *name, + mode_t mode, struct fuse_file_info *fi) +{ + int fd; + struct lo_data *lo = lo_data(req); + struct fuse_entry_param e; + int err; + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, "lo_create(parent=%" PRIu64 ", name=%s)\n", + parent, name); + + fd = openat(lo_fd(req, parent), name, + (fi->flags | O_CREAT) & ~O_NOFOLLOW, mode); + if (fd == -1) + return (void) fuse_reply_err(req, errno); + + fi->fh = fd; + if (lo->cache == CACHE_NEVER) + fi->direct_io = 1; + else if (lo->cache == CACHE_ALWAYS) + fi->keep_cache = 1; + + err = lo_do_lookup(req, parent, name, &e); + if (err) + fuse_reply_err(req, err); + else + fuse_reply_create(req, &e, fi); +} + +static void lo_fsyncdir(fuse_req_t req, fuse_ino_t ino, int datasync, + struct fuse_file_info *fi) +{ + int res; + int fd = dirfd(lo_dirp(fi)->dp); + (void) ino; + if (datasync) + res = fdatasync(fd); + else + res = fsync(fd); + fuse_reply_err(req, res == -1 ? errno : 0); +} + +static void lo_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) +{ + int fd; + char buf[64]; + struct lo_data *lo = lo_data(req); + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, "lo_open(ino=%" PRIu64 ", flags=%d)\n", + ino, fi->flags); + + /* With writeback cache, kernel may send read requests even + when userspace opened write-only */ + if (lo->writeback && (fi->flags & O_ACCMODE) == O_WRONLY) { + fi->flags &= ~O_ACCMODE; + fi->flags |= O_RDWR; + } + + /* With writeback cache, O_APPEND is handled by the kernel. + This breaks atomicity (since the file may change in the + underlying filesystem, so that the kernel's idea of the + end of the file isn't accurate anymore). In this example, + we just accept that. A more rigorous filesystem may want + to return an error here */ + if (lo->writeback && (fi->flags & O_APPEND)) + fi->flags &= ~O_APPEND; + + sprintf(buf, "/proc/self/fd/%i", lo_fd(req, ino)); + fd = open(buf, fi->flags & ~O_NOFOLLOW); + if (fd == -1) + return (void) fuse_reply_err(req, errno); + + fi->fh = fd; + if (lo->cache == CACHE_NEVER) + fi->direct_io = 1; + else if (lo->cache == CACHE_ALWAYS) + fi->keep_cache = 1; + fuse_reply_open(req, fi); +} + +static void lo_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) +{ + (void) ino; + + close(fi->fh); + fuse_reply_err(req, 0); +} + +static void lo_flush(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) +{ + int res; + (void) ino; + res = close(dup(fi->fh)); + fuse_reply_err(req, res == -1 ? errno : 0); +} + +static void lo_fsync(fuse_req_t req, fuse_ino_t ino, int datasync, + struct fuse_file_info *fi) +{ + int res; + (void) ino; + if (datasync) + res = fdatasync(fi->fh); + else + res = fsync(fi->fh); + fuse_reply_err(req, res == -1 ? errno : 0); +} + +static void lo_read(fuse_req_t req, fuse_ino_t ino, size_t size, + off_t offset, struct fuse_file_info *fi) +{ + struct fuse_bufvec buf = FUSE_BUFVEC_INIT(size); + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, "lo_read(ino=%" PRIu64 ", size=%zd, " + "off=%lu)\n", ino, size, (unsigned long) offset); + + buf.buf[0].flags = FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK; + buf.buf[0].fd = fi->fh; + buf.buf[0].pos = offset; + + fuse_reply_data(req, &buf, FUSE_BUF_SPLICE_MOVE); +} + +static void lo_write_buf(fuse_req_t req, fuse_ino_t ino, + struct fuse_bufvec *in_buf, off_t off, + struct fuse_file_info *fi) +{ + (void) ino; + ssize_t res; + struct fuse_bufvec out_buf = FUSE_BUFVEC_INIT(fuse_buf_size(in_buf)); + + out_buf.buf[0].flags = FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK; + out_buf.buf[0].fd = fi->fh; + out_buf.buf[0].pos = off; + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, "lo_write(ino=%" PRIu64 ", size=%zd, off=%lu)\n", + ino, out_buf.buf[0].size, (unsigned long) off); + + res = fuse_buf_copy(&out_buf, in_buf, 0); + if(res < 0) + fuse_reply_err(req, -res); + else + fuse_reply_write(req, (size_t) res); +} + +static void lo_statfs(fuse_req_t req, fuse_ino_t ino) +{ + int res; + struct statvfs stbuf; + + res = fstatvfs(lo_fd(req, ino), &stbuf); + if (res == -1) + fuse_reply_err(req, errno); + else + fuse_reply_statfs(req, &stbuf); +} + +/* CloudMachine: rclone's FUSE has no fallocate. macOS (DiskImages2) erases + * sparse-bundle bands with a hole punch over SMB, and a refusal there kills the + * whole image. Emulate hole punching / zero ranges by writing zeros, never past + * EOF (punch is always KEEP_SIZE); a plain allocation just extends the file. */ +static int punch_by_zeros(int fd, int mode, off_t offset, off_t length) +{ + struct stat st; + static char zeros[1 << 20]; + off_t end = offset + length, pos; + + if (fstat(fd, &st) == -1) + return errno; + if (mode & FALLOC_FL_KEEP_SIZE || mode & FALLOC_FL_PUNCH_HOLE) { + if (end > st.st_size) + end = st.st_size; + } + for (pos = offset; pos < end;) { + size_t n = (size_t)((end - pos) < (off_t)sizeof(zeros) ? (end - pos) : (off_t)sizeof(zeros)); + ssize_t w = pwrite(fd, zeros, n, pos); + if (w < 0) + return errno; + pos += w; + } + if (!(mode & (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) && offset + length > st.st_size) + if (ftruncate(fd, offset + length) == -1) + return errno; + return 0; +} + +static void lo_fallocate(fuse_req_t req, fuse_ino_t ino, int mode, + off_t offset, off_t length, struct fuse_file_info *fi) +{ + int err; + (void) ino; + + err = fallocate(fi->fh, mode, offset, length) < 0 ? errno : 0; + if (err == EOPNOTSUPP || err == ENOSYS) + err = punch_by_zeros(fi->fh, mode, offset, length); + fuse_reply_err(req, err); +} + +static void lo_flock(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi, + int op) +{ + int res; + (void) ino; + + res = flock(fi->fh, op); + + fuse_reply_err(req, res == -1 ? errno : 0); +} + +/* CloudMachine: rclone has no extended attributes, but Samba keeps the + * sparse flag (user.DOSATTRIB) and the Apple metadata streams in them; without + * them macOS treats the band files as non-sparse and refuses to erase a band. + * Attributes live in side files on local disk: XDIR//. rclone's + * inode is a hash of the path, so it is stable for a given band. */ +#define XDIR "/srv/tm-xattr" + +static void xdir_of(char *buf, size_t n, struct lo_inode *in) +{ + snprintf(buf, n, XDIR "/%llx", (unsigned long long)in->ino); +} + +static int xname_ok(const char *name) +{ + return name[0] && !strchr(name, '/') && strcmp(name, ".") && strcmp(name, ".."); +} + +static void lo_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name, + size_t size) +{ + char dir[256], path[1024]; + struct stat st; + char *value; + ssize_t r; + int fd; + + if (!xname_ok(name)) { fuse_reply_err(req, ENODATA); return; } + xdir_of(dir, sizeof(dir), lo_inode(req, ino)); + snprintf(path, sizeof(path), "%s/%s", dir, name); + fd = open(path, O_RDONLY); + if (fd == -1) { fuse_reply_err(req, errno == ENOENT ? ENODATA : errno); return; } + if (fstat(fd, &st) == -1) { int e = errno; close(fd); fuse_reply_err(req, e); return; } + if (size == 0) { close(fd); fuse_reply_xattr(req, st.st_size); return; } + if ((size_t)st.st_size > size) { close(fd); fuse_reply_err(req, ERANGE); return; } + value = malloc(st.st_size ? st.st_size : 1); + if (!value) { close(fd); fuse_reply_err(req, ENOMEM); return; } + r = pread(fd, value, st.st_size, 0); + close(fd); + if (r < 0) { int e = errno; free(value); fuse_reply_err(req, e); return; } + fuse_reply_buf(req, value, r); + free(value); +} + +static void lo_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size) +{ + char dir[256]; + DIR *d; + struct dirent *e; + char *buf = NULL; + size_t len = 0; + + xdir_of(dir, sizeof(dir), lo_inode(req, ino)); + d = opendir(dir); + if (d) { + while ((e = readdir(d))) { + size_t n; + char *nb; + if (!strcmp(e->d_name, ".") || !strcmp(e->d_name, "..") || e->d_name[0] == '#') + continue; + n = strlen(e->d_name) + 1; + nb = realloc(buf, len + n); + if (!nb) { closedir(d); free(buf); fuse_reply_err(req, ENOMEM); return; } + buf = nb; + memcpy(buf + len, e->d_name, n); + len += n; + } + closedir(d); + } + if (size == 0) fuse_reply_xattr(req, len); + else if (len > size) fuse_reply_err(req, ERANGE); + else fuse_reply_buf(req, buf ? buf : "", len); + free(buf); +} + +static void lo_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name, + const char *value, size_t size, int flags) +{ + char dir[256], path[1024], tmp[1100]; + int fd, exists; + + if (!xname_ok(name)) { fuse_reply_err(req, EINVAL); return; } + xdir_of(dir, sizeof(dir), lo_inode(req, ino)); + if (mkdir(XDIR, 0700) == -1 && errno != EEXIST) { fuse_reply_err(req, errno); return; } + if (mkdir(dir, 0700) == -1 && errno != EEXIST) { fuse_reply_err(req, errno); return; } + snprintf(path, sizeof(path), "%s/%s", dir, name); + exists = access(path, F_OK) == 0; + if ((flags & XATTR_CREATE) && exists) { fuse_reply_err(req, EEXIST); return; } + if ((flags & XATTR_REPLACE) && !exists) { fuse_reply_err(req, ENODATA); return; } + snprintf(tmp, sizeof(tmp), "%s/#%s.%d", dir, name, (int)getpid()); + fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd == -1) { fuse_reply_err(req, errno); return; } + if (size && write(fd, value, size) != (ssize_t)size) { int e = errno; close(fd); unlink(tmp); fuse_reply_err(req, e ? e : EIO); return; } + close(fd); + if (rename(tmp, path) == -1) { int e = errno; unlink(tmp); fuse_reply_err(req, e); return; } + fuse_reply_err(req, 0); +} + +static void lo_removexattr(fuse_req_t req, fuse_ino_t ino, const char *name) +{ + char dir[256], path[1024]; + + if (!xname_ok(name)) { fuse_reply_err(req, ENODATA); return; } + xdir_of(dir, sizeof(dir), lo_inode(req, ino)); + snprintf(path, sizeof(path), "%s/%s", dir, name); + if (unlink(path) == -1) { fuse_reply_err(req, errno == ENOENT ? ENODATA : errno); return; } + fuse_reply_err(req, 0); +} + +#ifdef HAVE_COPY_FILE_RANGE +static void lo_copy_file_range(fuse_req_t req, fuse_ino_t ino_in, off_t off_in, + struct fuse_file_info *fi_in, + fuse_ino_t ino_out, off_t off_out, + struct fuse_file_info *fi_out, size_t len, + int flags) +{ + ssize_t res; + + if (lo_debug(req)) + fuse_log(FUSE_LOG_DEBUG, "lo_copy_file_range(ino=%" PRIu64 "/fd=%lu, " + "off=%lu, ino=%" PRIu64 "/fd=%lu, " + "off=%lu, size=%zd, flags=0x%x)\n", + ino_in, fi_in->fh, off_in, ino_out, fi_out->fh, off_out, + len, flags); + + res = copy_file_range(fi_in->fh, &off_in, fi_out->fh, &off_out, len, + flags); + if (res < 0) + fuse_reply_err(req, errno); + else + fuse_reply_write(req, res); +} +#endif + +static void lo_lseek(fuse_req_t req, fuse_ino_t ino, off_t off, int whence, + struct fuse_file_info *fi) +{ + off_t res; + + (void)ino; + res = lseek(fi->fh, off, whence); + if (res != -1) + fuse_reply_lseek(req, res); + else + fuse_reply_err(req, errno); +} + +static const struct fuse_lowlevel_ops lo_oper = { + .init = lo_init, + .destroy = lo_destroy, + .lookup = lo_lookup, + .mkdir = lo_mkdir, + .mknod = lo_mknod, + .symlink = lo_symlink, + .link = lo_link, + .unlink = lo_unlink, + .rmdir = lo_rmdir, + .rename = lo_rename, + .forget = lo_forget, + .forget_multi = lo_forget_multi, + .getattr = lo_getattr, + .setattr = lo_setattr, + .readlink = lo_readlink, + .opendir = lo_opendir, + .readdir = lo_readdir, + .readdirplus = lo_readdirplus, + .releasedir = lo_releasedir, + .fsyncdir = lo_fsyncdir, + .create = lo_create, + .open = lo_open, + .release = lo_release, + .flush = lo_flush, + .fsync = lo_fsync, + .read = lo_read, + .write_buf = lo_write_buf, + .statfs = lo_statfs, + .fallocate = lo_fallocate, + .flock = lo_flock, + .getxattr = lo_getxattr, + .listxattr = lo_listxattr, + .setxattr = lo_setxattr, + .removexattr = lo_removexattr, +#ifdef HAVE_COPY_FILE_RANGE + .copy_file_range = lo_copy_file_range, +#endif + .lseek = lo_lseek, +}; + +int main(int argc, char *argv[]) +{ + struct fuse_args args = FUSE_ARGS_INIT(argc, argv); + struct fuse_session *se; + struct fuse_cmdline_opts opts; + struct fuse_loop_config config; + struct lo_data lo = { .debug = 0, + .writeback = 0 }; + int ret = -1; + + /* Don't mask creation mode, kernel already did that */ + umask(0); + + pthread_mutex_init(&lo.mutex, NULL); + lo.root.next = lo.root.prev = &lo.root; + lo.root.fd = -1; + lo.cache = CACHE_NORMAL; + + if (fuse_parse_cmdline(&args, &opts) != 0) + return 1; + if (opts.show_help) { + printf("usage: %s [options] \n\n", argv[0]); + fuse_cmdline_help(); + fuse_lowlevel_help(); + passthrough_ll_help(); + ret = 0; + goto err_out1; + } else if (opts.show_version) { + printf("FUSE library version %s\n", fuse_pkgversion()); + fuse_lowlevel_version(); + ret = 0; + goto err_out1; + } + + if(opts.mountpoint == NULL) { + printf("usage: %s [options] \n", argv[0]); + printf(" %s --help\n", argv[0]); + ret = 1; + goto err_out1; + } + + if (fuse_opt_parse(&args, &lo, lo_opts, NULL)== -1) + return 1; + + lo.debug = opts.debug; + lo.root.refcount = 2; + if (lo.source) { + struct stat stat; + int res; + + res = lstat(lo.source, &stat); + if (res == -1) { + fuse_log(FUSE_LOG_ERR, "failed to stat source (\"%s\"): %m\n", + lo.source); + exit(1); + } + if (!S_ISDIR(stat.st_mode)) { + fuse_log(FUSE_LOG_ERR, "source is not a directory\n"); + exit(1); + } + + } else { + lo.source = "/"; + } + if (!lo.timeout_set) { + switch (lo.cache) { + case CACHE_NEVER: + lo.timeout = 0.0; + break; + + case CACHE_NORMAL: + lo.timeout = 1.0; + break; + + case CACHE_ALWAYS: + lo.timeout = 86400.0; + break; + } + } else if (lo.timeout < 0) { + fuse_log(FUSE_LOG_ERR, "timeout is negative (%lf)\n", + lo.timeout); + exit(1); + } + + lo.root.fd = open(lo.source, O_PATH); + if (lo.root.fd == -1) { + fuse_log(FUSE_LOG_ERR, "open(\"%s\", O_PATH): %m\n", + lo.source); + exit(1); + } + + se = fuse_session_new(&args, &lo_oper, sizeof(lo_oper), &lo); + if (se == NULL) + goto err_out1; + + if (fuse_set_signal_handlers(se) != 0) + goto err_out2; + + if (fuse_session_mount(se, opts.mountpoint) != 0) + goto err_out3; + + fuse_daemonize(opts.foreground); + + /* Block until ctrl+c or fusermount -u */ + if (opts.singlethread) + ret = fuse_session_loop(se); + else { + config.clone_fd = opts.clone_fd; + config.max_idle_threads = opts.max_idle_threads; + ret = fuse_session_loop_mt(se, &config); + } + + fuse_session_unmount(se); +err_out3: + fuse_remove_signal_handlers(se); +err_out2: + fuse_session_destroy(se); +err_out1: + free(opts.mountpoint); + fuse_opt_free_args(&args); + + if (lo.root.fd >= 0) + close(lo.root.fd); + + return ret ? 1 : 0; +} diff --git a/vm/punchfs/test-punchfs.sh b/vm/punchfs/test-punchfs.sh new file mode 100755 index 0000000..3ff1206 --- /dev/null +++ b/vm/punchfs/test-punchfs.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# CloudMachine: checks punchfs inside the machine (run as root): +# limactl shell cmsmb sudo /opt/cloudmachine/punchfs/test-punchfs.sh +# +# It mounts a throwaway rclone mount of a local directory (a FUSE filesystem +# with no fallocate and no extended attributes - the same gaps as the Google +# Drive mount), puts punchfs on top, and checks the two things macOS needs: +# 1. hole punch / zero range succeed, read back as zeros, keep the size; +# 2. extended attributes can be set, read, listed and removed. +# Nothing touches /srv/tm or Google Drive. Attributes of the test file land in +# /srv/tm-xattr/ and are removed at the end. +set -euo pipefail + +PUNCHFS=${PUNCHFS:-/usr/local/bin/punchfs} +work=$(mktemp -d /var/tmp/punchfs-test.XXXXXX) +mkdir -p "$work/src" "$work/rc" "$work/punch" "$work/cache" +failures=0 + +cleanup() { + fusermount3 -uz "$work/punch" 2>/dev/null || true + fusermount3 -uz "$work/rc" 2>/dev/null || true + sleep 1 + rm -rf "$work" +} +trap cleanup EXIT + +check() { # check + local what=$1; shift + if "$@"; then echo "ok - $what"; else echo "FAIL - $what"; failures=$((failures + 1)); fi +} + +wait_mounted() { + for _ in $(seq 1 50); do mountpoint -q "$1" && return 0; sleep 0.2; done + echo "not mounted: $1" >&2; return 1 +} + +rclone mount "$work/src" "$work/rc" --config /dev/null --vfs-cache-mode full \ + --cache-dir "$work/cache" --log-level ERROR & +wait_mounted "$work/rc" +"$PUNCHFS" -f -o source="$work/rc" "$work/punch" & +wait_mounted "$work/punch" + +f="$work/punch/band" +MiB=1048576 +head -c $((4 * MiB)) /dev/zero | tr '\0' 'A' > "$f" + +# The layer below really refuses fallocate - otherwise this test proves nothing. +check "rclone below refuses hole punch (the gap punchfs covers)" \ + bash -c "! fallocate --punch-hole --offset 0 --length $MiB '$work/rc/band' 2>/dev/null" + +check "hole punch succeeds" fallocate --punch-hole --offset $MiB --length $MiB "$f" +check "size is kept after hole punch" test "$(stat -c %s "$f")" -eq $((4 * MiB)) +check "punched range reads as zeros" \ + cmp -s <(dd if="$f" bs=$MiB skip=1 count=1 status=none) <(head -c $MiB /dev/zero) +check "data before the hole is intact" \ + cmp -s <(dd if="$f" bs=$MiB count=1 status=none) <(head -c $MiB /dev/zero | tr '\0' 'A') +check "zero range with keep-size succeeds past EOF" \ + fallocate --zero-range --keep-size --offset $((3 * MiB)) --length $((2 * MiB)) "$f" +check "zero range did not grow the file" test "$(stat -c %s "$f")" -eq $((4 * MiB)) +check "zeroed range reads as zeros" \ + cmp -s <(dd if="$f" bs=$MiB skip=3 count=1 status=none) <(head -c $MiB /dev/zero) + +check "set text attribute" setfattr -n user.cmtest -v hello "$f" +check "read text attribute" test "$(getfattr --only-values -n user.cmtest "$f")" = hello +check "set binary attribute" setfattr -n user.cmtest.bin -v 0x00010203 "$f" +check "read binary attribute" \ + bash -c "getfattr -e hex -n user.cmtest.bin '$f' | grep -q '=0x00010203'" +check "list attributes" bash -c "getfattr -d -m 'user\\.cmtest' '$f' | grep -q user.cmtest.bin" +check "XATTR_CREATE refuses an existing name" \ + bash -c "! python3 -c 'import os,sys; os.setxattr(sys.argv[1], \"user.cmtest\", b\"x\", os.XATTR_CREATE)' '$f' 2>/dev/null" +check "remove attributes" bash -c "setfattr -x user.cmtest '$f' && setfattr -x user.cmtest.bin '$f'" +check "removed attribute is gone" bash -c "! getfattr -n user.cmtest '$f' 2>/dev/null" + +# The side directory is keyed by the inode punchfs sees, i.e. rclone's. +ino=$(printf '%x' "$(stat -c %i "$work/rc/band")") +rmdir --ignore-fail-on-non-empty "/srv/tm-xattr/$ino" 2>/dev/null || true + +if [ "$failures" -eq 0 ]; then echo "punchfs: all checks passed"; else echo "punchfs: $failures check(s) FAILED"; fi +exit "$failures" diff --git a/vm/samba/smb.conf b/vm/samba/smb.conf new file mode 100644 index 0000000..72cadb6 --- /dev/null +++ b/vm/samba/smb.conf @@ -0,0 +1,27 @@ +# CloudMachine: Samba share for Time Machine (managed by cloudmachine-agent vm-setup). +[global] + max open files = 500000 + kernel share modes = no + kernel oplocks = no + posix locking = no + strict locking = no + durable handles = yes + smb2 leases = yes + server role = standalone server + server min protocol = SMB2_10 + map to guest = never + vfs objects = catia fruit streams_xattr + fruit:metadata = stream + fruit:model = MacSamba + fruit:nfs_aces = no + fruit:posix_rename = yes + fruit:veto_appledouble = no + fruit:wipe_intentionally_left_blank_rfork = yes + fruit:delete_empty_adfiles = yes + log level = 1 +[CloudMachine] + path = /srv/tm-local + valid users = tm + read only = no + fruit:time machine = yes + fruit:time machine max size = 3500G diff --git a/vm/systemd/punchfs.service b/vm/systemd/punchfs.service new file mode 100644 index 0000000..afb168e --- /dev/null +++ b/vm/systemd/punchfs.service @@ -0,0 +1,21 @@ +# CloudMachine: /srv/tm (rclone) -> /srv/tm-punch. Emulates hole punching by +# writing zeros and keeps extended attributes in /srv/tm-xattr (see punchfs/README.md). +[Unit] +Description=punchfs: /srv/tm (rclone) -> /srv/tm-punch, hole punch emulated with zeros +Requires=rclone-tm.service +After=rclone-tm.service +Before=tm-bands-bind.service smbd.service +ConditionPathExists=/etc/rclone/rclone.conf + +[Service] +Type=simple +# passthrough_ll keeps one descriptor open for every inode the kernel has seen; +# a full backup has tens of thousands of bands, and the default 1024 ran out (EMFILE). +LimitNOFILE=524288 +ExecStart=/usr/local/bin/punchfs -f -o source=/srv/tm,allow_other /srv/tm-punch +ExecStop=/bin/fusermount3 -uz /srv/tm-punch +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/vm/systemd/rclone-tm.service b/vm/systemd/rclone-tm.service new file mode 100644 index 0000000..cab08af --- /dev/null +++ b/vm/systemd/rclone-tm.service @@ -0,0 +1,27 @@ +# CloudMachine: the Google Drive folder that holds the Time Machine bands. +# /etc/cloudmachine/vm.env (CM_REMOTE) is written by `cloudmachine-agent vm-setup`; +# /etc/cloudmachine/tm-ids.env (TM_UID, TM_GID) by the provisioning script. +[Unit] +Description=rclone: Google Drive -> /srv/tm +After=network-online.target +Wants=network-online.target +Before=smbd.service +ConditionPathExists=/etc/rclone/rclone.conf +ConditionPathExists=/etc/cloudmachine/vm.env + +[Service] +Type=notify +EnvironmentFile=/etc/cloudmachine/vm.env +EnvironmentFile=/etc/cloudmachine/tm-ids.env +ExecStart=/usr/local/bin/rclone mount ${CM_REMOTE} /srv/tm \ + --config /etc/rclone/rclone.conf --allow-other --uid ${TM_UID} --gid ${TM_GID} --umask 002 \ + --vfs-cache-mode full --vfs-cache-max-size 120G --vfs-cache-max-age 9999h --vfs-write-back 600s \ + --cache-dir /var/cache/rclone --dir-cache-time 9999h --poll-interval 0 --attr-timeout 5m \ + --transfers 8 --drive-chunk-size 32M --drive-use-trash=false --drive-stop-on-upload-limit \ + --rc --rc-addr 127.0.0.1:5572 --rc-no-auth --log-file /var/log/rclone-tm.log --log-level INFO +ExecStop=/bin/fusermount3 -uz /srv/tm +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/vm/systemd/smbd.service.d/cloudmachine.conf b/vm/systemd/smbd.service.d/cloudmachine.conf new file mode 100644 index 0000000..9a7b7c9 --- /dev/null +++ b/vm/systemd/smbd.service.d/cloudmachine.conf @@ -0,0 +1,6 @@ +# CloudMachine: Samba serves the share only on top of the Drive layers, and +# never before the rclone configuration has been copied in. +[Unit] +Requires=rclone-tm.service punchfs.service tm-bands-bind.service +After=rclone-tm.service punchfs.service tm-bands-bind.service +ConditionPathExists=/etc/rclone/rclone.conf diff --git a/vm/systemd/smbd.service.d/limits.conf b/vm/systemd/smbd.service.d/limits.conf new file mode 100644 index 0000000..19085f5 --- /dev/null +++ b/vm/systemd/smbd.service.d/limits.conf @@ -0,0 +1,3 @@ +# CloudMachine: one open file per band touched; the default 1024 is not enough. +[Service] +LimitNOFILE=524288 diff --git a/vm/systemd/tm-bands-bind.service b/vm/systemd/tm-bands-bind.service new file mode 100644 index 0000000..7383fa8 --- /dev/null +++ b/vm/systemd/tm-bands-bind.service @@ -0,0 +1,20 @@ +# CloudMachine: the image's metadata stays on local ext4, its bands live on +# Google Drive. /etc/cloudmachine/bundle.env (CM_BUNDLE=.sparsebundle) is +# written by `cloudmachine-agent vm-prepare-bundle`; until then this unit is +# skipped, so Time Machine can create its first, empty image on local disk. +[Unit] +Description=Bind: Time Machine bands on Google Drive +Requires=punchfs.service +After=punchfs.service +Before=smbd.service +ConditionPathExists=/etc/cloudmachine/bundle.env + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=/etc/cloudmachine/bundle.env +ExecStart=/bin/mount --bind /srv/tm-punch/bands "/srv/tm-local/${CM_BUNDLE}/bands" +ExecStop=/bin/umount -l "/srv/tm-local/${CM_BUNDLE}/bands" + +[Install] +WantedBy=multi-user.target From bd9e758efb33004231f46093ebba003156799b9a Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 12:16:36 +0100 Subject: [PATCH 2/6] fix(vm): ship passthrough_helpers.h; let the vm commands target another machine Tried on a throwaway machine created from the template (cmtest), next to the live one: - provisioning stopped at the punchfs build: punchfs.c includes libfuse's passthrough_helpers.h (BSD-2), which was neither in the repo nor in the template. Both now; the machine provisions and test-punchfs.sh passes. - CLOUDMACHINE_VM_INSTANCE and CLOUDMACHINE_VM_REMOTE point the commands at another machine and Drive folder; defaults unchanged (cmsmb, -tm). - vm-prepare-bundle refused whenever Time Machine was backing up anywhere; now only when a destination is this machine. vm-setup, vm-status and vm-prepare-bundle ran end to end on cmtest: an 875 MB-band image became 32 MB bands on Drive, fsck_apfs -n passed, the bind and smbd came up. Co-Authored-By: Claude Opus 5.5 --- .../CloudMachineCore/VMBundlePrep.swift | 7 +- .../Sources/CloudMachineCore/VMLayout.swift | 18 ++++- .../Sources/CloudMachineCore/VMStatus.swift | 17 +++++ .../CloudMachineAppTests/VMLayoutTests.swift | 8 ++ vm/README.md | 14 ++++ vm/cmsmb.yaml | 3 + vm/punchfs/passthrough_helpers.h | 76 +++++++++++++++++++ 7 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 vm/punchfs/passthrough_helpers.h diff --git a/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift b/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift index 8a81674..6757b3a 100644 --- a/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift +++ b/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift @@ -44,7 +44,12 @@ public enum VMBundlePrep { return fail( L10n.tr("tmutil did not answer - not touching the image while its state is unknown.")) case .some(true): - return fail(L10n.tr("Time Machine is backing up - stop it first: tmutil stopbackup")) + // A backup to another destination does not write this image (a second + // machine, or a throwaway one next to `cmsmb`); one that may is stopped. + guard await VMStatus.timeMachineTargetsMachine() == false else { + return fail(L10n.tr("Time Machine is backing up - stop it first: tmutil stopbackup")) + } + step("Time Machine is backing up, but not to \(VMLayout.instanceName) - going on") case .some(false): break } diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift index 0f5c2d1..2e1560e 100644 --- a/mac-app/Sources/CloudMachineCore/VMLayout.swift +++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift @@ -11,8 +11,18 @@ import Foundation /// This is the new path, next to the attached-image one; neither touches the /// other. public enum VMLayout { - /// Lima instance name. Fixed: the launchd agent and the guard look it up. - public static let instanceName = "cmsmb" + /// Lima instance name: `cmsmb`, which the launchd agents look up. + /// `CLOUDMACHINE_VM_INSTANCE` points the commands at another machine - + /// that is how a change is tried on a throwaway VM while `cmsmb` keeps + /// backing up. + public static var instanceName: String { + override("CLOUDMACHINE_VM_INSTANCE") ?? "cmsmb" + } + + static func override(_ name: String) -> String? { + guard let value = ProcessInfo.processInfo.environment[name], !value.isEmpty else { return nil } + return value + } /// SMB share name, as in `vm/samba/smb.conf`. public static let shareName = "CloudMachine" /// SMB user created by the provisioning script. @@ -45,8 +55,10 @@ public enum VMLayout { /// `gdrive:CloudMachine/-tm`: a folder of its own, next to the /// attached image's folder, so the two layouts never share bands. + /// `CLOUDMACHINE_VM_REMOTE` replaces it whole (see `instanceName`). public static var remote: String { - "\(DriveBufferService.remoteName):\(DriveBufferService.remoteRoot)/\(DriveFolder.name)-tm" + override("CLOUDMACHINE_VM_REMOTE") + ?? "\(DriveBufferService.remoteName):\(DriveBufferService.remoteRoot)/\(DriveFolder.name)-tm" } /// `vm/` among the resources (the repo root in a checkout, Resources in the app). diff --git a/mac-app/Sources/CloudMachineCore/VMStatus.swift b/mac-app/Sources/CloudMachineCore/VMStatus.swift index 108a960..5beb08b 100644 --- a/mac-app/Sources/CloudMachineCore/VMStatus.swift +++ b/mac-app/Sources/CloudMachineCore/VMStatus.swift @@ -36,6 +36,23 @@ public enum VMStatus { return report } + /// Whether Time Machine's destinations include this machine; `nil` = unknown + /// (no address, or `tmutil` did not answer). + public static func timeMachineTargetsMachine() async -> Bool? { + guard let address = await VMLayout.currentAddress(), + let info = try? await ProcessRunner.run( + "/usr/bin/tmutil", ["destinationinfo"], timeout: TimeMachineStatus.commandTimeout), + info.succeeded || info.stdout.contains("No destinations") + else { return nil } + return targets(urls: destinationURLs(destinationInfo: info.stdout), address: address) + } + + /// `smb://tm@
/...` among the destinations. The `@` and `/` keep + /// 192.168.64.1 from matching 192.168.64.10. + static func targets(urls: [String], address: String) -> Bool { + urls.contains { $0.contains("@\(address)/") } + } + /// `URL : smb://...` lines of `tmutil destinationinfo`. static func destinationURLs(destinationInfo: String) -> [String] { destinationInfo.split(separator: "\n").compactMap { raw in diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift index f0409b4..b3d23a1 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -192,4 +192,12 @@ final class VMLayoutTests: XCTestCase { "com.renacode.cloudmachine.vm-start.plist.template", ]) } + + func testTimeMachineTargetIsMatchedByWholeAddress() { + let urls = ["smb://tm@192.168.64.10/CloudMachine"] + XCTAssertTrue(VMStatus.targets(urls: urls, address: "192.168.64.10")) + XCTAssertFalse(VMStatus.targets(urls: urls, address: "192.168.64.1")) + XCTAssertFalse(VMStatus.targets(urls: urls, address: "192.168.64.11")) + XCTAssertFalse(VMStatus.targets(urls: [], address: "192.168.64.10")) + } } diff --git a/vm/README.md b/vm/README.md index 42f7e5f..6da5568 100644 --- a/vm/README.md +++ b/vm/README.md @@ -41,3 +41,17 @@ Each layer is there for a reason: cloudmachine-agent vm-prepare-bundle # 32 MB bands, bands moved to Drive cloudmachine-agent vm-status cloudmachine-agent vm-guard # one check; the launchd agent runs it every minute + +## Trying a change without touching the live machine + +Every `vm-*` command takes the machine and the Drive folder from two +variables, so a change can be tried on a throwaway machine while `cmsmb` +keeps backing up: + +```sh +export CLOUDMACHINE_VM_INSTANCE=cmtest CLOUDMACHINE_VM_REMOTE=gdrive:CloudMachine/test-vm-tm +cloudmachine-agent vm-setup && cloudmachine-agent vm-status +``` + +`vm-prepare-bundle` refuses while Time Machine is backing up to *this* +machine; a backup going to another destination does not stop it. diff --git a/vm/cmsmb.yaml b/vm/cmsmb.yaml index 7e643f0..eafab0b 100644 --- a/vm/cmsmb.yaml +++ b/vm/cmsmb.yaml @@ -49,6 +49,9 @@ provision: - mode: data path: /opt/cloudmachine/punchfs/punchfs.c file: punchfs/punchfs.c +- mode: data + path: /opt/cloudmachine/punchfs/passthrough_helpers.h + file: punchfs/passthrough_helpers.h - mode: data path: /opt/cloudmachine/punchfs/test-punchfs.sh file: punchfs/test-punchfs.sh diff --git a/vm/punchfs/passthrough_helpers.h b/vm/punchfs/passthrough_helpers.h new file mode 100644 index 0000000..6b77c33 --- /dev/null +++ b/vm/punchfs/passthrough_helpers.h @@ -0,0 +1,76 @@ +/* + * FUSE: Filesystem in Userspace + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE + */ + +/* + * Creates files on the underlying file system in response to a FUSE_MKNOD + * operation + */ +static int mknod_wrapper(int dirfd, const char *path, const char *link, + int mode, dev_t rdev) +{ + int res; + + if (S_ISREG(mode)) { + res = openat(dirfd, path, O_CREAT | O_EXCL | O_WRONLY, mode); + if (res >= 0) + res = close(res); + } else if (S_ISDIR(mode)) { + res = mkdirat(dirfd, path, mode); + } else if (S_ISLNK(mode) && link != NULL) { + res = symlinkat(link, dirfd, path); + } else if (S_ISFIFO(mode)) { + res = mkfifoat(dirfd, path, mode); +#ifdef __FreeBSD__ + } else if (S_ISSOCK(mode)) { + struct sockaddr_un su; + int fd; + + if (strlen(path) >= sizeof(su.sun_path)) { + errno = ENAMETOOLONG; + return -1; + } + fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd >= 0) { + /* + * We must bind the socket to the underlying file + * system to create the socket file, even though + * we'll never listen on this socket. + */ + su.sun_family = AF_UNIX; + strncpy(su.sun_path, path, sizeof(su.sun_path)); + res = bindat(dirfd, fd, (struct sockaddr*)&su, + sizeof(su)); + if (res == 0) + close(fd); + } else { + res = -1; + } +#endif + } else { + res = mknodat(dirfd, path, mode, rdev); + } + + return res; +} From 0434abd2fcff0e1313b63d923e703d77a9c9940f Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 12:40:49 +0100 Subject: [PATCH 3/6] fix(vm-guard): decide on the unsent backlog (vfs/queue), not the cache size 27.10.2026 12:24 the guard paused the first full backup at 116 GiB and never resumed it: it read diskCache.bytesUsed, the whole cache on disk, which keeps uploaded bands for reading until --vfs-cache-max-size and does not fall when the upload catches up. At 12:37 the real backlog was 30 GiB (966 bands in vfs/queue) with the cache still at 116. The reading now also carries the sum of vfs/queue sizes; pause and resume use it, and a missing queue answer counts as unknown (do nothing), never as zero. vm-status shows both numbers. Regression test with the 27.10 values. Co-Authored-By: Claude Opus 5.5 --- .../CloudMachineCore/L10nPolish+VM.swift | 4 +- .../Sources/CloudMachineCore/VMGuard.swift | 15 ++++--- .../Sources/CloudMachineCore/VMLayout.swift | 41 +++++++++++++++++-- .../Sources/CloudMachineCore/VMStatus.swift | 7 ++-- .../VMGuardDecisionTests.swift | 35 +++++++++++++++- .../CloudMachineAppTests/VMLayoutTests.swift | 12 ++++++ 6 files changed, 97 insertions(+), 17 deletions(-) diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift index 5dab27b..0e65770 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift @@ -154,8 +154,8 @@ extension L10nPolish { "nieznany", "Unpacking the image failed: %@": "Rozpakowanie obrazu nie powiodło się: %@", - "Upload buffer: %@ GiB, queued %@, uploading %@": - "Bufor wysyłki: %@ GiB, w kolejce %@, wysyłane %@", + "Still to upload: %@ GiB in %@ bands, uploading %@ (cache on disk %@ GiB)": + "Do wysłania: %@ GiB w %@ pasmach, wysyłane %@ (cache na dysku %@ GiB)", "Upload buffer: no answer from rclone (rc %@)": "Bufor wysyłki: brak odpowiedzi rclone (rc %@)", "When Time Machine has created its empty image, stop the backup (tmutil stopbackup) and run: cloudmachine-agent vm-prepare-bundle": diff --git a/mac-app/Sources/CloudMachineCore/VMGuard.swift b/mac-app/Sources/CloudMachineCore/VMGuard.swift index 593db16..0f90413 100644 --- a/mac-app/Sources/CloudMachineCore/VMGuard.swift +++ b/mac-app/Sources/CloudMachineCore/VMGuard.swift @@ -4,7 +4,8 @@ import Foundation /// than rclone uploads to Google Drive (750 GB a day at most), and the buffer /// is the machine's 200 GB disk. One `step()` per launchd run (every 60 s). /// -/// Pause when the buffer holds more than `highGiB` or the disk has less than +/// Pause when more than `highGiB` is still to upload (`unsentBytes` - not the +/// cache size, see `VMLayout.BufferStats`) or the disk has less than /// `minFreeGiB` free. Resume below `lowGiB` - and ONLY if this guard paused /// the backup (the marker file); a backup the person stopped stays stopped. /// No answer from the machine or from tmutil means: do nothing. @@ -42,8 +43,10 @@ public struct VMGuard: Sendable { reading: VMLayout.Reading?, backupRunning: Bool?, pausedByGuard: Bool, thresholds: Thresholds ) -> Action { - guard let reading else { return .nothing } - let usedGiB = reading.buffer.bytesUsed / gib + // No queue = unknown backlog: the cache size would stand in for it, and + // it is exactly the number that kept the backup paused on 27.10. + guard let reading, let unsent = reading.buffer.unsentBytes else { return .nothing } + let usedGiB = unsent / gib let freeGiB = reading.freeBytes / gib if usedGiB > Int64(thresholds.highGiB) || freeGiB < Int64(thresholds.minFreeGiB) { return backupRunning == true ? .pause : .nothing @@ -109,14 +112,14 @@ public struct VMGuard: Sendable { let paused = probes.markerExists() let action = Self.decide( reading: reading, backupRunning: running, pausedByGuard: paused, thresholds: thresholds) - let used = reading.buffer.bytesUsed / Self.gib + let used = (reading.buffer.unsentBytes ?? -1) / Self.gib let free = reading.freeBytes / Self.gib let queued = reading.buffer.uploadsQueued switch action { case .pause: if await probes.tmutil("stopbackup") { probes.setMarker(true) - probes.log("VM guard: PAUSE - buffer \(used) GiB, queue \(queued), free \(free) GiB") + probes.log("VM guard: PAUSE - unsent \(used) GiB, queue \(queued), free \(free) GiB") } else { probes.log("VM guard: tmutil stopbackup FAILED - buffer \(used) GiB, free \(free) GiB") } @@ -126,7 +129,7 @@ public struct VMGuard: Sendable { // for good with nobody left to resume it. if await probes.tmutil("startbackup") { probes.setMarker(false) - probes.log("VM guard: RESUME - buffer \(used) GiB, free \(free) GiB") + probes.log("VM guard: RESUME - unsent \(used) GiB, free \(free) GiB") } else { probes.log("VM guard: tmutil startbackup FAILED - will try again on the next run") } diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift index 2e1560e..809405c 100644 --- a/mac-app/Sources/CloudMachineCore/VMLayout.swift +++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift @@ -165,11 +165,38 @@ public enum VMLayout { return nil } - /// What rclone's `vfs/stats` says about the upload buffer. + /// What rclone says about the upload buffer. + /// + /// `bytesUsed` is the whole cache on disk, uploaded bands included: rclone + /// keeps them for reading until the cache reaches `--vfs-cache-max-size`. + /// It does NOT fall when the upload catches up - on 27.10.2026 it stayed at + /// 116 GiB for 13 minutes while the real backlog was 30 GiB, and a guard + /// deciding on it never resumed the backup. Decisions use `unsentBytes`, + /// the sum of `vfs/queue`. public struct BufferStats: Equatable, Sendable { public var bytesUsed: Int64 public var uploadsQueued: Int public var uploadsInProgress: Int + /// Bytes still to upload; `nil` = `vfs/queue` did not answer. + public var unsentBytes: Int64? + + public init( + bytesUsed: Int64, uploadsQueued: Int, uploadsInProgress: Int, unsentBytes: Int64? = nil + ) { + self.bytesUsed = bytesUsed + self.uploadsQueued = uploadsQueued + self.uploadsInProgress = uploadsInProgress + self.unsentBytes = unsentBytes + } + } + + /// Sum of the item sizes in `vfs/queue`; `nil` when it is not that JSON. + static func unsentBytes(vfsQueueJSON: String) -> Int64? { + guard let data = vfsQueueJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let queue = object["queue"] as? [[String: Any]] + else { return nil } + return queue.reduce(Int64(0)) { $0 + ((($1["size"] as? NSNumber)?.int64Value) ?? 0) } } static func bufferStats(vfsStatsJSON: String) -> BufferStats? { @@ -191,23 +218,29 @@ public enum VMLayout { public var freeBytes: Int64 } - /// The script prints `vfs/stats` JSON on one line, then `CMFREE `. + /// The script prints `vfs/stats` JSON on one line, `CMQUEUE ` + /// on the next, then `CMFREE `. static let readingScript = "curl -s -m 20 -X POST http://\(Guest.rcAddress)/vfs/stats | tr -d '\\n'; echo; " + + "printf 'CMQUEUE '; curl -s -m 20 -X POST http://\(Guest.rcAddress)/vfs/queue | tr -d '\\n'; echo; " + "echo CMFREE $(df -B1 --output=avail \(Guest.cacheDir) | tail -1 | tr -dc 0-9)" /// `nil` unless BOTH parts are there: half an answer is no answer. static func reading(scriptOutput: String) -> Reading? { var stats: BufferStats? var free: Int64? + var unsent: Int64? for line in scriptOutput.split(separator: "\n") { - if line.hasPrefix("CMFREE ") { + if line.hasPrefix("CMQUEUE ") { + unsent = unsentBytes(vfsQueueJSON: String(line.dropFirst("CMQUEUE ".count))) + } else if line.hasPrefix("CMFREE ") { free = Int64(line.dropFirst("CMFREE ".count).trimmingCharacters(in: .whitespaces)) } else if line.hasPrefix("{") { stats = bufferStats(vfsStatsJSON: String(line)) } } - guard let stats, let free else { return nil } + guard var stats, let free else { return nil } + stats.unsentBytes = unsent return Reading(buffer: stats, freeBytes: free) } diff --git a/mac-app/Sources/CloudMachineCore/VMStatus.swift b/mac-app/Sources/CloudMachineCore/VMStatus.swift index 5beb08b..7a7bed8 100644 --- a/mac-app/Sources/CloudMachineCore/VMStatus.swift +++ b/mac-app/Sources/CloudMachineCore/VMStatus.swift @@ -100,9 +100,10 @@ public enum VMStatus { if let reading = report.reading { out.append( L10n.tr( - "Upload buffer: %@ GiB, queued %@, uploading %@", - "\(reading.buffer.bytesUsed / VMGuard.gib)", "\(reading.buffer.uploadsQueued)", - "\(reading.buffer.uploadsInProgress)")) + "Still to upload: %@ GiB in %@ bands, uploading %@ (cache on disk %@ GiB)", + reading.buffer.unsentBytes.map { "\($0 / VMGuard.gib)" } ?? "?", + "\(reading.buffer.uploadsQueued)", "\(reading.buffer.uploadsInProgress)", + "\(reading.buffer.bytesUsed / VMGuard.gib)")) out.append(L10n.tr("Free in machine: %@ GiB", "\(reading.freeBytes / VMGuard.gib)")) } else { out.append( diff --git a/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift b/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift index 33e8adc..4fec2d7 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift @@ -13,12 +13,43 @@ final class VMGuardDecisionTests: XCTestCase { private let thresholds = VMGuard.Thresholds() private let gib: Int64 = 1 << 30 - private func reading(usedGiB: Int64, freeGiB: Int64 = 150, queued: Int = 3) -> VMLayout.Reading { + /// `usedGiB` is what is still to upload; the cache on disk is bigger on + /// purpose (116 GiB, as on 27.10) so no rule can lean on it by accident. + private func reading( + usedGiB: Int64, freeGiB: Int64 = 150, queued: Int = 3, cacheGiB: Int64 = 116 + ) -> VMLayout.Reading { VMLayout.Reading( - buffer: .init(bytesUsed: usedGiB * gib, uploadsQueued: queued, uploadsInProgress: 1), + buffer: .init( + bytesUsed: cacheGiB * gib, uploadsQueued: queued, uploadsInProgress: 1, + unsentBytes: usedGiB * gib), freeBytes: freeGiB * gib) } + /// 27.10.2026 12:24-12:37: paused at 116 GiB of cache, then the backlog fell + /// to 30 GiB while the cache stayed at 116 - the guard deciding on the + /// cache never resumed. The backlog is what counts. + func testResumesOnTheBacklogWhileTheCacheStaysFull() { + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 30, cacheGiB: 116), backupRunning: false, pausedByGuard: true, + thresholds: thresholds), .resume) + XCTAssertEqual( + VMGuard.decide( + reading: reading(usedGiB: 30, cacheGiB: 116), backupRunning: true, pausedByGuard: false, + thresholds: thresholds), .nothing, "a full cache alone must not pause") + } + + func testUnknownBacklogDoesNothing() { + var r = reading(usedGiB: 0) + r.buffer.unsentBytes = nil + for paused in [false, true] { + XCTAssertEqual( + VMGuard.decide( + reading: r, backupRunning: true, pausedByGuard: paused, thresholds: thresholds), + .nothing) + } + } + // MARK: - The decision func testDefaultsAreTheOnesTheFirstBackupRanWith() { diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift index b3d23a1..283b42c 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -200,4 +200,16 @@ final class VMLayoutTests: XCTestCase { XCTAssertFalse(VMStatus.targets(urls: urls, address: "192.168.64.11")) XCTAssertFalse(VMStatus.targets(urls: [], address: "192.168.64.10")) } + + func testReadingCarriesTheUnsentBacklogFromVfsQueue() { + let stats = #"{"diskCache":{"bytesUsed":124554051584,"uploadsQueued":2,"uploadsInProgress":1}}"# + let queue = #"{"queue":[{"name":"bands/a","size":33554432},{"name":"bands/b","size":1048576}]}"# + let r = VMLayout.reading(scriptOutput: "\(stats)\nCMQUEUE \(queue)\nCMFREE 1\n") + XCTAssertEqual(r?.buffer.unsentBytes, 33_554_432 + 1_048_576) + XCTAssertEqual(r?.buffer.bytesUsed, 124_554_051_584) + let noQueue = VMLayout.reading(scriptOutput: "\(stats)\nCMQUEUE \nCMFREE 1\n") + XCTAssertNotNil(noQueue, "stats and free space still make a reading") + XCTAssertNil(noQueue?.buffer.unsentBytes, "an empty queue answer is unknown, not zero") + XCTAssertEqual(VMLayout.unsentBytes(vfsQueueJSON: #"{"queue":[]}"#), 0) + } } From ed2847321c17b265f4236fa7b7ff81161c3a28ab Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 15:59:44 +0100 Subject: [PATCH 4/6] feat(vm): shape SMB to the upload rate; keep the machine out of Time Machine Both from the first full backup on 27.10.2026, which finished at 15:40 (214 GiB on Drive in 32 MB bands): - Time Machine wrote faster than rclone uploaded, so the guard paused it every ~15 minutes and Time Machine reported each pause as an interruption. smb-throttle shapes incoming SMB (port 445 via ifb0 + HTB 320 Mbit, fq_codel) to roughly the upload rate; the backup then ran without a pause. Policing was tried first and held TCP at ~2.7 MB/s (12% drops) - a test keeps the shaper a queue. - ~/.lima, with the machine's disk and rclone's buffer, was in the backup; the backup sat for half an hour on that one ever-changing file. vm-setup excludes it and vm-guard renews the exclusion on the hour. Co-Authored-By: Claude Opus 5.5 --- .../CloudMachineCore/L10nPolish+VM.swift | 4 ++++ .../Sources/CloudMachineCore/VMGuard.swift | 7 ++++++- .../Sources/CloudMachineCore/VMLayout.swift | 19 ++++++++++++++++++ .../Sources/CloudMachineCore/VMSetup.swift | 12 +++++++++++ .../VMGuardDecisionTests.swift | 18 +++++++++++++++++ .../CloudMachineAppTests/VMLayoutTests.swift | 14 +++++++++++++ vm/cmsmb.yaml | 9 +++++++++ vm/smb-throttle.sh | 20 +++++++++++++++++++ vm/systemd/smb-throttle.service | 16 +++++++++++++++ 9 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 vm/smb-throttle.sh create mode 100644 vm/systemd/smb-throttle.service diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift index 0e65770..c372e6c 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift @@ -156,6 +156,10 @@ extension L10nPolish { "Rozpakowanie obrazu nie powiodło się: %@", "Still to upload: %@ GiB in %@ bands, uploading %@ (cache on disk %@ GiB)": "Do wysłania: %@ GiB w %@ pasmach, wysyłane %@ (cache na dysku %@ GiB)", + "Excluded %@ (the machine and its upload buffer) from Time Machine.": + "Wykluczono %@ (maszyna i jej bufor wysyłki) z Time Machine.", + "WARNING: could not exclude %@ from Time Machine - run: tmutil addexclusion %@": + "UWAGA: nie udało się wykluczyć %@ z Time Machine - uruchom: tmutil addexclusion %@", "Upload buffer: no answer from rclone (rc %@)": "Bufor wysyłki: brak odpowiedzi rclone (rc %@)", "When Time Machine has created its empty image, stop the backup (tmutil stopbackup) and run: cloudmachine-agent vm-prepare-bundle": diff --git a/mac-app/Sources/CloudMachineCore/VMGuard.swift b/mac-app/Sources/CloudMachineCore/VMGuard.swift index 0f90413..342fd42 100644 --- a/mac-app/Sources/CloudMachineCore/VMGuard.swift +++ b/mac-app/Sources/CloudMachineCore/VMGuard.swift @@ -69,6 +69,8 @@ public struct VMGuard: Sendable { public var tmutil: @Sendable (String) async -> Bool public var log: @Sendable (String) -> Void public var minute: @Sendable () -> Int + /// Renews the Time Machine exclusion of the machine's directory (hourly). + public var excludeMachine: @Sendable () async -> Void = {} public static let live = Probes( reading: { await VMLayout.currentReading() }, @@ -85,7 +87,8 @@ public struct VMGuard: Sendable { }, tmutil: { await BufferGuardService.tmutil($0) }, log: { CMLogger.log($0) }, - minute: { Calendar.current.component(.minute, from: Date()) }) + minute: { Calendar.current.component(.minute, from: Date()) }, + excludeMachine: { await VMLayout.excludeMachineFromTimeMachine() }) } public var thresholds: Thresholds @@ -137,6 +140,8 @@ public struct VMGuard: Sendable { break } if probes.minute() == 0 { + // The exclusion is an xattr; a recreated ~/.lima would silently lose it. + await probes.excludeMachine() let state = running.map { $0 ? "1" : "0" } ?? "unknown" probes.log( "VM guard ok: buffer \(used) GiB, queue \(queued), free \(free) GiB, TM running=\(state)") diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift index 809405c..4189535 100644 --- a/mac-app/Sources/CloudMachineCore/VMLayout.swift +++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift @@ -61,6 +61,25 @@ public enum VMLayout { ?? "\(DriveBufferService.remoteName):\(DriveBufferService.remoteRoot)/\(DriveFolder.name)-tm" } + /// Lima's directory with the machine's disk (`LIMA_HOME`, else `~/.lima`). + /// The disk holds rclone's buffer: left in the backup, Time Machine copies + /// its own backlog through the machine - on 27.10.2026 that kept the first + /// backup on one ever-changing 17 GB file for half an hour. + public static var limaHome: URL { + if let custom = override("LIMA_HOME") { return URL(fileURLWithPath: custom) } + return FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".lima") + } + + /// Keeps `limaHome` out of Time Machine. The exclusion is an xattr on the + /// directory, so it is renewed rather than set once (as for the old buffer, + /// `DriveBufferService.excludeBufferFromTimeMachine`). + @discardableResult + public static func excludeMachineFromTimeMachine() async -> Bool { + let result = try? await ProcessRunner.run( + "/usr/bin/tmutil", ["addexclusion", limaHome.path], timeout: 30) + return result?.succeeded == true + } + /// `vm/` among the resources (the repo root in a checkout, Resources in the app). public static var templateDir: URL? { guard let root = CMPaths.resourcesRoot else { return nil } diff --git a/mac-app/Sources/CloudMachineCore/VMSetup.swift b/mac-app/Sources/CloudMachineCore/VMSetup.swift index 348f7f1..b3e888f 100644 --- a/mac-app/Sources/CloudMachineCore/VMSetup.swift +++ b/mac-app/Sources/CloudMachineCore/VMSetup.swift @@ -73,6 +73,18 @@ public enum VMSetup { messages.append(L10n.tr("Machine %@ was already running.", VMLayout.instanceName)) } + if await VMLayout.excludeMachineFromTimeMachine() { + messages.append( + L10n.tr( + "Excluded %@ (the machine and its upload buffer) from Time Machine.", + VMLayout.limaHome.path)) + } else { + messages.append( + L10n.tr( + "WARNING: could not exclude %@ from Time Machine - run: tmutil addexclusion %@", + VMLayout.limaHome.path, VMLayout.limaHome.path)) + } + // 2. The Drive folder. let env = VMLayout.envLine("CM_REMOTE", VMLayout.remote) guard diff --git a/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift b/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift index 4fec2d7..17dd741 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMGuardDecisionTests.swift @@ -234,4 +234,22 @@ final class VMGuardDecisionTests: XCTestCase { XCTAssertTrue(fake.marker) XCTAssertTrue(fake.log.contains { $0.contains("NO ANSWER") }, "\(fake.log)") } + + /// 27.10.2026: ~/.lima was in the backup and the first backup sat on the + /// machine's own disk. The guard renews the exclusion on the hour. + func testRenewsTheMachineExclusionOnTheHour() async { + final class Count: @unchecked Sendable { var n = 0 } + let count = Count() + let fake = Fake() + fake.reading = reading(usedGiB: 10) + fake.running = true + var probes = fake.probes + probes.excludeMachine = { count.n += 1 } + probes.minute = { 0 } + _ = await VMGuard(thresholds: thresholds, probes: probes).step() + XCTAssertEqual(count.n, 1) + probes.minute = { 17 } + _ = await VMGuard(thresholds: thresholds, probes: probes).step() + XCTAssertEqual(count.n, 1, "only on the hour") + } } diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift index 283b42c..d5b8d8e 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -175,6 +175,20 @@ final class VMLayoutTests: XCTestCase { XCTAssertTrue(smbConf.contains("max open files = 500000")) } + /// 27.10.2026: policing (`police ... drop`) held SMB at ~2.7 MB/s; the + /// shaper must stay a queue (ifb + htb), and the template must install it. + func testSMBShaperQueuesInsteadOfDropping() throws { + let vm = Self.repoRoot.appendingPathComponent("vm") + let script = try String( + contentsOf: vm.appendingPathComponent("smb-throttle.sh"), encoding: .utf8) + let yaml = try String(contentsOf: vm.appendingPathComponent("cmsmb.yaml"), encoding: .utf8) + XCTAssertTrue(script.contains("mirred egress redirect dev ifb0")) + XCTAssertTrue(script.contains("htb rate")) + XCTAssertFalse(script.contains("police rate"), "policing drops packets and collapses TCP") + XCTAssertTrue(yaml.contains("file: systemd/smb-throttle.service")) + XCTAssertTrue(yaml.contains("systemctl enable --now smb-throttle.service")) + } + /// `install-launchd` installs every template at the TOP of `launchd/`. The /// machine's agents must not be there: a Mac on the attached-image layout /// would get an agent starting a machine it does not have. diff --git a/vm/cmsmb.yaml b/vm/cmsmb.yaml index eafab0b..52b0a8e 100644 --- a/vm/cmsmb.yaml +++ b/vm/cmsmb.yaml @@ -52,6 +52,13 @@ provision: - mode: data path: /opt/cloudmachine/punchfs/passthrough_helpers.h file: punchfs/passthrough_helpers.h +- mode: data + path: /usr/local/sbin/smb-throttle + file: smb-throttle.sh + permissions: 755 +- mode: data + path: /etc/systemd/system/smb-throttle.service + file: systemd/smb-throttle.service - mode: data path: /opt/cloudmachine/punchfs/test-punchfs.sh file: punchfs/test-punchfs.sh @@ -111,6 +118,8 @@ provision: # /etc/cloudmachine/bundle.env, which `vm-prepare-bundle` writes. systemctl daemon-reload systemctl enable rclone-tm.service punchfs.service tm-bands-bind.service smbd.service + # The SMB shaper needs nothing from the Mac; start it every boot. + systemctl enable --now smb-throttle.service if [ -s /etc/rclone/rclone.conf ] && [ -s /etc/cloudmachine/vm.env ]; then systemctl start rclone-tm.service punchfs.service tm-bands-bind.service smbd.service fi diff --git a/vm/smb-throttle.sh b/vm/smb-throttle.sh new file mode 100644 index 0000000..684b50d --- /dev/null +++ b/vm/smb-throttle.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Shapes (does not police!) incoming SMB: traffic to port 445 goes through ifb0 + HTB. +# Policing dropped 12% of packets and TCP fell to ~2.7 MB/s (27.10.2026); a queue holds the rate without loss. +# Why at all: Time Machine writes faster than rclone uploads to Drive; without this the +# guard has to pause the backup over and over, and Time Machine reports every pause as an interruption. +RATE="${RATE:-320mbit}" +case "$1" in + start) + modprobe ifb numifbs=1 2>/dev/null || modprobe ifb + ip link show ifb0 >/dev/null 2>&1 || ip link add ifb0 type ifb + ip link set ifb0 up + tc qdisc del dev lima0 ingress 2>/dev/null; tc qdisc del dev ifb0 root 2>/dev/null + tc qdisc add dev lima0 handle ffff: ingress + tc filter add dev lima0 parent ffff: protocol ip prio 1 u32 match ip dport 445 0xffff action mirred egress redirect dev ifb0 + tc qdisc add dev ifb0 root handle 1: htb default 10 r2q 100 + tc class add dev ifb0 parent 1: classid 1:10 htb rate "$RATE" ceil "$RATE" burst 1mb + tc qdisc add dev ifb0 parent 1:10 handle 10: fq_codel ;; + stop) + tc qdisc del dev lima0 ingress 2>/dev/null; tc qdisc del dev ifb0 root 2>/dev/null; true ;; +esac diff --git a/vm/systemd/smb-throttle.service b/vm/systemd/smb-throttle.service new file mode 100644 index 0000000..d652d66 --- /dev/null +++ b/vm/systemd/smb-throttle.service @@ -0,0 +1,16 @@ +[Unit] +Description=CloudMachine: shape incoming SMB (Time Machine) to the Google Drive upload rate +After=network-online.target +Wants=network-online.target +Before=smbd.service + +[Service] +Type=oneshot +RemainAfterExit=yes +# 320 Mbit/s = ~38 MB/s, measured upload to Drive 25-65 MiB/s (27.10.2026). +Environment=RATE=320mbit +ExecStart=/usr/local/sbin/smb-throttle start +ExecStop=/usr/local/sbin/smb-throttle stop + +[Install] +WantedBy=multi-user.target From 71697d26def28a87bfda60a8efb4c238bbe62763 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 16:30:25 +0100 Subject: [PATCH 5/6] feat: one layout - Time Machine over SMB to the machine; remove the disk-image layout macOS 27 refuses a disk image as a new Time Machine destination, so the rclone mount on the Mac and the image in /Volumes/CloudMachine are gone: DriveBufferService, BackupImageService, ImageProbe, AttachStall, MountRestart, OrphanMount, InstallationTraces, BufferGuardService, ShutdownPrep, FuseInstaller, BufferReadiness, UploadDrain, OpenPortAlert, CMLock, the POC harnesses, the gdrive-buffer/gdrive-attach/buffer-guard agents and the commands that drove them. - install-launchd installs app, backup-health, vm-start and vm-guard; the retired agents are booted out and their plists deleted on launch, by install-launchd and by backup-health; leftovers are only logged. - backup-health judges the machine (Lima state, services, upload queue, free space, punchfs descriptors, the SMB destination) and the date of the last completed backup. - prepare-shutdown stops the backup, drains the upload for a while and stops the machine cleanly. - The app shows the machine and walks through install-rclone, configure-remote, brew install lima, vm-setup, tmutil setdestination (password masked on screen) and vm-prepare-bundle. - vm/cmsmb.yaml forwards no ports: rclone's password-less remote control answered on the Mac's localhost; vm-status and the watchdog report it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AU1MhxJW1e9oTB2XvjX8WT --- README.md | 156 ++- ...m.renacode.cloudmachine.app.plist.template | 4 +- ....cloudmachine.backup-health.plist.template | 5 +- ...e.cloudmachine.buffer-guard.plist.template | 28 - ....cloudmachine.gdrive-attach.plist.template | 32 - ....cloudmachine.gdrive-buffer.plist.template | 34 - ...acode.cloudmachine.vm-guard.plist.template | 4 +- ...acode.cloudmachine.vm-start.plist.template | 0 mac-app/Package.swift | 25 +- .../CloudMachineAgent/BuildAppCommand.swift | 2 +- .../CloudMachineAgent/CloudMachineAgent.swift | 30 +- .../CloudMachineAgent/DriveCommands.swift | 605 ----------- .../MaintenanceCommands.swift | 166 +++ .../CloudMachineAgent/SetupCommands.swift | 11 +- .../CloudMachineAgent/VMCommands.swift | 40 +- .../CloudMachineApp/Models/AppStatus.swift | 338 +++--- .../CloudMachineApp/Models/SetupPlan.swift | 108 +- .../Services/CloudMachineController.swift | 203 ++-- .../CloudMachineApp/Views/DashboardView.swift | 337 +++--- .../Views/MenuBarContentView.swift | 53 +- .../CloudMachineCore/AgentRepair.swift | 149 ++- .../CloudMachineCore/AttachStall.swift | 156 --- .../CloudMachineCore/BackupHealth.swift | 417 ++++---- .../CloudMachineCore/BackupImageService.swift | 946 ----------------- .../CloudMachineCore/BufferGuardService.swift | 813 --------------- .../CloudMachineCore/BufferReadiness.swift | 51 - .../CloudMachineCore/CMActionResult.swift | 45 +- mac-app/Sources/CloudMachineCore/CMLock.swift | 280 ----- .../Sources/CloudMachineCore/CMPaths.swift | 11 +- .../Sources/CloudMachineCore/CMTooling.swift | 82 +- .../CloudMachineCore/DriveBufferService.swift | 959 ------------------ .../CloudMachineCore/DriveFolder.swift | 40 +- .../CloudMachineCore/EdgeTriggeredLog.swift | 23 - .../CloudMachineCore/FuseInstaller.swift | 189 ---- .../CloudMachineCore/HealthAlert.swift | 6 +- .../Sources/CloudMachineCore/ImageProbe.swift | 322 ------ .../CloudMachineCore/InstallationTraces.swift | 103 -- mac-app/Sources/CloudMachineCore/L10n.swift | 1 + .../CloudMachineCore/L10nPolish+Agent.swift | 114 +-- .../CloudMachineCore/L10nPolish+App.swift | 44 - .../CloudMachineCore/L10nPolish+Machine.swift | 176 ++++ .../CloudMachineCore/L10nPolish+Safety.swift | 76 +- .../CloudMachineCore/L10nPolish+Storage.swift | 68 +- .../CloudMachineCore/L10nPolish+System.swift | 82 +- .../CloudMachineCore/L10nPolish+VM.swift | 2 - .../CloudMachineCore/LaunchdInstaller.swift | 101 +- .../CloudMachineCore/MachineBudget.swift | 16 +- .../CloudMachineCore/MountRestart.swift | 207 ---- .../CloudMachineCore/OpenPortAlert.swift | 89 -- .../CloudMachineCore/OrphanMount.swift | 133 --- .../CloudMachineCore/ProcessRunner.swift | 6 +- .../CloudMachineCore/RcloneInstaller.swift | 12 +- .../CloudMachineCore/RemoteConfigurer.swift | 167 +-- .../CloudMachineCore/ShutdownPrep.swift | 123 --- .../CloudMachineCore/StatusLines.swift | 107 +- .../CloudMachineCore/TimeMachineStatus.swift | 226 ++--- .../CloudMachineCore/UploadDrain.swift | 90 -- .../CloudMachineCore/UploadState.swift | 2 +- .../CloudMachineCore/VMBundlePrep.swift | 4 +- .../Sources/CloudMachineCore/VMGuard.swift | 4 +- .../Sources/CloudMachineCore/VMLayout.swift | 50 +- .../Sources/CloudMachineCore/VMSetup.swift | 15 +- .../Sources/CloudMachineCore/VMShutdown.swift | 111 ++ .../Sources/CloudMachineCore/VMStatus.swift | 149 ++- .../CloudMachineCore/WatchdogHeartbeat.swift | 2 +- .../AmplificationCommand.swift | 143 --- .../CloudMachinePOC/CloudMachinePOC.swift | 22 - .../Sources/CloudMachinePOC/POCSupport.swift | 173 ---- .../CloudMachinePOC/PullPlugCommand.swift | 347 ------- .../AgentRepairTests.swift | 93 +- .../AppStatusFixtures.swift | 39 + .../AppStatusHealthTests.swift | 340 +++---- .../AttachStallTests.swift | 152 --- .../BackupHealthTests.swift | 199 ++-- .../BackupImageServiceTests.swift | 282 ----- .../BackupVisibilityTests.swift | 106 +- .../BufferGuardDecisionTests.swift | 705 ------------- .../BufferReadinessTests.swift | 103 -- .../CloudMachineAppTests/CMLockTests.swift | 80 -- .../DriveLayerTests.swift | 535 +--------- .../HealthAlertTests.swift | 2 +- .../ImageProbeTests.swift | 283 ------ .../OpenRemoteControlTests.swift | 219 ---- .../OrphanMountTests.swift | 59 -- .../PullPlugHarnessTests.swift | 118 --- .../QueueStatsParsingTests.swift | 134 --- .../RcloneLogReadabilityTests.swift | 119 --- .../RcloneLogRotationTests.swift | 97 -- .../CloudMachineAppTests/SetupPlanTests.swift | 90 +- .../ShutdownPrepTests.swift | 57 -- .../StatusLinesTests.swift | 90 +- .../TimeMachineDestinationStateTests.swift | 71 +- .../TimeMachineStatusTests.swift | 78 +- .../TimeMachineTimeoutTests.swift | 87 +- .../UploadDrainTests.swift | 84 -- .../CloudMachineAppTests/VMLayoutTests.swift | 91 +- packaging/README.md | 11 +- packaging/homebrew/cloudmachine.rb.in | 30 +- vm/README.md | 21 +- vm/cmsmb.yaml | 11 + 100 files changed, 2563 insertions(+), 11458 deletions(-) delete mode 100644 launchd/com.renacode.cloudmachine.buffer-guard.plist.template delete mode 100644 launchd/com.renacode.cloudmachine.gdrive-attach.plist.template delete mode 100644 launchd/com.renacode.cloudmachine.gdrive-buffer.plist.template rename launchd/{vm => }/com.renacode.cloudmachine.vm-guard.plist.template (86%) rename launchd/{vm => }/com.renacode.cloudmachine.vm-start.plist.template (100%) delete mode 100644 mac-app/Sources/CloudMachineAgent/DriveCommands.swift create mode 100644 mac-app/Sources/CloudMachineAgent/MaintenanceCommands.swift delete mode 100644 mac-app/Sources/CloudMachineCore/AttachStall.swift delete mode 100644 mac-app/Sources/CloudMachineCore/BackupImageService.swift delete mode 100644 mac-app/Sources/CloudMachineCore/BufferGuardService.swift delete mode 100644 mac-app/Sources/CloudMachineCore/BufferReadiness.swift delete mode 100644 mac-app/Sources/CloudMachineCore/CMLock.swift delete mode 100644 mac-app/Sources/CloudMachineCore/DriveBufferService.swift delete mode 100644 mac-app/Sources/CloudMachineCore/EdgeTriggeredLog.swift delete mode 100644 mac-app/Sources/CloudMachineCore/FuseInstaller.swift delete mode 100644 mac-app/Sources/CloudMachineCore/ImageProbe.swift delete mode 100644 mac-app/Sources/CloudMachineCore/InstallationTraces.swift create mode 100644 mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift delete mode 100644 mac-app/Sources/CloudMachineCore/MountRestart.swift delete mode 100644 mac-app/Sources/CloudMachineCore/OpenPortAlert.swift delete mode 100644 mac-app/Sources/CloudMachineCore/OrphanMount.swift delete mode 100644 mac-app/Sources/CloudMachineCore/ShutdownPrep.swift delete mode 100644 mac-app/Sources/CloudMachineCore/UploadDrain.swift create mode 100644 mac-app/Sources/CloudMachineCore/VMShutdown.swift delete mode 100644 mac-app/Sources/CloudMachinePOC/AmplificationCommand.swift delete mode 100644 mac-app/Sources/CloudMachinePOC/CloudMachinePOC.swift delete mode 100644 mac-app/Sources/CloudMachinePOC/POCSupport.swift delete mode 100644 mac-app/Sources/CloudMachinePOC/PullPlugCommand.swift create mode 100644 mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/AttachStallTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/BackupImageServiceTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/BufferGuardDecisionTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/BufferReadinessTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/CMLockTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/ImageProbeTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/OpenRemoteControlTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/OrphanMountTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/PullPlugHarnessTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/QueueStatsParsingTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/RcloneLogReadabilityTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/RcloneLogRotationTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/ShutdownPrepTests.swift delete mode 100644 mac-app/Tests/CloudMachineAppTests/UploadDrainTests.swift diff --git a/README.md b/README.md index 8a2533c..53ff6f7 100644 --- a/README.md +++ b/README.md @@ -7,35 +7,35 @@ Native Time Machine, backed by Google Drive. No external disk, no NAS, nothing plugged in at home. -Time Machine writes to a disk image that macOS mounts like any other volume. The -image's contents live on Google Drive, behind a local write buffer. Backups land -in the buffer at SSD speed and drain to the cloud in the background, so a dropped -connection stalls the upload instead of interrupting the backup. +Time Machine backs up over SMB to a small Linux machine that runs on this Mac +([Lima](https://lima-vm.io), Apple Virtualization). The machine keeps the +backup's small files on its own disk and its bands on Google Drive, behind a +local upload buffer: backups land in the buffer at disk speed and drain to the +cloud in the background, so a dropped connection stalls the upload instead of +interrupting the backup. ``` -Time Machine - -> /Volumes/CloudMachine attached sparsebundle; Time Machine sees plain APFS - -> ~/.cloudmachine/drive rclone mount over FUSE-T - -> ~/.cloudmachine/cache 100 GB write buffer - -> gdrive:CloudMachine/mac-studio Google Drive +Time Machine --SMB--> machine cmsmb (Lima) + Samba /srv/tm-local/.sparsebundle image metadata, local + bands/ -> punchfs -> rclone mount 120 GB upload buffer + gdrive:CloudMachine/-tm Google Drive ``` -**No network filesystem sits in the write path.** That is the whole point. -Time Machine over SMB to a NAS or a cloud share is the common approach and the -fragile one — a sparsebundle written directly over a network link corrupts when -the link drops. Here Time Machine talks to a locally attached image and never -knows the cloud exists. +Why a machine at all: macOS 27 no longer accepts a disk image as a new Time +Machine destination (`tmutil setdestination` answers error 45), and refuses SMB +to the Mac's own address. A share served from a machine on the Mac is what Time +Machine still accepts. Why each layer inside it is there: [`vm/README.md`](vm/README.md). --- ## Requirements -- macOS 14 (Sonoma) or newer, and an administrator password for one step - (pointing Time Machine at CloudMachine). +- macOS 14 (Sonoma) or newer, Apple silicon, and an administrator password for + one step (pointing Time Machine at the machine). - A Google account with room to spare. -- [Homebrew](https://brew.sh). -- Nothing else at runtime. CloudMachine installs its own `rclone` and its own - copy of FUSE-T. +- [Homebrew](https://brew.sh) and Lima from it (`brew install lima`). +- About 200 GB of free disk: the machine's disk file grows with the upload + buffer. The menu-bar app, CLI output and notifications follow the system language: Polish when Polish is the first preferred language, English otherwise. @@ -47,25 +47,9 @@ so they read the same whoever sends them to you. - **Not notarised.** Releases are signed with a self-signed certificate, not with an Apple Developer ID. The Homebrew cask clears the quarantine flag; a DMG downloaded by hand gets Gatekeeper's "unidentified developer" warning. - -### macOS 27: Time Machine over SMB to a local machine (preview) - -macOS 27 no longer accepts a disk image as a new Time Machine destination -(`tmutil setdestination` answers error 45), so the layout above cannot be set -up from scratch there. The replacement, being rolled out next to it: a small -Linux machine on this Mac serves Time Machine an SMB share, and the backup's -bands go from it to Google Drive (details in [`vm/README.md`](vm/README.md)). - -- Requires [Lima](https://lima-vm.io) from Homebrew (`brew install lima`) and - a connected Google Drive (`configure-remote`). -- `cloudmachine-agent vm-setup` creates and starts the machine and prints - a `sudo tmutil setdestination smb://...` command - the one step that needs - an administrator, once. -- When Time Machine has created its empty image, stop the backup and run - `cloudmachine-agent vm-prepare-bundle` (32 MB bands, moved to Drive). -- `cloudmachine-agent vm-status` shows the machine; `vm-setup --install-agents` - adds the agents that start it at login and pause Time Machine while the - upload buffer is full. +- **Google's daily upload limit and a full Drive are not named in the app.** + rclone's log that says so lives in the machine; what the app does show is + the backlog growing and the guard pausing Time Machine. --- @@ -78,33 +62,43 @@ open -a CloudMachine CloudMachine lives in the menu bar. Its window opens on a **Required Setup Steps** card listing what is left to do on this Mac, in order, each with a -button — or, for the two steps an app cannot do, a command to copy: +button — or, for the steps an app cannot do, a command to copy: -1. **Install rclone** and **Install FUSE-T.** CloudMachine downloads its own - copies; nothing else is installed system-wide. +1. **Install rclone.** CloudMachine downloads its own copy; it signs in to + Google Drive and writes the configuration the machine gets a copy of. 2. **Connect Google Drive** — a command to run in Terminal. It opens Google's sign-in in the browser and waits for your approval. Enter your own OAuth credentials first (see [below](#your-own-google-oauth-credentials)). -3. **Grant Full Disk Access** — opens the right pane of System Settings. +3. **Install Lima** — `brew install lima`, to copy. +4. **Set up the machine** — creates and starts `cmsmb` (the first start + downloads Ubuntu and installs packages, several minutes), copies rclone's + configuration into it and sets its SMB password. +5. **Grant Full Disk Access** — opens the right pane of System Settings. Without it CloudMachine cannot read when Time Machine last *finished* a backup, which is the one check that matters. -4. **Install agents** — the launchd agents that keep the Drive mounted, attach - the image and watch the backup. -5. **Create image**, then **Attach image** — the backup image on Google Drive. - The size is a ceiling, not an allocation: the image is sparse, and Drive - only holds what has been written. -6. **Point Time Machine at CloudMachine** — a `sudo tmutil setdestination` - command to copy, because only an administrator can change it. +6. **Install agents** — the launchd agents that start the machine at login, + pause Time Machine while the upload buffer is full, and watch the backup. +7. **Point Time Machine at the machine** — a `sudo tmutil setdestination + smb://tm:…@
/CloudMachine` command. The card shows the password + masked; **Copy** puts the real command on the clipboard. +8. **Convert the image** — once Time Machine has created its first, empty + image, stop that backup (`tmutil stopbackup`) and press the button + (`cloudmachine-agent vm-prepare-bundle`): the image gets 32 MB bands, and + the bands move to Google Drive. When the card disappears, setup is done. Turn on automatic backups in System Settings → General → Time Machine, or run `sudo tmutil enable`. +The same from Terminal: `install-rclone`, `configure-remote`, `vm-setup +--install-agents` (prints the `tmutil` command), then `vm-prepare-bundle`; +`vm-status` shows everything at once. + ### Several Macs, one Google account Install CloudMachine on each Mac and go through the same steps; they can all use the same Google account and the same OAuth credentials. Each Mac backs up -into its own folder, `gdrive:CloudMachine/`, holding -`.sparsebundle`, so their backups never mix. +into its own folder, `gdrive:CloudMachine/-tm`, so their backups never +mix. The folder name is chosen once, in the **Connect Google Drive** step: the card has a **Folder on Google Drive** field, filled in from the computer name, and @@ -122,33 +116,38 @@ It works two ways: - **Time Machine quota.** The card gives a `sudo tmutil setquota` command to copy; Time Machine then deletes this Mac's oldest backups to stay under it. - The quota is 70% of the limit, because the image on Drive grows larger than - the backup inside it (see `renacode-infra: docs/projekty/cloudmachine/design.md`). + The quota is 70% of the limit, because the bands on Drive grow larger than + the backup inside them (see `renacode-infra: docs/projekty/cloudmachine/design.md`). - **An alarm on the real usage.** The watchdog measures this Mac's folder on Drive every few hours and warns from 90% of the limit, and again above it. - The card and `drive-status` show the usage. + The card and `vm-status` show the usage. ### Upgrading and uninstalling -`brew upgrade --cask renacode/tap/cloudmachine` replaces the app without -restarting the Google Drive mount. The cask in the tap can be a version -behind the latest GitHub Release: it is updated by a pull request after each -release (see `packaging/README.md`). -When Homebrew reopens the app, it reloads the background agents so they run -the new version; `cloudmachine-agent drive-status` shows `Agents: OK`. Changes -to the mount itself wait for its next start. One such change is rclone's -control interface moving to a private socket: until then the old mount takes -commands on `127.0.0.1:5572` without a password, and the app window shows a red -card, the menu bar an alert, and the watchdog a notification (once, then after -1, 2, 4 and every 7 days). **Restart the mount** on that card - or -`cloudmachine-agent restart-mount` - detaches the image, waits until everything -has reached Google Drive, restarts the mount and attaches the image again; it -refuses while Time Machine is backing up. Restarting the Mac after -`prepare-shutdown` does the same. If a new version does not show up, run -`brew update` first - Homebrew refreshes the tap only now and then. Neither `brew uninstall` nor -`--zap` touches the launchd agents or the upload buffer in `~/.cloudmachine`, -which may hold backups that have not reached Google Drive yet. Run -`cloudmachine-agent prepare-shutdown` before uninstalling. +`brew upgrade --cask renacode/tap/cloudmachine` replaces the app; the machine +keeps running. The cask in the tap can be a version behind the latest GitHub +Release: it is updated by a pull request after each release (see +`packaging/README.md`). If a new version does not show up, run `brew update` +first - Homebrew refreshes the tap only now and then. When Homebrew reopens +the app, it reloads the background agents so they run the new version; +`cloudmachine-agent vm-status` shows `Agents: OK`. + +**Coming from the disk-image layout** (an rclone mount on the Mac and an image +in `/Volumes/CloudMachine`): the app removes its launchd agents +(`gdrive-buffer`, `gdrive-attach`, `buffer-guard`) on its first launch and +lists in `~/Library/Logs/CloudMachine/cloudmachine.log` what that layout left +behind - `~/.cloudmachine/cache`, `drive`, `fuse`, the old image in +`gdrive:CloudMachine/` - without deleting any of it. The new layout +starts a new backup in `-tm`; delete the old one when you no longer +need to restore from it. + +Before restarting the Mac, **Prepare for restart** in the menu (or +`cloudmachine-agent prepare-shutdown`) stops the backup, gives the upload up +to 30 minutes, and shuts the machine down cleanly; it starts again at the next +login. Neither `brew uninstall` nor `--zap` touches the launchd agents or the +machine, whose buffer may hold backups that have not reached Google Drive yet: +run `prepare-shutdown`, then `limactl delete cmsmb` and remove +`~/Library/LaunchAgents/com.renacode.cloudmachine.*.plist`. ### Your own Google OAuth credentials @@ -166,13 +165,13 @@ compete with every other rclone user. The connection uses scope `drive.file`, which grants access only to files this application itself created. Full `drive` scope would hand out read, write and **delete** over the entire Google account, which is far more than a folder of -disk-image bands needs — especially with `--drive-use-trash=false`, where a +backup bands needs — especially with `--drive-use-trash=false`, where a delete has no bin to recover from. On a Mac that is already connected, keep the connection it has. A new sign-in with different credentials sees only what it created itself, so the existing backup would vanish from view; `configure-remote` refuses to replace a working -connection unless told twice, and checks the new token against the image first +connection unless told twice, and checks the new token against the backup first (see `renacode-infra: docs/projekty/cloudmachine/setup-cli.md`, "Replacing the connection"). --- @@ -184,17 +183,16 @@ repository (`renacode-infra: docs/projekty/cloudmachine/`): - Running CloudMachine (`operations.md`) — checking from Terminal, knowing when backups stop, reading the logs, before rebooting. -- Restoring (`restoring.md`) — getting files back, and checking the image. +- Restoring (`restoring.md`) — getting files back. - How it works, and what it costs (`design.md`) — measured traffic and storage, and why each piece is the way it is. - Setting up from Terminal (`setup-cli.md`) — the same setup without the app. -- Building from source (`building.md`) — local builds, tests, releases, - measurement harnesses. +- Building from source (`building.md`) — local builds, tests, releases. --- ## Licence -MIT. FUSE-T and rclone keep their own licences. One file is GPLv2: +MIT. rclone, Lima and Samba keep their own licences. One file is GPLv2: `vm/punchfs/punchfs.c`, based on libfuse's passthrough example (see `vm/punchfs/README.md`). diff --git a/launchd/com.renacode.cloudmachine.app.plist.template b/launchd/com.renacode.cloudmachine.app.plist.template index 384c445..5b94cc0 100644 --- a/launchd/com.renacode.cloudmachine.app.plist.template +++ b/launchd/com.renacode.cloudmachine.app.plist.template @@ -21,8 +21,8 @@ would launch it over and over. The user closing the window should stay closed, not be an invitation to restart. - The backups themselves do NOT depend on this app - the gdrive-buffer, - gdrive-attach and buffer-guard agents do them. This is only a viewer. + The backups themselves do NOT depend on this app - the machine (started + by vm-start) and the vm-guard agent do them. This is only a viewer. --> StandardOutPath __CM_LOG_DIR__/launchd-app.out.log diff --git a/launchd/com.renacode.cloudmachine.backup-health.plist.template b/launchd/com.renacode.cloudmachine.backup-health.plist.template index 6ace24c..fc44ffe 100644 --- a/launchd/com.renacode.cloudmachine.backup-health.plist.template +++ b/launchd/com.renacode.cloudmachine.backup-health.plist.template @@ -23,8 +23,9 @@ Every 30 minutes, with an hourly cycle and a 3 h threshold - often enough that a broken cycle does not sit unnoticed for a day, and rarely enough to cost nothing. The watchdog reads the local Time Machine preferences file, - NOT the Google Drive mount: a supervisor that hangs together with what it - supervises goes silent exactly when it should speak. + not the backup destination, and every question to the machine has a time + limit: a supervisor that hangs together with what it supervises goes + silent exactly when it should speak. --> StartInterval 1800 diff --git a/launchd/com.renacode.cloudmachine.buffer-guard.plist.template b/launchd/com.renacode.cloudmachine.buffer-guard.plist.template deleted file mode 100644 index 03590ed..0000000 --- a/launchd/com.renacode.cloudmachine.buffer-guard.plist.template +++ /dev/null @@ -1,28 +0,0 @@ - - - - - Label - com.renacode.cloudmachine.buffer-guard - ProgramArguments - - __CM_AGENT_BIN__ - buffer-guard - - RunAtLoad - - - KeepAlive - - ThrottleInterval - 60 - StandardOutPath - __CM_LOG_DIR__/launchd-buffer-guard.out.log - StandardErrorPath - __CM_LOG_DIR__/launchd-buffer-guard.err.log - - diff --git a/launchd/com.renacode.cloudmachine.gdrive-attach.plist.template b/launchd/com.renacode.cloudmachine.gdrive-attach.plist.template deleted file mode 100644 index 493810d..0000000 --- a/launchd/com.renacode.cloudmachine.gdrive-attach.plist.template +++ /dev/null @@ -1,32 +0,0 @@ - - - - - Label - com.renacode.cloudmachine.gdrive-attach - ProgramArguments - - __CM_AGENT_BIN__ - attach-image - - RunAtLoad - - - StartInterval - 900 - StandardOutPath - __CM_LOG_DIR__/launchd-gdrive-attach.out.log - StandardErrorPath - __CM_LOG_DIR__/launchd-gdrive-attach.err.log - - diff --git a/launchd/com.renacode.cloudmachine.gdrive-buffer.plist.template b/launchd/com.renacode.cloudmachine.gdrive-buffer.plist.template deleted file mode 100644 index 25915a3..0000000 --- a/launchd/com.renacode.cloudmachine.gdrive-buffer.plist.template +++ /dev/null @@ -1,34 +0,0 @@ - - - - - Label - com.renacode.cloudmachine.gdrive-buffer - ProgramArguments - - __CM_AGENT_BIN__ - mount-drive - - RunAtLoad - - - KeepAlive - - ThrottleInterval - 30 - StandardOutPath - __CM_LOG_DIR__/launchd-gdrive-buffer.out.log - StandardErrorPath - __CM_LOG_DIR__/launchd-gdrive-buffer.err.log - - diff --git a/launchd/vm/com.renacode.cloudmachine.vm-guard.plist.template b/launchd/com.renacode.cloudmachine.vm-guard.plist.template similarity index 86% rename from launchd/vm/com.renacode.cloudmachine.vm-guard.plist.template rename to launchd/com.renacode.cloudmachine.vm-guard.plist.template index cf92f6e..c90f183 100644 --- a/launchd/vm/com.renacode.cloudmachine.vm-guard.plist.template +++ b/launchd/com.renacode.cloudmachine.vm-guard.plist.template @@ -6,8 +6,8 @@ com.renacode.cloudmachine.vm-guard ProgramArguments diff --git a/launchd/vm/com.renacode.cloudmachine.vm-start.plist.template b/launchd/com.renacode.cloudmachine.vm-start.plist.template similarity index 100% rename from launchd/vm/com.renacode.cloudmachine.vm-start.plist.template rename to launchd/com.renacode.cloudmachine.vm-start.plist.template diff --git a/mac-app/Package.swift b/mac-app/Package.swift index 937a487..7119911 100644 --- a/mac-app/Package.swift +++ b/mac-app/Package.swift @@ -29,32 +29,9 @@ let package = Package( ], path: "Sources/CloudMachineAgent" ), - .executableTarget( - // Measurement harnesses (formerly gdrive/poc-*.sh). Deliberately a - // SEPARATE binary: they measure the behaviour of hdiutil and FUSE-T, - // not our code, and they are not part of the running system - - // `build-app` does not copy them into the bundle. A separate target - // rather than agent subcommands precisely so that they cannot be - // run on production by accident: each of them creates and deletes - // disk images. - name: "cloudmachine-poc", - dependencies: [ - "CloudMachineCore", - .product(name: "ArgumentParser", package: "swift-argument-parser") - ], - path: "Sources/CloudMachinePOC" - ), .testTarget( name: "CloudMachineAppTests", - // `cloudmachine-poc` has been here since 2026-09-25, on purpose: - // the harnesses measure the behaviour of hdiutil and FUSE-T, but - // the WAY they report on it is ordinary code and it broke - // silently - `pullplug` reported "The image survived every - // floor pull" after a run in which writing never started. The - // tests touch ONLY the pure parts (result classification, - // summary, attempting to write to a directory that does not - // exist) - none of them creates a disk image. - dependencies: ["CloudMachineApp", "CloudMachineCore", "cloudmachine-poc"], + dependencies: ["CloudMachineApp", "CloudMachineCore"], path: "Tests/CloudMachineAppTests" ) ] diff --git a/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift b/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift index 70c2f2f..1559b85 100644 --- a/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift +++ b/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift @@ -19,7 +19,7 @@ struct BuildApp: AsyncParsableCommand { commandName: "build-app", abstract: L10n.tr( - "Builds CloudMachine.app (Release) - GUI + cloudmachine-agent in Contents/MacOS/, plus launchd/config as Resources." + "Builds CloudMachine.app (Release) - GUI + cloudmachine-agent in Contents/MacOS/, plus launchd/, vm/ and config/ as Resources." ) ) diff --git a/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift b/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift index 6462080..57468fb 100644 --- a/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift +++ b/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift @@ -11,32 +11,24 @@ struct CloudMachineAgent: AsyncParsableCommand { "CloudMachine - verification, Google Drive setup and installation. Called by launchd on a schedule, or by hand from Terminal." ), subcommands: [ - InstallLaunchd.self, + // Setup, in the order it is done. + InstallRclone.self, ConfigureRemote.self, + VMSetupCommand.self, + VMPrepareBundle.self, + InstallLaunchd.self, + // Running. + VMStatusCommand.self, + VMGuardCommand.self, + BackupHealthCommand.self, + PrepareShutdown.self, SetLimit.self, + // Building and the rest. InstallDependencies.self, BuildApp.self, MakeDmg.self, SetupSigningCert.self, - // The Google Drive layer - replaced the scripts in gdrive/. - InstallRclone.self, - InstallFuse.self, - MountDrive.self, - CreateImage.self, - AttachImage.self, Version.self, - DetachImage.self, - VerifyImage.self, - BufferGuard.self, - BackupHealthCommand.self, - PrepareShutdown.self, - RestartMount.self, - DriveStatus.self, - // The machine layout (Time Machine over SMB) - see vm/README.md. - VMSetupCommand.self, - VMPrepareBundle.self, - VMGuardCommand.self, - VMStatusCommand.self, ] ) } diff --git a/mac-app/Sources/CloudMachineAgent/DriveCommands.swift b/mac-app/Sources/CloudMachineAgent/DriveCommands.swift deleted file mode 100644 index 00c9410..0000000 --- a/mac-app/Sources/CloudMachineAgent/DriveCommands.swift +++ /dev/null @@ -1,605 +0,0 @@ -import ArgumentParser -import CloudMachineCore -import Foundation - -/// Subcommands of the Google Drive layer. They replace the scripts in -/// `gdrive/` - from now on launchd and the GUI call only this binary, not the -/// shell. - -// MARK: - Buffer - -struct MountDrive: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "mount-drive", - abstract: L10n.tr( - "Mounts Google Drive with a write buffer. Stays in the foreground (for launchd).")) - - func run() async throws { - // A mount left behind by an rclone that quit (27.10) would otherwise be - // "already mounted" forever - see `OrphanMount`. - await OrphanMount.clearBeforeMount() - if DriveBufferService.isMounted { - print(L10n.tr("Already mounted: %@", DriveBufferService.mountPoint.path)) - return - } - - // The FUSE-T uninstaller deletes the whole contents of /usr/local/lib under - // its path, including our link - we recreate it before checking anything. - FuseInstaller.ensureSystemLink() - - // Without FUSE, rclone exits immediately with "cgofuse: cannot find FUSE". - // The agent has KeepAlive, so it would retry over and over every 30 s and - // flood the log - better to stop right away and say what is missing. - let readiness = CMTooling.checkReadiness() - guard readiness.ready else { - for (what, how) in zip(readiness.missing, readiness.remedies) { - FileHandle.standardError.write(Data(L10n.tr("Missing: %@\n %@\n", what, how).utf8)) - } - throw ExitCode(1) - } - - await DriveBufferService.excludeBufferFromTimeMachine() - let args = try DriveBufferService.prepare() - - // Our own copy of the NFS server, if present - then a separate FUSE-T - // installation in the system is not needed. - if FileManager.default.isExecutableFile(atPath: CMTooling.bundledNfsServer.path) { - setenv("FUSE_NFSSRV_PATH", CMTooling.bundledNfsServer.path, 1) - } - - // We replace ourselves with rclone instead of supervising it: launchd is - // meant to watch the process that actually holds the mount, not a middleman. - let rclone = CMTooling.managedRclonePath.path - var argv: [UnsafeMutablePointer?] = ([rclone] + args).map { strdup($0) } - argv.append(nil) - execv(rclone, &argv) - - FileHandle.standardError.write(Data(L10n.tr("Could not start %@\n", rclone).utf8)) - throw ExitCode(1) - } -} - -// MARK: - Backup image - -struct CreateImage: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "create-image", - abstract: L10n.tr("Creates the backup image on Google Drive. One-off.")) - - @Option(name: .long, help: ArgumentHelp(L10n.tr("Declared size in GB (the image is sparse)."))) - var sizeGB: Int = 4000 - - func run() async throws { - let result = await BackupImageService.create(sizeGB: sizeGB) - print(result.message) - if result.succeeded { - print(L10n.tr("Next step: %@, then", "cloudmachine-agent attach-image")) - print(" sudo tmutil setdestination \(BackupImageService.targetPath.path)") - } else { - throw ExitCode(1) - } - } -} - -struct AttachImage: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "attach-image", - abstract: L10n.tr("Attaches the backup image as the Time Machine destination.")) - - func run() async throws { - // After `prepare-shutdown` the image stays detached until the restart - // (`ShutdownPrep`) - this agent runs every 900 s and would otherwise undo it. - if ShutdownPrep.holdsAttach() { - print( - L10n.tr( - "The Mac is prepared for a restart - not attaching the image. This is not an error: after the restart, or %@ min after the preparation, the image is attached again.", - "\(Int(ShutdownPrep.window / 60))")) - return - } - // Time Machine must not see the destination before the buffer is ready - - // otherwise it decides that the backup disk has disappeared. How long we - // wait and for what exactly - see `BufferReadiness`. - let ready = await BufferReadiness.wait( - sleep: { seconds in - try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) - }, - probe: { - BufferReadiness.isReady( - mounted: DriveBufferService.isMounted, - imageVisible: BackupImageService.exists) - }) - if !ready { - print( - L10n.tr( - "The buffer did not come up within %@ min - not attaching the image.", - "\(Int(BufferReadiness.defaultTimeout / 60))")) - print( - L10n.tr( - "Time Machine now has NO DESTINATION. Check: %@", "cloudmachine-agent drive-status")) - throw ExitCode(1) - } - var result = await BackupImageService.attach() - print(result.message) - - // A series of EAGAINs from `hdiutil` ends in a restart of the mount - see - // `AttachStall`. Its result replaces the attach's: a restart that brought - // the image up is a success of this tick. - if let restart = await AttachStall.afterAttach(result) { - print( - L10n.tr( - "The image kept being refused - the Google Drive mount was restarted: %@", restart.message - )) - result = restart - } - - // Three cases, not two. This command runs under launchd every 900 s, so - // its exit code ends up as an entry in `launchd-gdrive-attach.err.log` - - // the place a person looks when asking "is the backup working". "Image - // busy with a detach that is in progress right now" is NOT a failure: the - // next tick in 15 minutes will find the image free and attach it. Recording - // that as an error is the same pattern this code fights in the other - // direction - a normal state read as a failure, instead of a missing answer - // read as an answer. - // - // The TYPE of the result (`CMActionResult.Disposition`) decides, not the - // message text - matching on text breaks with the first rewording of the - // sentence and nobody notices. The `switch` is exhaustive, so a new case - // will not slip through here silently. - switch result.disposition { - case .ok: - break - case .skipped: - print(L10n.tr("This is not an error - the agent's next run will try again.")) - case .failed: - throw ExitCode(1) - } - } -} - -struct DetachImage: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "detach-image", - abstract: L10n.tr("Detaches the image and waits until everything reaches Google Drive.")) - - @Flag( - name: .long, - help: ArgumentHelp( - L10n.tr("Do not wait for the upload - RISKY, see BackupImageService.detach."))) - var noWait = false - - func run() async throws { - let result = await BackupImageService.detach(waitForUpload: !noWait) - print(result.message) - if !result.succeeded { throw ExitCode(1) } - } -} - -struct VerifyImage: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "verify-image", - abstract: - L10n.tr( - "Checks the image's consistency with fsck_apfs (hdiutil verify does not work on a sparsebundle)." - )) - - func run() async throws { - let result = await BackupImageService.verify() - print(result.message) - if !result.succeeded { throw ExitCode(1) } - } -} - -// MARK: - Buffer guard - -struct BufferGuard: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "buffer-guard", - abstract: - L10n.tr( - "Pauses Time Machine when the unsent backlog grows faster than the upload goes.")) - - // The default thresholds come FROM `Thresholds`, which derives them from the - // buffer size - we do NOT type them in here a second time by hand. - // - // The typed-in numbers (150/40/80) matched the derived ones only by - // accident, for a 100 GB buffer. launchd runs `buffer-guard` WITHOUT - // arguments, so it was exactly these literals that reached production - - // deriving the thresholds from `cacheSizeGB` was dead in practice, and the - // three tests guarding that derivation checked `Thresholds()` directly and - // passed without touching the path that actually runs. After a change to - // `cacheSizeGB` the thresholds would have drifted apart silently: the guard - // would either pause the backup nonstop, or never pause it at all. - // The thresholds refer to the UNSENT BACKLOG, not to the cache size - the - // old measure sat at the limit constantly, so the resume threshold was - // unreachable (one PAUSE and zero RESUMES in the whole log). See - // `Thresholds.init`. - @Option( - name: .long, - help: ArgumentHelp(L10n.tr("Above this many GB of unsent backlog we pause Time Machine."))) - var highGB: Int = BufferGuardService.Thresholds().highGB - - @Option( - name: .long, help: ArgumentHelp(L10n.tr("Below this many GB of unsent backlog we resume."))) - var lowGB: Int = BufferGuardService.Thresholds().lowGB - - @Option( - name: .long, - help: ArgumentHelp( - L10n.tr("Below this many GB free on disk we pause regardless of the buffer."))) - var minFreeGB: Int = BufferGuardService.Thresholds().minFreeGB - - @Option(name: .long, help: ArgumentHelp(L10n.tr("How often to check, in seconds."))) - var interval: Int = 30 - - func run() async throws { - let guardService = BufferGuardService( - thresholds: .init(highGB: highGB, lowGB: lowGB, minFreeGB: minFreeGB)) - CMLogger.log( - "Buffer guard: pause above \(highGB) GB backlog / resume below \(lowGB) GB / min. free on disk \(minFreeGB) GB" - ) - - // Beside the loop, not in it: a probe of the loop hanging on a dead mount - // must not hold up the one check that clears it. - Task.detached { await OrphanMount.watch() } - - // Forever: the guard has to outlive every backup, not just the first one. - while true { - // The mount runs for days and rclone never trims its own log; the - // guard is the one process that runs as long. A `stat` per tick. - DriveBufferService.trimLogIfLarge() - await guardService.step() - try? await Task.sleep(nanoseconds: UInt64(interval) * 1_000_000_000) - } - } -} - -// MARK: - Backup cycle watchdog - -struct BackupHealthCommand: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "backup-health", - abstract: - L10n.tr( - "Checks whether the hourly cycle STILL works (date of the last SUCCESSFUL backup) and reports failures." - )) - - @Option( - name: .long, - help: ArgumentHelp( - L10n.tr("After this many hours without a successful backup we consider the cycle broken."))) - var maxAgeHours: Double = BackupHealth.maxAgeHours - - @Flag( - name: .long, help: ArgumentHelp(L10n.tr("Only print the state, without a system notification.")) - ) - var quiet = false - - @Option( - name: .long, - help: ArgumentHelp( - L10n.tr( - "A different Time Machine preferences file - to test the watchdog on a known sample."))) - var preferences: String = BackupHealth.preferencesPath - - func run() async throws { - let report = await BackupHealth.currentReport( - maxAgeHours: maxAgeHours, preferencesFile: preferences) - - // The "watchdog ran" marker - BEFORE printing anything and before deciding - // the exit code, because a run that found a failure is just as much a run - // as one that found nothing. Without it, the only symptom of a dead or hung - // watchdog would be silence - and silence is the normal state here (see - // `WatchdogHeartbeat`). - WatchdogHeartbeat.record() - - // The watchdog runs every 30 minutes whether or not anyone opened the app, - // so it is also what brings back an agent launchd stopped being able to - // start (see `AgentRepair`) - most importantly the mount. - let repaired = await AgentRepair.repairBroken() - if !repaired.isEmpty { - print( - L10n.tr("Reloaded agents that could not start: %@", repaired.joined(separator: ", "))) - } - - if let lastSuccess = report.lastSuccess { - print(L10n.tr("Last successful backup: %@", BackupHealth.stamp(lastSuccess))) - } else { - print(L10n.tr("Last successful backup: NONE")) - } - if let lastAttempt = report.lastAttempt { - print(L10n.tr("Last attempt: %@", BackupHealth.stamp(lastAttempt))) - } - - // Deferred for the startup period - not a failure, but we do not hide them. - for problem in report.deferred { - print(L10n.tr("WAITING (system startup): %@", problem.summary)) - } - - // Not a backup failure, so not in `report` and not in the exit code - - // but a notification of its own, see `OpenPortAlert`. - let portOpen = DriveBufferService.legacyPortOpenNow() - if portOpen { - print( - L10n.tr( - "WARNING: remote control is OPEN on %@ - run restart-mount (or prepare-shutdown, then restart the Mac).", - DriveBufferService.legacyRCAddress)) - } - if !quiet { await OpenPortAlert.report(open: portOpen) } - - guard !report.healthy else { - print(L10n.tr("Backup cycle: OK")) - if !quiet { await HealthAlert.report(report) } - return - } - - for problem in report.problems { - print(L10n.tr("FAILURE: %@", problem.summary)) - print(L10n.tr(" %@", problem.detail)) - } - if !quiet { await HealthAlert.report(report) } - - // A non-zero exit code, so that launchd, `&&` in a script and a person - // looking at `echo $?` get the same signal as the text above. - throw ExitCode(1) - } -} - -// MARK: - Safe shutdown - -struct PrepareShutdown: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "prepare-shutdown", - abstract: L10n.tr( - "Prepares for a restart: pauses the backup, detaches the image and waits for the upload.")) - - func run() async throws { - let result = await ShutdownPrep.perform(progress: { print($0) }) - print(result.message) - guard result.succeeded else { - print(L10n.tr("Check the state: %@", "cloudmachine-agent drive-status")) - throw ExitCode(1) - } - } -} - -// MARK: - Restarting the mount - -struct RestartMount: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "restart-mount", - abstract: L10n.tr( - "Restarts the Google Drive mount on the private socket: detaches the image, waits for the upload, restarts the mount and attaches the image again." - )) - - func run() async throws { - print( - L10n.tr( - "Detaching the image and waiting for the upload, then restarting the mount. This can take several minutes; Time Machine has no destination meanwhile." - )) - let result = await MountRestart.perform() - print(result.message) - if !result.succeeded { throw ExitCode(1) } - } -} - -// MARK: - Status - -struct DriveStatus: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "drive-status", - abstract: L10n.tr("State of the buffer, the upload queue and Time Machine.")) - - func run() async throws { - let readiness = CMTooling.checkReadiness() - print( - L10n.tr( - "Tools: %@", - readiness.ready - ? "OK" : L10n.tr("missing: %@", readiness.missing.joined(separator: ", ")))) - let mounted = DriveBufferService.mountedState() - print(L10n.tr("Drive mount: %@", StatusLines.mounted(mounted))) - print( - L10n.tr( - "Remote control: %@", - StatusLines.remoteControl(DriveBufferService.rcTransport, mounted: mounted))) - print( - L10n.tr( - "Drive folder: %@", - "\(DriveBufferService.remoteName):\(DriveBufferService.remotePath)")) - // Since 26.09.2026 the readability probe has a time limit - which is why - // this tool does not hang on a dead mount (on 25.09.2026 it hung for over - // 25 s and had to be killed), but reports that there is no answer. - let image = await BackupImageService.attachmentReading() - print( - L10n.tr( - "Image attached: %@", - BackupImageService.describe(image.attachment, probeTimedOut: image.probeTimedOut))) - - // We read the queue BEFORE the buffer lines, because both of them use it. - // A second query to rclone would cost up to 60 s with a clogged buffer - // (see `DriveBufferService.queueStats`). - let queueStats = await DriveBufferService.queueStats() - - // Two lines, because these are TWO DIFFERENT quantities. A single line - // "Buffer: 103 GB of 100G" looked like an answer to "is the upload keeping - // up", and it was not: the cache size sits at the limit constantly. The - // buffer guard made decisions on that number and that is why it never - // resumed the backup even once. - print( - L10n.tr( - "Cache on disk: %@", - StatusLines.cacheSize( - BufferGuardService.cacheSizeGB(stats: queueStats), - limitGB: DriveBufferService.cacheSizeGB))) - print( - L10n.tr( - "To upload: %@", - StatusLines.backlog( - BufferGuardService.backlogGB(stats: queueStats), items: queueStats?.unsentItems))) - // NOT `\(BufferGuardService.freeGB()) GB` - that returns `Int?` since a - // missing measurement stopped pretending to be zero, and interpolating the - // optional value printed `Free on disk: Optional(427) GB`. The compiler - // only said so with a warning, so it stopped neither the build nor the - // tests. - print(L10n.tr("Free on disk: %@", StatusLines.freeDisk(BufferGuardService.freeGB()))) - - if let stats = queueStats { - print( - L10n.tr( - "Upload queue: %@ in progress, %@ queued, %@ errors", "\(stats.uploadsInProgress)", - "\(stats.uploadsQueued)", "\(stats.erroredFiles)")) - } else { - print(L10n.tr("Upload queue: (rc interface unreachable)")) - } - - let safe = await BackupImageService.safeToRebootNow() - print( - L10n.tr( - "Restart without asking: %@", - safe ? L10n.tr("YES - queue empty") : L10n.tr("NO - run prepare-shutdown first"))) - - // The same answer as on the card in the UI - one source, so that the CLI - // and the GUI cannot claim different things about the same state. - let upload = UploadState.from( - // `UploadState` has no "unknown whether mounted" state, and adding it - // would touch files outside my scope. `?? false` then gives `.mountDown` - // ("Upload is not working") - so it warns instead of reassuring, and the - // "Drive mount" line above already says outright that it is UNKNOWN. A - // false alarm is the right direction of error here. - mounted: mounted ?? false, - queueKnown: queueStats != nil, - queued: queueStats?.uploadsQueued ?? 0, - inProgress: queueStats?.uploadsInProgress ?? 0, - failedFiles: queueStats?.erroredFiles ?? 0, - bufferOutOfSpace: queueStats?.outOfSpace ?? false, - driveFull: DriveBufferService.hitStorageQuota(), - dailyQuotaExhausted: DriveBufferService.uploadStalled()) - print(L10n.tr("Upload: %@", upload.headline)) - if !upload.isNominal { - print(" \(upload.explanation.replacingOccurrences(of: "\n", with: " "))") - } - - if let mountPoint = await TimeMachineStatus.currentDestinationMountPoint() { - print(L10n.tr("TM destination: %@", mountPoint)) - } else { - print(L10n.tr("TM destination: none")) - } - let limit = MachineBudget.limitGB() - print( - L10n.tr( - "Space limit: %@", - MachineBudget.summary(limitGB: limit, usage: MachineBudget.storedUsage()))) - if let limit { - let target = BackupImageService.targetPath.path - let quota = await TimeMachineStatus.destinationQuotaGB(forMountPointContaining: target) - if !MachineBudget.quotaMatches(currentQuotaGB: quota, limitGB: limit), - let id = await TimeMachineStatus.destinationID(forMountPointContaining: target) - { - print( - L10n.tr( - " Time Machine quota not set to match - run: %@", - MachineBudget.setQuotaCommand(destinationID: id, limitGB: limit))) - } - } - if await TimeMachineStatus.isRunning(), let progress = await TimeMachineStatus.currentProgress() - { - let percent = (progress.percent ?? 0) * 100 - print( - L10n.tr( - "Backup: running, %@%% (%@)", String(format: "%.1f", percent), - progress.phase ?? "?")) - } else { - print(L10n.tr("Backup: not running")) - } - // Who watches the watchdog. Without this line "no alarm" meant both - // "the backup works" and "nobody checked" at once - see `WatchdogHeartbeat`. - print(L10n.tr("Backup watchdog: %@", StatusLines.watchdogRun(WatchdogHeartbeat.current()))) - let broken = await AgentRepair.brokenAgents() - print( - L10n.tr( - "Agents: %@", - broken.isEmpty - ? "OK" - : L10n.tr( - "CANNOT START: %@ - open CloudMachine or run backup-health to reload them", - broken.joined(separator: ", ")))) - - // At the very end and not aligned to the column - this is not another - // status line but something meant to interrupt the reading. Since recently - // `HealthAlert` does not close the case with a marker until the - // notification has been delivered, so an undelivered alarm no longer gets - // lost for 12 h - but without this block nobody would find out about it, - // because a system notification that failed is by definition not seen. - for line in StatusLines.undeliveredAlert(HealthAlert.lastDeliveryFailure()) { - print(line) - } - } -} - -// MARK: - FUSE installation - -struct InstallFuse: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "install-fuse", - abstract: L10n.tr( - "Pulls FUSE-T into CloudMachine, so there is no separate app in the system.")) - - func run() async throws { - let result = await FuseInstaller.install() - print(result.message) - if !result.succeeded { throw ExitCode(1) } - } -} - -// MARK: - rclone installation - -struct InstallRclone: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "install-rclone", - abstract: L10n.tr( - "Downloads the official rclone binary (the Homebrew one cannot mount).")) - - func run() async throws { - let result = await RcloneInstaller.install() - print(result.message) - if !result.succeeded { throw ExitCode(1) } - } -} - -// MARK: - Version - -/// Answers the question "is what runs the same as what is in the repository". -/// -/// `1.1.0` alone does not answer it - so we print the commit and the state of -/// the tree at build time, and with no bundle we say outright that it is a -/// build from the working tree, instead of making up a number. -struct Version: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "version", - abstract: L10n.tr( - "Prints the version, the build number and the commit this binary was built from.")) - - @Flag(name: .long, help: ArgumentHelp(L10n.tr("Only one line, without a description."))) - var short = false - - func run() async throws { - guard let version = AppVersionReader.current() else { - print(L10n.tr("Build from the working tree (outside a bundle) - no version data.")) - return - } - guard !short else { - print(version.summary) - return - } - print(L10n.tr("Version: %@", version.shortVersion)) - print(L10n.tr("Build: %@", version.build)) - print(L10n.tr("Commit: %@", version.commit)) - if version.dirty { - print("") - print( - L10n.tr( - "WARNING: built from a DIRTY tree - the binary contains code that\n is in no commit. The commit number does NOT describe\n what actually runs." - )) - } - } -} diff --git a/mac-app/Sources/CloudMachineAgent/MaintenanceCommands.swift b/mac-app/Sources/CloudMachineAgent/MaintenanceCommands.swift new file mode 100644 index 0000000..a64c210 --- /dev/null +++ b/mac-app/Sources/CloudMachineAgent/MaintenanceCommands.swift @@ -0,0 +1,166 @@ +import ArgumentParser +import CloudMachineCore +import Foundation + +/// The commands around the machine that are not `vm-*`: the watchdog, the +/// clean shutdown, the Mac's rclone and the version. + +// MARK: - Backup cycle watchdog + +struct BackupHealthCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "backup-health", + abstract: + L10n.tr( + "Checks whether the hourly cycle STILL works (date of the last SUCCESSFUL backup) and reports failures." + )) + + @Option( + name: .long, + help: ArgumentHelp( + L10n.tr("After this many hours without a successful backup we consider the cycle broken."))) + var maxAgeHours: Double = BackupHealth.maxAgeHours + + @Flag( + name: .long, help: ArgumentHelp(L10n.tr("Only print the state, without a system notification.")) + ) + var quiet = false + + @Option( + name: .long, + help: ArgumentHelp( + L10n.tr( + "A different Time Machine preferences file - to test the watchdog on a known sample."))) + var preferences: String = BackupHealth.preferencesPath + + func run() async throws { + let report = await BackupHealth.currentReport( + maxAgeHours: maxAgeHours, preferencesFile: preferences) + + // The "watchdog ran" marker - BEFORE printing anything and before deciding + // the exit code, because a run that found a failure is just as much a run + // as one that found nothing. Without it, the only symptom of a dead or hung + // watchdog would be silence - and silence is the normal state here (see + // `WatchdogHeartbeat`). + WatchdogHeartbeat.record() + + // The watchdog runs every 30 minutes whether or not anyone opened the app, + // so it is also what brings back an agent launchd stopped being able to + // start (see `AgentRepair`), and what clears the retired layout's agents. + await AgentRepair.removeRetiredAgents() + let repaired = await AgentRepair.repairBroken() + if !repaired.isEmpty { + print( + L10n.tr("Reloaded agents that could not start: %@", repaired.joined(separator: ", "))) + } + + if let lastSuccess = report.lastSuccess { + print(L10n.tr("Last successful backup: %@", BackupHealth.stamp(lastSuccess))) + } else { + print(L10n.tr("Last successful backup: NONE")) + } + if let lastAttempt = report.lastAttempt { + print(L10n.tr("Last attempt: %@", BackupHealth.stamp(lastAttempt))) + } + + // Deferred for the startup period - not a failure, but we do not hide them. + for problem in report.deferred { + print(L10n.tr("WAITING (system startup): %@", problem.summary)) + } + + guard !report.healthy else { + print(L10n.tr("Backup cycle: OK")) + if !quiet { await HealthAlert.report(report) } + return + } + + for problem in report.problems { + print(L10n.tr("FAILURE: %@", problem.summary)) + print(L10n.tr(" %@", problem.detail)) + } + if !quiet { await HealthAlert.report(report) } + + // A non-zero exit code, so that launchd, `&&` in a script and a person + // looking at `echo $?` get the same signal as the text above. + throw ExitCode(1) + } +} + +// MARK: - Safe shutdown + +struct PrepareShutdown: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "prepare-shutdown", + abstract: L10n.tr( + "Prepares for a restart: stops the backup, waits a while for the upload, then stops the machine cleanly." + )) + + @Option( + name: .long, + help: ArgumentHelp(L10n.tr("At most this many minutes of waiting for the upload."))) + var waitMinutes: Int = Int(VMShutdown.drainTimeout / 60) + + func run() async throws { + let result = await VMShutdown.perform( + drainTimeout: TimeInterval(max(0, waitMinutes) * 60), progress: { print($0) }) + print(result.message) + guard result.succeeded else { + print(L10n.tr("Check the state: %@", "cloudmachine-agent vm-status")) + throw ExitCode(1) + } + } +} + +// MARK: - rclone installation + +struct InstallRclone: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "install-rclone", + abstract: L10n.tr( + "Downloads the official rclone binary into ~/.cloudmachine/bin (Google sign-in and the machine's rclone.conf)." + )) + + func run() async throws { + let result = await RcloneInstaller.install() + print(result.message) + if !result.succeeded { throw ExitCode(1) } + } +} + +// MARK: - Version + +/// Answers the question "is what runs the same as what is in the repository". +/// +/// `1.1.0` alone does not answer it - so we print the commit and the state of +/// the tree at build time, and with no bundle we say outright that it is a +/// build from the working tree, instead of making up a number. +struct Version: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "version", + abstract: L10n.tr( + "Prints the version, the build number and the commit this binary was built from.")) + + @Flag(name: .long, help: ArgumentHelp(L10n.tr("Only one line, without a description."))) + var short = false + + func run() async throws { + guard let version = AppVersionReader.current() else { + print(L10n.tr("Build from the working tree (outside a bundle) - no version data.")) + return + } + guard !short else { + print(version.summary) + return + } + print(L10n.tr("Version: %@", version.shortVersion)) + print(L10n.tr("Build: %@", version.build)) + print(L10n.tr("Commit: %@", version.commit)) + if version.dirty { + print("") + print( + L10n.tr( + "WARNING: built from a DIRTY tree - the binary contains code that\n is in no commit. The commit number does NOT describe\n what actually runs." + )) + } + } +} diff --git a/mac-app/Sources/CloudMachineAgent/SetupCommands.swift b/mac-app/Sources/CloudMachineAgent/SetupCommands.swift index 4ab3001..ec44c80 100644 --- a/mac-app/Sources/CloudMachineAgent/SetupCommands.swift +++ b/mac-app/Sources/CloudMachineAgent/SetupCommands.swift @@ -7,7 +7,8 @@ struct InstallLaunchd: AsyncParsableCommand { commandName: "install-launchd", abstract: L10n.tr( - "Generates and installs the launchd agents (Drive buffer, image attach, buffer guard).") + "Generates and installs the launchd agents (app, backup watchdog, machine start, machine guard)." + ) ) func run() async throws { @@ -38,7 +39,7 @@ struct ConfigureRemote: AsyncParsableCommand { help: ArgumentHelp( L10n.tr( - "With --replace-existing: confirms that the new sign-in (scope 'drive.file') may not see the existing backup. The new token is still checked against the image first." + "With --replace-existing: confirms that the new sign-in (scope 'drive.file') may not see the existing backup. The new token is still checked against the backup first." )) ) var acceptScopeChange = false @@ -66,7 +67,7 @@ struct InstallDependencies: AsyncParsableCommand { commandName: "install-dependencies", abstract: L10n.tr( - "Installs rclone through Homebrew - WARNING: this build CANNOT mount, see install-rclone." + "Installs rclone through Homebrew - CloudMachine itself uses the one from install-rclone." )) func run() async throws { @@ -93,8 +94,8 @@ struct SetLimit: AsyncParsableCommand { throw ExitCode.failure } print(L10n.tr("Limit for this Mac: %@ GB.", "\(gb)")) - if let id = await TimeMachineStatus.destinationID( - forMountPointContaining: BackupImageService.targetPath.path) + if let destinations = await TimeMachineStatus.destinationsReading(), + let id = VMStatus.machineDestination(in: destinations, address: nil)?.id { print(L10n.tr("Now set the Time Machine quota (needs an administrator password):")) print(" " + MachineBudget.setQuotaCommand(destinationID: id, limitGB: gb)) diff --git a/mac-app/Sources/CloudMachineAgent/VMCommands.swift b/mac-app/Sources/CloudMachineAgent/VMCommands.swift index 7ebc42f..bfa98d2 100644 --- a/mac-app/Sources/CloudMachineAgent/VMCommands.swift +++ b/mac-app/Sources/CloudMachineAgent/VMCommands.swift @@ -2,8 +2,7 @@ import ArgumentParser import CloudMachineCore import Foundation -// The machine layout (Time Machine over SMB to a Lima VM) - see vm/README.md. -// It runs beside the attached-image commands; none of these touch the image. +// The machine (Time Machine over SMB to a Lima VM) - see vm/README.md. struct VMSetupCommand: AsyncParsableCommand { static let configuration = CommandConfiguration( @@ -20,7 +19,7 @@ struct VMSetupCommand: AsyncParsableCommand { @Flag( name: .long, - help: ArgumentHelp(L10n.tr("Also install the vm-start and vm-guard launchd agents."))) + help: ArgumentHelp(L10n.tr("Also install the launchd agents (same as install-launchd)."))) var installAgents = false func run() async throws { @@ -28,7 +27,7 @@ struct VMSetupCommand: AsyncParsableCommand { for line in result.messages { print(line) } guard result.succeeded else { throw ExitCode.failure } if installAgents { - let agents = await LaunchdInstaller.installVMAgents() + let agents = await LaunchdInstaller.install() print(agents.message) if !agents.succeeded { throw ExitCode.failure } } @@ -104,6 +103,39 @@ struct VMStatusCommand: AsyncParsableCommand { func run() async throws { let (lines, healthy) = await VMStatus.lines() for line in lines { print(line) } + print(L10n.tr("Drive folder: %@", VMLayout.remote)) + + let limit = MachineBudget.limitGB() + print( + L10n.tr( + "Space limit: %@", + MachineBudget.summary(limitGB: limit, usage: MachineBudget.storedUsage()))) + if let limit, let destinations = await TimeMachineStatus.destinationsReading(), + let ours = VMStatus.machineDestination(in: destinations, address: nil), let id = ours.id, + !MachineBudget.quotaMatches(currentQuotaGB: ours.quotaGB, limitGB: limit) + { + print( + L10n.tr( + " Time Machine quota not set to match - run: %@", + MachineBudget.setQuotaCommand(destinationID: id, limitGB: limit))) + } + + // Who watches the watchdog: without this line "no alarm" meant both "the + // backup works" and "nobody checked" - see `WatchdogHeartbeat`. + print(L10n.tr("Backup watchdog: %@", StatusLines.watchdogRun(WatchdogHeartbeat.current()))) + let broken = await AgentRepair.brokenAgents() + print( + L10n.tr( + "Agents: %@", + broken.isEmpty + ? "OK" + : L10n.tr( + "CANNOT START: %@ - open CloudMachine or run backup-health to reload them", + broken.joined(separator: ", ")))) + // At the very end: an alarm whose notification failed is seen ONLY here. + for line in StatusLines.undeliveredAlert(HealthAlert.lastDeliveryFailure()) { + print(line) + } if !healthy { throw ExitCode.failure } } } diff --git a/mac-app/Sources/CloudMachineApp/Models/AppStatus.swift b/mac-app/Sources/CloudMachineApp/Models/AppStatus.swift index 1b51d7c..4fb1fcd 100644 --- a/mac-app/Sources/CloudMachineApp/Models/AppStatus.swift +++ b/mac-app/Sources/CloudMachineApp/Models/AppStatus.swift @@ -10,128 +10,147 @@ enum DependencyState: Equatable { case ready } +/// The Lima machine itself, as `limactl list` reports it. +enum MachineState: Equatable { + case unknown + /// `limactl` did not answer (or is not installed) - nothing is known. + case noAnswer + /// There is no machine `cmsmb` yet: `vm-setup` creates it. + case notCreated + /// The machine exists but does not run; Lima's word for it (`Stopped`, ...). + case notRunning(String) + case running + + /// Translates the `limactl list` answer. Pure, so a test can show that a + /// missing answer does not pass for a stopped machine. + static func from(instanceStatus: String??) -> MachineState { + switch instanceStatus { + case .none: return .noAnswer + case .some(.none): return .notCreated + case .some(.some("Running")): return .running + case .some(.some(let other)): return .notRunning(other) + } + } +} + enum TimeMachineState: Equatable { case unknown - /// Time Machine does not point at our image - there is effectively no backup. + /// No Time Machine destination is the machine's share - there is effectively no backup. case notRegistered /// `tmutil` DID NOT ANSWER within the time limit, so we know nothing about the destination. /// - /// A separate state for the same reason as `TimeMachineStatus.DestinationReading.noAnswer` - /// and `BufferStatus.queueKnown`: a missing answer has no right to pretend to be a result. - /// Until 25.09.2026 the panel showed "Time Machine does not point to - /// CloudMachine" here - a sentence that sounds true and is false, sending - /// a person off to register the destination again while the destination is - /// intact and it was the read that hung (`tmutil destinationinfo` reaches - /// into the Google Drive mount). + /// A separate state, because a missing answer has no right to pretend to be a + /// result: "Time Machine does not point to CloudMachine" sends a person off to + /// register a destination that may be intact, while it was the read that hung. case noAnswer - case registered(mountPoint: String) + case registered(url: String) } extension TimeMachineState { - /// Translates the `tmutil` answer into a panel state. - /// - /// Extracted from `CloudMachineController.refreshTimeMachine()` and pure, + /// Translates the `tmutil destinationinfo` answer into a panel state. Pure, /// so a test can show that THREE answers give THREE states. - /// Previously the controller asked `currentDestinationMountPoint()`, which returns - /// `nil` both when there is no destination and when there is no answer - so both paths - /// ended in the same `.notRegistered`. The `backup-health` watchdog has - /// distinguished them since 23.09.2026 (`destinationReading()`); the panel did not. - static func from(_ reading: TimeMachineStatus.DestinationReading, target: String) + /// + /// `address` is the machine's current address; without it any CloudMachine + /// share counts (see `VMStatus.isMachineShare`). + static func from(_ destinations: [TimeMachineStatus.Destination]?, address: String?) -> TimeMachineState { - switch reading { - case .mountPoint(let path): - return path == target ? .registered(mountPoint: path) : .notRegistered - case .none: return .notRegistered - case .noAnswer: return .noAnswer - } + guard let destinations else { return .noAnswer } + guard let ours = VMStatus.machineDestination(in: destinations, address: address), + let url = ours.url + else { return .notRegistered } + return .registered(url: url) } } -/// State of the buffer between Time Machine and Google Drive. -struct BufferStatus: Equatable { - var mounted: Bool = false - var imageAttached: Bool = false - var sizeGB: Int = 0 - /// `nil` = there was NO measurement (statfs failed), not "zero gigabytes" - - /// see `BufferGuardService.freeGB()`. The same distinction as - /// `queueKnown` below. - var freeDiskGB: Int? - /// How many files are waiting to be uploaded. This number matters more than the - /// buffer size: if it grows and does not return to zero between backups, uploading - /// is not keeping up with writing. - var uploadsQueued: Int = 0 - var uploadsInProgress: Int = 0 - /// Whether the counters above come from a reading at all. - /// - /// `false` by default, and that matters more than it looks: a freshly created - /// `BufferStatus` has nothing but zeros, which are not a measurement. A default of `true` - /// would mean "empty queue" and the menu bar would light up green before - /// anything had been checked. - var queueKnown: Bool = false - var erroredFiles: Int = 0 - /// There is no space left on Google Drive. It will NOT pass on its own. - var driveFull: Bool = false - /// The Google daily UPLOAD limit (750 GB) is exhausted. It passes on its own. - /// - /// Kept separate from `driveFull`, because these are two different situations with - /// the same symptom: one means "wait", the other "free up space". Previously they were - /// a single field and the interface could not tell them apart. - var dailyQuotaExhausted: Bool = false - /// rclone has nowhere to put data - the buffer is full of nothing but unsent files. - var outOfSpace: Bool = false +/// What the machine said about itself: services, the upload buffer, the image. +struct MachineStatus: Equatable { + var state: MachineState = .unknown + var address: String? + /// `systemctl is-active` per unit; empty = not read. + var services: [String: String] = [:] + /// `nil` = no reading (the machine is down or rclone did not answer). + var reading: VMLayout.Reading? + var punchfsOpenFiles: Int? + var punchfsFileLimit: Int? + var bundlePrepared: Bool? + var imageCount: Int? - var draining: Bool { uploadsInProgress > 0 || uploadsQueued > 0 } + var isRunning: Bool { state == .running } - /// The "Cloud sync queue" row. - /// - /// An empty queue is "everything uploaded" only when rclone abandoned - /// nothing on the way: an abandoned band drops out of the queue exactly like - /// an uploaded one (see `DriveBufferService.QueueStats.isQuiet`). Until - /// 09.10.2026 the row then said "Everything uploaded" - in red, above a row - /// counting the errors. + /// `nil` = not known (no answer from the machine). + var servicesHealthy: Bool? { + guard isRunning, !services.isEmpty else { return nil } + return VMStatus.servicesHealthy(services) + } + + /// Whether the counters below come from a reading at all. A fresh + /// `MachineStatus` has nothing but zeros, which are not a measurement. + var queueKnown: Bool { reading?.buffer.unsentBytes != nil } + + var unsentGiB: Int? { reading?.buffer.unsentBytes.map { Int($0 / VMGuard.gib) } } + var cacheGiB: Int? { reading.map { Int($0.buffer.bytesUsed / VMGuard.gib) } } + var freeGiB: Int? { reading.map { Int($0.freeBytes / VMGuard.gib) } } + var erroredFiles: Int { reading?.buffer.erroredFiles ?? 0 } + var uploadsQueued: Int { reading?.buffer.uploadsQueued ?? 0 } + var uploadsInProgress: Int { reading?.buffer.uploadsInProgress ?? 0 } + + var draining: Bool { (reading?.buffer.unsentBytes ?? 0) > 0 } + + /// Time Machine has created its image, and it still has 875 MB bands on the + /// machine's local disk: `vm-prepare-bundle` is due. + var bundleNeedsPreparing: Bool { + bundlePrepared == false && (imageCount ?? 0) > 0 + } + + var punchfsNearLimit: Bool { + guard let open = punchfsOpenFiles, let limit = punchfsFileLimit else { return false } + return Double(open) > Double(limit) * VMStatus.fileWarningRatio + } + + /// The "To upload" row. An empty queue is "everything uploaded" only when + /// rclone abandoned nothing on the way: an abandoned band drops out of the + /// queue exactly like an uploaded one. var queueSummary: String { - guard queueKnown else { return L10n.tr("not read") } + guard let unsent = unsentGiB else { return L10n.tr("not read") } if draining { return L10n.tr( - "%@ in progress, %@ queued", "\(uploadsInProgress)", "\(uploadsQueued)") + "%@ GiB in %@ bands, %@ uploading", "\(unsent)", "\(uploadsQueued)", + "\(uploadsInProgress)") } if erroredFiles > 0 { - return L10n.tr("%@ fragments abandoned - only on this Mac", "\(erroredFiles)") + return L10n.tr("%@ fragments abandoned - only in the machine", "\(erroredFiles)") } return L10n.tr("Everything uploaded") } /// The single source of truth on whether the backup reaches the Drive - and why not. + /// + /// `driveFull` and `dailyQuotaExhausted` are not read here: the rclone log + /// that tells them lives in the machine. A stalled upload still shows - the + /// backlog grows and the guard pauses Time Machine. var uploadState: UploadState { UploadState.from( - mounted: mounted, + mounted: isRunning && services["rclone-tm"] == "active", queueKnown: queueKnown, queued: uploadsQueued, inProgress: uploadsInProgress, failedFiles: erroredFiles, - bufferOutOfSpace: outOfSpace, - driveFull: driveFull, - dailyQuotaExhausted: dailyQuotaExhausted) + bufferOutOfSpace: reading?.buffer.outOfSpace ?? false, + driveFull: false, + dailyQuotaExhausted: false) } } /// Whether the backup cycle is STILL working - measured by the date of the last SUCCESSFUL backup. /// -/// Until 23.09.2026 the interface did not ask this question even once: `grep -rn -/// "BackupHealth" Sources/CloudMachineApp/` returned not a single hit. -/// The panel computed health solely from the state of the DEVICES - the mount, the image, the -/// Time Machine destination, the queue - that is, from the MOMENTARY state. The failure described -/// in the `BackupHealth` header as the most dangerous one looks exactly the other way round: -/// everything mounted, the image attached, the queue empty, and Time Machine has not -/// finished a backup for two days. The panel then showed "Healthy / Ready". +/// The state of the machine at this moment can be flawless while Time Machine +/// has not finished a backup for two days; this is the one measure that moves +/// only on success. struct BackupCycleStatus: Equatable { - /// Whether the Time Machine preferences could be read at all. - /// - /// `false` by default, and that matters more than it looks - just as with - /// `queueKnown`: a freshly created state is not a measurement, and missing Full - /// Disk Access (the most common reason the preferences file is unreadable) - /// must not pass for the absence of a problem. + /// Whether the Time Machine preferences could be read at all. `false` by + /// default: a freshly created state is not a measurement, and missing Full + /// Disk Access must not pass for the absence of a problem. var known: Bool = false /// Date of the last COMPLETED backup. `nil` = there is not a single one. var lastSuccess: Date? @@ -144,11 +163,8 @@ struct BackupCycleStatus: Equatable { lastSuccess.map { now.timeIntervalSince($0) } } - /// Whether the last SUCCESSFUL backup is recent enough. - /// - /// No reading and no backup both give `false` - either one means that nobody - /// has confirmed the backup works, and a green badge is exactly such a - /// confirmation. + /// Whether the last SUCCESSFUL backup is recent enough. No reading and no + /// backup both give `false` - nobody has confirmed the backup works. func isFresh(now: Date = Date(), maxAgeHours: Double = BackupHealth.maxAgeHours) -> Bool { guard known, let age = age(now: now) else { return false } return age <= maxAgeHours * 3600 @@ -186,23 +202,15 @@ struct BackupProgressInfo: Equatable { final class AppStatus: ObservableObject { @Published var dependencyState: DependencyState = .unknown @Published var remoteConfigured: Bool = false - @Published var buffer = BufferStatus() + @Published var machine = MachineStatus() /// The answer to "when was a BACKUP last made" - the only measure /// that moves only on success. @Published var backupCycle = BackupCycleStatus() @Published var timeMachineState: TimeMachineState = .unknown /// When the `backup-health` watchdog last RAN. `nil` = the panel has not - /// asked yet (not: "has never run" - that is a separate state, `.never`). - /// - /// The panel shows this for the same reason it shows the age of the last - /// backup: the watchdog runs with `StartInterval 1800` and without `KeepAlive`, so - /// when unloaded or hung it gives no symptom other than silence - and silence - /// is the normal state here. - /// - /// DELIBERATELY not part of `healthy`: the panel computes backup freshness ITSELF, from the - /// same preferences file the watchdog computes it from. A dead watchdog therefore does not - /// mean the backup is not working - it means nobody reports a failure, - /// and that is a different failure with its own red row. + /// asked yet. DELIBERATELY not part of `healthy`: a dead watchdog does not + /// mean the backup is not working - it means nobody reports a failure, and + /// that is a different failure with its own red row. @Published var watchdog: WatchdogHeartbeat.Freshness? @Published var backupProgress: BackupProgressInfo? @Published var lastAction: LastRunResult? @@ -217,30 +225,21 @@ final class AppStatus: ObservableObject { @Published var budgetUsage: MachineBudget.Usage? @Published var timeMachineQuotaGB: Double? @Published var timeMachineDestinationID: String? - /// Whether the mount agent is loaded in launchd; `nil` until asked. + /// Whether the backup watchdog agent is loaded in launchd; `nil` until asked. @Published var agentsInstalled: Bool? - /// Whether this Mac's image exists on the mounted Drive; `nil` while the - /// Drive is not mounted, because then nobody can know. - @Published var imageExists: Bool? - /// The Drive mount still takes commands on the open TCP port - a mount from - /// before the private socket that survived the upgrade (see - /// `DriveBufferService.legacyPortOpen`). Until 11.10.2026 only one line of - /// `drive-status` said so; the window was green. + /// rclone's remote control in the machine answers on the Mac's + /// 127.0.0.1:5572 (Lima's port forwarding) - any web page can send it + /// commands. See `VMStatus.remoteControlOpenOnMac`. @Published var remoteControlOpen: Bool = false - /// The Mac was prepared for a restart (`ShutdownPrep`): the image is - /// detached ON PURPOSE until the restart. - @Published var preparedForRestart: Bool = false @Published var isBusy: Bool = false @Published var busyLabel: String = "" @Published var errorMessage: String? - /// When the state was last read successfully. Shown in the interface, because - /// a frozen view looks exactly like a failure - and those are two different things - /// that the user has to tell apart without looking into the logs. + /// When the state was last read successfully. A frozen view looks exactly + /// like a failure - and those are two different things. @Published var lastRefresh: Date? /// Whether the backup watchdog is RUNNING. `false` also when the panel has not - /// asked yet - unchecked has no right to show green, just like - /// `queueKnown` and `BackupCycleStatus.known`. + /// asked yet - unchecked has no right to show green. var watchdogRunning: Bool { if case .fresh = watchdog { return true } return false @@ -252,38 +251,33 @@ final class AppStatus: ObservableObject { return L10n.tr("Missing: %@", what.joined(separator: ", ")) } if !remoteConfigured { return L10n.tr("Google Drive not connected") } - if !buffer.mounted { return L10n.tr("Buffer is not working") } - if !buffer.imageAttached { - return detachedForRestart - ? L10n.tr("Ready for restart - the image is detached on purpose") - : L10n.tr("Backup image not attached") + switch machine.state { + case .notCreated: return L10n.tr("The CloudMachine machine is not set up yet") + case .notRunning: return L10n.tr("The CloudMachine machine is stopped") + case .noAnswer: return L10n.tr("The CloudMachine machine does not answer") + case .unknown, .running: break + } + if machine.servicesHealthy == false { + return L10n.tr("A service in the CloudMachine machine is not running") } if case .notRegistered = timeMachineState { return L10n.tr("Time Machine does not point to CloudMachine") } - // No answer from tmutil MUST sound different from a changed destination: it is - // the first sentence a person reads, and it decides what they will do. - // "Does not point" tells them to register the destination again - a needless and misleading - // step when the destination is intact and it was the read that hung. + // No answer from tmutil MUST sound different from a changed destination: + // "does not point" tells a person to register the destination again. if case .noAnswer = timeMachineState { return L10n.tr( "UNKNOWN whether Time Machine points to CloudMachine - tmutil did not answer") } - // ONE source speaks about uploading - otherwise the menu bar and the status card could - // claim different things. Files that rclone did not upload exist ONLY - // on this Mac, which is exactly where the backup has no right to be the only - // copy; that is precisely why `UploadState` puts them ahead of the daily limit. - let upload = buffer.uploadState + // ONE source speaks about uploading - otherwise the menu bar and the status + // card could claim different things. + let upload = machine.uploadState if !upload.isNominal { return upload.headline } // Backups work in this state, but any web page can delete them - so it // comes before every sentence that sounds like "all is well". - if remoteControlOpen { - return L10n.tr("Drive remote control is open - restart the mount") - } + if remoteControlOpen { return L10n.tr("rclone's remote control is open on this Mac") } if backupProgress != nil { return L10n.tr("Backup in progress") } - // The state of the devices can be flawless while there has been no backup for two days. - // This sentence must come BEFORE "Ready", otherwise the headline contradicts the - // badge next to it (healthy = false, and the text says "Ready"). + // The machine can be flawless while there has been no backup for two days. if !backupCycle.isFresh() { guard backupCycle.known else { return L10n.tr("Unknown when the last backup was made") } guard let age = backupCycle.age() else { @@ -295,51 +289,51 @@ final class AppStatus: ObservableObject { return L10n.tr("Ready") } - /// Whether the state is really good. - /// - /// WARNING: `erroredFiles` and `outOfSpace` MUST be here. Without them the menu bar - /// showed a green badge and "Ready" while some of the image's bands never - /// reached the Drive - and such a backup may not open. Broken - /// looked exactly the same as working. - /// - /// SECOND WARNING, from 23.09.2026: `backupCycle` MUST be here for the same - /// reason. All the other conditions describe the state of the DEVICES at this moment, - /// and each of them can be met when no backup has been made for two days. - /// The green badge is meant to say "the data is safe", and that follows - /// solely from a backup having BEEN MADE - not from the disk being attached. - /// - /// THIRD, from 11.10.2026: `remoteControlOpen`. A backup that any web page - /// can delete is not safe, however fresh it is. - /// A detached image that is not a failure: the preparation for a restart - /// detached it, and the agents attach it again after the restart. - var detachedForRestart: Bool { preparedForRestart && !buffer.imageAttached } - + /// Whether the state is really good: every device AND a fresh completed + /// backup AND no open remote control. The green badge says "the data is + /// safe", and that follows solely from a backup having BEEN MADE. var healthy: Bool { - guard case .ready = dependencyState, remoteConfigured, buffer.mounted, buffer.imageAttached, - case .registered = timeMachineState, buffer.uploadState.isNominal, - backupCycle.isFresh(), !remoteControlOpen + guard case .ready = dependencyState, remoteConfigured, machine.isRunning, + machine.servicesHealthy == true, case .registered = timeMachineState, + machine.uploadState.isNominal, backupCycle.isFresh(), !remoteControlOpen else { return false } return true } - /// Whether "Restart the mount" may be offered: the port is open, nothing - /// else is running, and Time Machine is not backing up. `MountRestart` - /// checks the backup itself again - this only keeps the button honest. - var canRestartMount: Bool { - remoteControlOpen && !isBusy && backupProgress == nil - } - /// Whether "Back up now" can do anything: the backup has somewhere to go. - /// - /// NOT `healthy`, which until 09.10.2026 decided this too. `healthy` also - /// asks for a fresh backup - so after two days without one, with every - /// device in place, the only button that fixes that was grey. Likewise with - /// an unread queue or the Google daily limit: Time Machine still writes into - /// the buffer then. Only what makes `tmutil startbackup` pointless blocks it. + /// NOT `healthy` - after two days without a backup, the button that fixes + /// that must not be grey. var canStartBackup: Bool { - guard case .ready = dependencyState, remoteConfigured, buffer.mounted, buffer.imageAttached, + guard remoteConfigured, machine.isRunning, machine.servicesHealthy == true, case .registered = timeMachineState, !isBusy else { return false } return true } + + /// Whether "Start the machine" makes sense. + var canStartMachine: Bool { + if case .notRunning = machine.state { return !isBusy } + return false + } +} + +extension AppStatus { + /// The verdict's colour, shared by the menu bar and the window: green only + /// for a really good state, red for what needs a person, amber for the rest + /// (unknown, not checked yet, passes by itself). + var overallTone: StatusTone { + if healthy { return .success } + if case .missing = dependencyState { return .danger } + if !remoteConfigured { return .danger } + switch machine.state { + case .notCreated, .notRunning, .noAnswer: return .danger + case .unknown, .running: break + } + if machine.servicesHealthy == false { return .danger } + if case .notRegistered = timeMachineState { return .danger } + if machine.uploadState.needsAttention { return .danger } + if remoteControlOpen { return .danger } + if backupCycle.known && !backupCycle.isFresh() { return .danger } + return .warning + } } diff --git a/mac-app/Sources/CloudMachineApp/Models/SetupPlan.swift b/mac-app/Sources/CloudMachineApp/Models/SetupPlan.swift index 96f0c29..52880e8 100644 --- a/mac-app/Sources/CloudMachineApp/Models/SetupPlan.swift +++ b/mac-app/Sources/CloudMachineApp/Models/SetupPlan.swift @@ -6,22 +6,23 @@ import Foundation /// /// Installed from Homebrew, the app is the first thing a person opens, so the /// setup lives here: each step either has a button that does it, or - for the -/// two that cannot run from the app - a command to copy. Google sign-in needs a -/// browser round trip that `configure-remote` waits on in a terminal, and -/// pointing Time Machine at a disk needs `sudo`. +/// ones that cannot run from the app - a command to copy. Google sign-in needs +/// a browser round trip that `configure-remote` waits on in a terminal, +/// installing Lima is Homebrew's job, and pointing Time Machine at the machine +/// needs `sudo`. /// /// A pure function of what the controller measured, so the order and the /// conditions are tested without a GUI. `nil` inputs mean "not measured -/// yet" and never produce a step: telling someone to create an image that +/// yet" and never produce a step: telling someone to set up a machine that /// already exists is worse than showing the step a few seconds late. struct SetupStep: Equatable { enum Action: Equatable { case installRclone - case installFuse + case setUpMachine + case startMachine case grantFullDiskAccess case installAgents - case createImage - case attachImage + case prepareBundle } let title: String @@ -31,21 +32,27 @@ struct SetupStep: Equatable { /// The step where this Mac's Drive folder is chosen: the card shows a name /// field and puts `--folder NAME` into the command. var choosesFolder = false + /// What the card shows instead of `command`, when `command` holds a secret + /// (the SMB password): the Copy button still copies `command`. + var displayedCommand: String? } enum SetupPlan { struct Inputs: Equatable { var hasRclone: Bool - var hasFuse: Bool var remoteConfigured: Bool + var hasLima: Bool + /// `nil` = Lima did not answer. + var machineExists: Bool? + var machineRunning: Bool var hasFullDiskAccess: Bool var agentsInstalled: Bool? - var mounted: Bool - var imageExists: Bool? - var imageAttached: Bool var timeMachineNotPointingHere: Bool + var bundleNeedsPreparing: Bool var connectCommand: String - var setDestinationCommand: String + /// `nil` while the machine has no address or no stored password. + var setDestinationCommand: String? + var maskedSetDestinationCommand: String? } /// `configure-remote`, with `--folder` only for a valid name - an invalid @@ -56,20 +63,18 @@ enum SetupPlan { return "\(base) --folder \(folder)" } + static let installLimaCommand = "brew install lima" + static func steps(_ input: Inputs) -> [SetupStep] { var steps: [SetupStep] = [] if !input.hasRclone { steps.append( SetupStep( - title: L10n.tr("Install rclone (the official build, which can mount)"), command: nil, - action: .installRclone)) - } - if !input.hasFuse { - steps.append( - SetupStep(title: L10n.tr("Install FUSE-T"), command: nil, action: .installFuse)) + title: L10n.tr("Install rclone (signs in to Google Drive for the machine)"), + command: nil, action: .installRclone)) } - // Before anything that mounts: the Drive folder of this Mac is chosen - // here, and a mount started earlier would use the legacy name. + // The Drive folder of this Mac is chosen here, before the machine is + // created: the machine takes its folder from it. if !input.remoteConfigured { steps.append( SetupStep( @@ -77,6 +82,28 @@ enum SetupPlan { command: input.connectCommand, action: nil, choosesFolder: true)) return steps } + if !input.hasLima { + steps.append( + SetupStep( + title: L10n.tr("Install Lima, which runs the CloudMachine machine: run this in Terminal"), + command: installLimaCommand, action: nil)) + return steps + } + if input.machineExists == false { + steps.append( + SetupStep( + title: L10n.tr( + "Set up the CloudMachine machine (the first time it downloads Ubuntu - several minutes)" + ), + command: nil, action: .setUpMachine)) + return steps + } + if input.machineExists == true, !input.machineRunning { + steps.append( + SetupStep( + title: L10n.tr("Start the CloudMachine machine"), command: nil, action: .startMachine)) + return steps + } if !input.hasFullDiskAccess { steps.append( SetupStep( @@ -87,28 +114,39 @@ enum SetupPlan { if input.agentsInstalled == false { steps.append( SetupStep( - title: L10n.tr("Install the background agents (mount, image attach, watchdogs)"), + title: L10n.tr( + "Install the background agents (machine start, buffer guard, backup watchdog)"), command: nil, action: .installAgents)) return steps } - guard input.mounted else { return steps } - if input.imageExists == false { - steps.append( - SetupStep( - title: L10n.tr("Create the backup image on Google Drive"), command: nil, - action: .createImage)) + guard input.machineRunning else { return steps } + if input.timeMachineNotPointingHere { + if let command = input.setDestinationCommand { + steps.append( + SetupStep( + title: L10n.tr( + "Point Time Machine at the machine: run this in Terminal (needs sudo; Copy includes the password)" + ), + command: command, action: nil, + displayedCommand: input.maskedSetDestinationCommand)) + } else { + // No password file or no address: `vm-setup` writes the first and + // reads the second, and prints the command itself. + steps.append( + SetupStep( + title: L10n.tr( + "Set up the CloudMachine machine again, so it prints the Time Machine command"), + command: nil, action: .setUpMachine)) + } return steps } - if input.imageExists == true, !input.imageAttached { - steps.append( - SetupStep(title: L10n.tr("Attach the backup image"), command: nil, action: .attachImage)) - return steps - } - if input.imageAttached, input.timeMachineNotPointingHere { + if input.bundleNeedsPreparing { steps.append( SetupStep( - title: L10n.tr("Point Time Machine at CloudMachine: run this in Terminal (needs sudo)"), - command: input.setDestinationCommand, action: nil)) + title: L10n.tr( + "Time Machine has created its image: stop the backup, then convert the image to 32 MB bands" + ), + command: nil, action: .prepareBundle)) } return steps } diff --git a/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift b/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift index 8392b44..69d3149 100644 --- a/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift +++ b/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift @@ -3,7 +3,7 @@ import Foundation import SwiftUI /// Connects the interface to the services in `CloudMachineCore`. It contains no backup -/// logic itself - what it knows about Google Drive, the image and the buffer lives in the services, +/// logic itself - what it knows about Google Drive and the machine lives in the services, /// so that the CLI and the GUI do exactly the same thing. @MainActor final class CloudMachineController: ObservableObject { @@ -20,7 +20,7 @@ final class CloudMachineController: ObservableObject { /// How often we refresh the answer to "when was a backup last made". /// /// Less often than the rest of the panel (10 s), and deliberately: `BackupHealth.currentReport()` - /// reads the Time Machine preferences file, asks rclone for the Drive capacity + /// reads the Time Machine preferences file, asks the machine, rclone for the Drive capacity /// and tmutil for the backup destination - that is seconds of work, not microseconds. The backup /// cycle is hourly, so an answer from five minutes ago is just as /// good as one from five seconds ago. @@ -41,8 +41,7 @@ final class CloudMachineController: ObservableObject { var cycles = 0 while !Task.isCancelled { await self?.refreshAll() - // An agent can stop being startable later too (gdrive-buffer only - // notices when its rclone exits), so check every few minutes. + // An agent can stop being startable later too, so check every few minutes. cycles += 1 if cycles % 30 == 0 { await AgentRepair.repairBroken() } try? await Task.sleep(nanoseconds: UInt64(interval) * 1_000_000_000) @@ -58,7 +57,7 @@ final class CloudMachineController: ObservableObject { func refreshAll() async { await refreshCredentials() await refreshDependencies() - await refreshBuffer() + await refreshMachine() await refreshTimeMachine() await refreshProgress() await refreshBackupCycle() @@ -66,15 +65,10 @@ final class CloudMachineController: ObservableObject { // run in it. See `WatchdogHeartbeat`: without this the only symptom of an // unloaded watchdog is silence, and silence is the normal state here. status.watchdog = WatchdogHeartbeat.current() - status.preparedForRestart = ShutdownPrep.holdsAttach() status.lastRefresh = Date() } - /// Age of the last SUCCESSFUL backup. - /// - /// The panel did not ask this question even once (see `BackupCycleStatus`), - /// so the failure "everything attached, and no backup for two days" looked - /// in it exactly the same as a working system. The source is the same one + /// Age of the last SUCCESSFUL backup. The source is the same one /// the `backup-health` watchdog uses - a single source of truth, so that /// the panel and the watchdog cannot claim different things about the same thing. private func refreshBackupCycle(force: Bool = false) async { @@ -106,25 +100,20 @@ final class CloudMachineController: ObservableObject { readiness.ready ? .ready : .missing(readiness.missing, readiness.remedies) // No answer keeps what we knew: flipping to "not connected" would offer // the "Connect Google Drive" card on a working installation. - if let configured = await RemoteConfigurer.isConfigured( - remoteName: DriveBufferService.remoteName) - { + if let configured = await RemoteConfigurer.isConfigured(remoteName: DriveFolder.remoteName) { status.remoteConfigured = configured } // A REAL read of the file that actually matters - see - // `BackupHealth.preferencesReadable`. Previously this was - // `isReadableFile` (that is, `access(R_OK)`) on the DIRECTORY - // `~/Library/Application Support/com.apple.TCC`: the wrong path and a check - // that proves nothing under TCC. + // `BackupHealth.preferencesReadable`. status.hasFullDiskAccess = preferencesReads.readable( after: BackupHealth.preferencesReadable(), shown: status.hasFullDiskAccess) - status.driveFolderPath = "\(DriveBufferService.remoteName):\(DriveBufferService.remotePath)" + status.driveFolderPath = VMLayout.remote if !status.remoteConfigured, status.suggestedDriveFolder.isEmpty { let key = await MachineIdentity.currentKey() status.suggestedDriveFolder = DriveFolder.isValid(key) ? key : "this-mac" } status.agentsInstalled = await LaunchdInstaller.isInstalled( - label: "com.renacode.cloudmachine.gdrive-buffer") + label: "com.renacode.cloudmachine.vm-guard") } /// State of our own OAuth credentials. We check ONLY that the entry exists - @@ -146,78 +135,51 @@ final class CloudMachineController: ObservableObject { clientID: clientID, clientSecret: clientSecret) await refreshCredentials() return RemoteConfigurer.credentialsSavedMessage( - remoteConfigured: await RemoteConfigurer.isConfigured( - remoteName: DriveBufferService.remoteName)) + remoteConfigured: await RemoteConfigurer.isConfigured(remoteName: DriveFolder.remoteName)) } catch { return L10n.tr("Not saved: %@", error.localizedDescription) } } - private func refreshBuffer() async { - let stats = await DriveBufferService.queueStats() - var buffer = BufferStatus() - buffer.mounted = DriveBufferService.isMounted - // A dead image (in the mount table, but unreadable) counts as - // NOT ATTACHED - from Time Machine's point of view that is exactly what it is. - // - // `await`, not a computed property: this function runs on `@MainActor` - // every 10 s, and the probe contains a `read()` on FUSE-T. Until 26.09.2026 this was - // an ordinary, blocking read - a wedged volume froze the whole interface - // (window movement, menus, buttons) for as long as the I/O took, that is, potentially - // forever. Now the probe runs on its own thread with a time limit, and the panel - // waits for the result without blocking the main thread. - buffer.imageAttached = await BackupImageService.attachment().isUsable - // We take the buffer size from rclone; walking the directory ourselves means 6504 - // stat calls every 10 seconds on the disk the backup is being written to. - buffer.sizeGB = BufferGuardService.bufferGB(stats: stats) - buffer.freeDiskGB = BufferGuardService.freeGB() - // Two separate questions, because the answers mean different things: lack of space has to be - // fixed, the daily limit passes on its own. - buffer.driveFull = DriveBufferService.hitStorageQuota() - buffer.dailyQuotaExhausted = DriveBufferService.uploadStalled() - // Distinguishes "read" from "came out as zero" - without it, no answer from - // rclone looked like an empty queue. - buffer.queueKnown = stats != nil - if let stats { - buffer.uploadsQueued = stats.uploadsQueued - buffer.uploadsInProgress = stats.uploadsInProgress - buffer.erroredFiles = stats.erroredFiles - buffer.outOfSpace = stats.outOfSpace + /// The machine: Lima's view of it, then - only when it runs - one script in + /// it for the services, the upload buffer and the image, and its address. + private func refreshMachine() async { + var machine = MachineStatus() + let instance: String?? + do { + instance = .some(try await VMLayout.instanceStatus()) + } catch { + instance = .none + } + machine.state = MachineState.from(instanceStatus: instance) + if machine.isRunning, let report = await VMStatus.currentReport() { + machine.services = report.services + machine.reading = report.reading + machine.punchfsOpenFiles = report.punchfsOpenFiles + machine.punchfsFileLimit = report.punchfsFileLimit + machine.bundlePrepared = report.bundlePrepared + machine.imageCount = report.imageCount + machine.address = await VMLayout.currentAddress() } - status.buffer = buffer - status.remoteControlOpen = DriveBufferService.legacyPortOpenNow() - status.imageExists = - buffer.mounted - ? FileManager.default.fileExists(atPath: BackupImageService.imagePath.path) : nil + status.machine = machine + status.remoteControlOpen = VMStatus.remoteControlOpenOnMac() } - /// `destinationReading()`, and NOT `currentDestinationMountPoint()`. - /// - /// The latter returns `nil` both when there is no destination and when there is no - /// answer from tmutil, so the panel showed "Time Machine does not point to - /// CloudMachine" even when it knew NOTHING about the destination. The direction of the error - /// was safe (a false alarm instead of false calm), but the message - /// sent a person off to register a destination that is intact. The distinction has existed - /// in `DestinationReading` since 23.09.2026 and the `backup-health` watchdog already - /// uses it - the panel is the last place that conflated these two things. - /// - /// The destination may also exist and point somewhere else - then there is no backup on Drive, - /// even though Time Machine looks configured; that is still - /// `.notRegistered`. + /// `destinationsReading()` keeps "no answer" apart from "no destination": + /// the panel must not tell a person to register a destination that is intact + /// while it was the read that hung. A destination pointing somewhere else is + /// still `.notRegistered`. private func refreshTimeMachine() async { - status.timeMachineState = TimeMachineState.from( - await TimeMachineStatus.destinationReading(), - target: BackupImageService.targetPath.path) + let destinations = await TimeMachineStatus.destinationsReading() + status.timeMachineState = TimeMachineState.from(destinations, address: status.machine.address) // Cheap reads only: the limit from machines.json, the last stored usage // measurement and `tmutil destinationinfo`. Measuring the Drive folder // is the watchdog's job (or the Measure button). status.budgetLimitGB = MachineBudget.limitGB() status.budgetUsage = MachineBudget.storedUsage() - let target = BackupImageService.targetPath.path - status.timeMachineQuotaGB = await TimeMachineStatus.destinationQuotaGB( - forMountPointContaining: target) - status.timeMachineDestinationID = await TimeMachineStatus.destinationID( - forMountPointContaining: target) + let ours = destinations.flatMap { VMStatus.machineDestination(in: $0, address: nil) } + status.timeMachineQuotaGB = ours?.quotaGB + status.timeMachineDestinationID = ours?.id } /// Stores this Mac's limit, then measures the Drive folder so the card @@ -284,12 +246,6 @@ final class CloudMachineController: ObservableObject { // MARK: - Actions - func installFuse() async { - await run(L10n.tr("Installing FUSE-T"), log: "Installing FUSE-T") { - await FuseInstaller.install() - } - } - /// Opens System Settings at Full Disk Access. Granting it is a decision /// only the user can make; the app can only take them to the right pane. func openFullDiskAccessSettings() { @@ -301,14 +257,28 @@ final class CloudMachineController: ObservableObject { } var setupPlan: [SetupStep] { - SetupPlan.steps( + let machineExists: Bool? + switch status.machine.state { + case .unknown, .noAnswer: machineExists = nil + case .notCreated: machineExists = false + case .notRunning, .running: machineExists = true + } + let address = status.machine.address + let password = VMSetup.storedPassword() + return SetupPlan.steps( SetupPlan.Inputs( - hasRclone: CMTooling.hasManagedRclone, hasFuse: CMTooling.hasFuse, - remoteConfigured: status.remoteConfigured, hasFullDiskAccess: status.hasFullDiskAccess, - agentsInstalled: status.agentsInstalled, mounted: status.buffer.mounted, - imageExists: status.imageExists, imageAttached: status.buffer.imageAttached, + hasRclone: CMTooling.hasManagedRclone, remoteConfigured: status.remoteConfigured, + hasLima: VMLayout.limactlPath != nil, machineExists: machineExists, + machineRunning: status.machine.isRunning, hasFullDiskAccess: status.hasFullDiskAccess, + agentsInstalled: status.agentsInstalled, timeMachineNotPointingHere: status.timeMachineState == .notRegistered, - connectCommand: connectDriveCommand, setDestinationCommand: setDestinationCommand)) + bundleNeedsPreparing: status.machine.bundleNeedsPreparing, + connectCommand: connectDriveCommand, + setDestinationCommand: address.flatMap { address in + password.map { VMSetup.destinationCommand(password: $0, address: address) } + }, + maskedSetDestinationCommand: address.map { VMSetup.maskedDestinationCommand(address: $0) }) + ) } func installRclone() async { @@ -329,41 +299,36 @@ final class CloudMachineController: ObservableObject { agent: CMPaths.agentBinaryPath?.path ?? "cloudmachine-agent", folder: folder) } - func createImage(sizeGB: Int) async { - await run(L10n.tr("Creating the backup image"), log: "Creating the backup image") { - await BackupImageService.create(sizeGB: sizeGB) - } - } - - func attachImage() async { - // An explicit attach ends a preparation for restart that is no longer wanted. - ShutdownPrep.clear() - await run(L10n.tr("Attaching the image"), log: "Attaching the image") { - await BackupImageService.attach() + /// `vm-setup` from the app. The `tmutil` command it returns holds the SMB + /// password, so it is NOT part of the message (which goes to the log and on + /// screen); the setup card builds it from the password file instead. + func setUpMachine() async { + await run(L10n.tr("Setting up the CloudMachine machine"), log: "Setting up the machine") { + let result = await VMSetup.run(replaceRcloneConfig: false) + return CMActionResult( + succeeded: result.succeeded, message: result.messages.joined(separator: "\n")) } } - /// Pauses the backup, detaches the image and waits until everything has - /// reached Drive, then keeps the image detached until the restart - /// (`ShutdownPrep`). The menu asks for confirmation first. - func prepareForRestart() async { - await run(L10n.tr("Preparing for restart"), log: "Preparing for restart") { - await ShutdownPrep.perform() + func startMachine() async { + await run(L10n.tr("Starting the CloudMachine machine"), log: "Starting the machine") { + await VMLayout.start() } } - /// Restarts the Drive mount so it comes back on the private socket. The - /// view asks for confirmation first; `MountRestart` refuses by itself - /// while Time Machine is backing up. - func restartMount() async { - await run(L10n.tr("Restarting the Google Drive mount"), log: "Restarting the Drive mount") { - await MountRestart.perform() + /// `vm-prepare-bundle`: refuses by itself while Time Machine backs up to the machine. + func prepareBundle() async { + await run(L10n.tr("Converting the image to 32 MB bands"), log: "Preparing the image") { + let outcome = await VMBundlePrep.run(bundleOverride: nil) + return CMActionResult(succeeded: outcome.succeeded, message: outcome.message) } } - func verifyImage() async { - await run(L10n.tr("Checking image consistency"), log: "Checking image consistency") { - await BackupImageService.verify() + /// Stops the backup, waits a while for the upload, then stops the machine + /// (`VMShutdown`). The menu asks for confirmation first. + func prepareForRestart() async { + await run(L10n.tr("Preparing for restart"), log: "Preparing for restart") { + await VMShutdown.perform() } } @@ -393,12 +358,6 @@ final class CloudMachineController: ObservableObject { } } - /// A command the user has to paste themselves - `tmutil setdestination` - /// requires root, and the app has no sudoers rule. - var setDestinationCommand: String { - "sudo tmutil setdestination '\(BackupImageService.targetPath.path)'" - } - // MARK: - Shared action handling /// `label` is shown in the interface (translated); `logName` goes to the log, diff --git a/mac-app/Sources/CloudMachineApp/Views/DashboardView.swift b/mac-app/Sources/CloudMachineApp/Views/DashboardView.swift index dd02c26..3de4fea 100644 --- a/mac-app/Sources/CloudMachineApp/Views/DashboardView.swift +++ b/mac-app/Sources/CloudMachineApp/Views/DashboardView.swift @@ -63,14 +63,11 @@ private struct DashboardContent: View { @State private var clientID = "" @State private var clientSecret = "" @State private var credentialsMessage: String? - /// Logical size of a new backup image. Sparse: Drive holds only what is written. - @State private var imageSizeGB = "4000" /// This Mac's folder on Google Drive, chosen before connecting. Empty means /// "not edited yet" and shows the suggestion from the computer name. @State private var driveFolder = "" /// The limit being typed; empty = show the stored one. @State private var limitText = "" - @State private var confirmingMountRestart = false var body: some View { HStack(spacing: 0) { @@ -197,7 +194,7 @@ private struct DashboardContent: View { let tone = backupCycleTone.worst(status.watchdogRunning ? .success : .danger) return tone == .success ? nil : tone case .storage: - let tone = uploadTone(status.buffer.uploadState) + let tone = uploadTone(status.machine.uploadState) .worst(freeSpaceTone) .worst(budgetTone == .neutral ? .success : budgetTone) return tone == .success ? nil : tone @@ -254,7 +251,7 @@ private struct DashboardContent: View { } HStack(alignment: .top, spacing: 16) { - bufferSummaryCard + machineSummaryCard driveSummaryCard } .fixedSize(horizontal: false, vertical: true) @@ -276,31 +273,34 @@ private struct DashboardContent: View { } private var bufferTone: StatusTone { - if status.buffer.outOfSpace { return .danger } + if status.machine.reading?.buffer.outOfSpace == true { return .danger } return freeSpaceTone == .success ? .brand : freeSpaceTone } - /// The same 80 GB threshold the details row has always used. + /// Free space in the machine, against the watchdog's threshold (above the + /// guard's pause threshold, so amber comes before the guard brakes). private var freeSpaceTone: StatusTone { - guard let free = status.buffer.freeDiskGB else { return .warning } - return free > 80 ? .success : .danger + guard let free = status.machine.freeGiB else { return .warning } + return free >= BackupHealth.machineFreeWarningGB ? .success : .danger } - private var bufferSummaryCard: some View { + private var machineSummaryCard: some View { VStack(alignment: .leading, spacing: 14) { HStack(spacing: 10) { IconTile( - systemImage: "internaldrive.fill", + systemImage: "server.rack", gradient: LinearGradient( colors: [RenaCodeTheme.colorCyan, RenaCodeTheme.colorIndigo], startPoint: .topLeading, endPoint: .bottomTrailing)) - Text(L10n.tr("SSD buffer")) + Text(L10n.tr("Machine buffer")) .font(.system(size: 16, weight: .semibold)) .foregroundStyle(RenaCodeTheme.textMain) } HStack(alignment: .firstTextBaseline) { - Text(L10n.tr("%@ GB", "\(status.buffer.sizeGB)")) + // The unsent backlog, not the cache size: the cache keeps uploaded + // bands too and sits near its limit long after the upload caught up. + Text(status.machine.unsentGiB.map { L10n.tr("%@ GiB", "\($0)") } ?? "?") .font(.system(size: 34, weight: .bold).monospacedDigit()) .foregroundStyle(RenaCodeTheme.textMain) .lineLimit(1) @@ -308,8 +308,6 @@ private struct DashboardContent: View { Spacer(minLength: 8) - // Where the mockup has a sparkline: the app keeps no history of the - // buffer, so the card shows the queue, which it does read. VStack(alignment: .trailing, spacing: 2) { Text(L10n.tr("Waiting to upload")) .font(.system(size: 11)) @@ -320,14 +318,11 @@ private struct DashboardContent: View { } } - // A missing measurement must look different from a number - see - // `BufferGuardService.freeGB()`. Simply putting the optional value - // into the text gave "Free on disk: Optional(427) GB", and the compiler - // reported it ONLY as a warning, so no test would have caught it. + // A missing measurement must look different from a number. Text( L10n.tr( - "Free on disk: %@", - status.buffer.freeDiskGB.map { L10n.tr("%@ GB", "\($0)") } ?? L10n.tr("not measured")) + "Free in the machine: %@", + status.machine.freeGiB.map { L10n.tr("%@ GiB", "\($0)") } ?? L10n.tr("not measured")) ) .font(.system(size: 13)) .foregroundStyle(freeSpaceTone == .success ? RenaCodeTheme.textMuted : freeSpaceTone.color) @@ -339,20 +334,20 @@ private struct DashboardContent: View { /// Without a queue reading there is no right to say "nothing" - the zeros /// are then a missing measurement, not a result. private var queueText: String { - guard status.buffer.queueKnown else { return "?" } - return status.buffer.draining - ? L10n.tr("%@ files", "\(status.buffer.uploadsQueued)") : L10n.tr("nothing") + guard status.machine.queueKnown else { return "?" } + return status.machine.draining + ? L10n.tr("%@ bands", "\(status.machine.uploadsQueued)") : L10n.tr("nothing") } /// Cyan while the queue drains: that is the normal state during a backup, /// and amber there made a working upload look like a warning. private var queueTone: StatusTone { - guard status.buffer.queueKnown else { return .warning } - return status.buffer.draining ? .info : .success + guard status.machine.queueKnown else { return .warning } + return status.machine.draining ? .info : .success } private var driveSummaryCard: some View { - let upload = status.buffer.uploadState + let upload = status.machine.uploadState return VStack(alignment: .leading, spacing: 14) { HStack(spacing: 10) { IconTile(systemImage: "icloud.fill", gradient: RenaCodeTheme.aiGradient) @@ -421,21 +416,7 @@ private struct DashboardContent: View { /// Green only when the state is really good; a hard failure is red, the rest /// (unknown, not yet checked, passes by itself) amber. - private var healthTone: StatusTone { - if status.healthy { return .success } - if case .missing = status.dependencyState { return .danger } - // Detached on purpose before a restart - amber, not "action needed". - if status.detachedForRestart && status.remoteConfigured && status.buffer.mounted { - return .warning - } - if !status.remoteConfigured || !status.buffer.mounted || !status.buffer.imageAttached { - return .danger - } - if case .notRegistered = status.timeMachineState { return .danger } - if status.buffer.uploadState.needsAttention { return .danger } - if status.backupCycle.known && !status.backupCycle.isFresh() { return .danger } - return .warning - } + private var healthTone: StatusTone { status.overallTone } private var healthCard: some View { VStack(alignment: .leading, spacing: 12) { @@ -473,24 +454,61 @@ private struct DashboardContent: View { private var timeMachineText: String { switch status.timeMachineState { - case .registered: return L10n.tr("Points to CloudMachine") + case .registered: return L10n.tr("Points to the machine") case .notRegistered: return L10n.tr("Does not point to CloudMachine") case .noAnswer: return L10n.tr("tmutil did not answer") case .unknown: return L10n.tr("not checked") } } + private var machineTone: StatusTone { + switch status.machine.state { + case .running: return .success + case .unknown: return .neutral + case .noAnswer: return .warning + case .notCreated, .notRunning: return .danger + } + } + + private var machineText: String { + switch status.machine.state { + case .running: return L10n.tr("Running") + case .notRunning(let state): return state + case .notCreated: return L10n.tr("Not set up") + case .noAnswer: return L10n.tr("limactl did not answer") + case .unknown: return L10n.tr("not checked") + } + } + + private var servicesTone: StatusTone { + switch status.machine.servicesHealthy { + case .some(true): return .success + case .some(false): return .danger + case .none: return status.machine.isRunning ? .warning : .neutral + } + } + + private var servicesText: String { + switch status.machine.servicesHealthy { + case .some(true): return L10n.tr("All running") + case .some(false): + let down = VMLayout.services.filter { + let state = status.machine.services[$0] + return state != "active" && !($0 == "tm-bands-bind" && state == "inactive") + } + return L10n.tr("Not running: %@", down.joined(separator: ", ")) + case .none: return L10n.tr("not read") + } + } + private var disksTone: StatusTone { - (status.buffer.mounted ? StatusTone.success : .danger) - .worst(status.buffer.imageAttached ? .success : .danger) + machineTone.worst(servicesTone == .neutral ? .warning : servicesTone) .worst(timeMachineTone == .neutral ? .warning : timeMachineTone) } private var disksPill: String { - if !status.buffer.mounted { return L10n.tr("Drive disconnected") } - if !status.buffer.imageAttached { return L10n.tr("Not attached") } switch disksTone { - case .success: return L10n.tr("Attached") + case .success: return L10n.tr("Running") case .danger: return L10n.tr("Action needed") default: return L10n.tr("Attention") } @@ -499,7 +517,7 @@ private struct DashboardContent: View { private var disksCard: some View { VStack(alignment: .leading, spacing: 12) { HStack { - Text(L10n.tr("Disks")) + Text(L10n.tr("Machine")) .font(.system(size: 13, weight: .bold)) .textCase(.uppercase) .tracking(0.6) @@ -509,14 +527,8 @@ private struct DashboardContent: View { } VStack(alignment: .leading, spacing: 6) { - compactRow( - "Google Drive", - status.buffer.mounted ? L10n.tr("Mounted") : L10n.tr("Inactive"), - tone: status.buffer.mounted ? .success : .danger) - compactRow( - L10n.tr("Backup image"), - status.buffer.imageAttached ? L10n.tr("Attached") : L10n.tr("Detached"), - tone: status.buffer.imageAttached ? .success : .danger) + compactRow(VMLayout.instanceName, machineText, tone: machineTone) + compactRow(L10n.tr("Services"), servicesText, tone: servicesTone) compactRow("Time Machine", timeMachineText, tone: timeMachineTone) } } @@ -584,66 +596,25 @@ private struct DashboardContent: View { // MARK: - Open remote control - /// The mount still takes commands on the open TCP port. Says what that - /// means, and offers the one fix that does not need a restart of the Mac. + /// rclone's remote control in the machine answers on the Mac's localhost. + /// Says what that means and where the fix is; the fix restarts the machine, + /// so it is left to the person. private var openRemoteControlCard: some View { VStack(alignment: .leading, spacing: 12) { cardTitle( - L10n.tr("Remote control of the Drive mount is open"), systemImage: "lock.open.fill", + L10n.tr("rclone's remote control is open on this Mac"), systemImage: "lock.open.fill", tone: .danger) Text( L10n.tr( - "The mount was started by an older version of CloudMachine and still takes commands on %@ without a password. Any web page open in a browser can send it one - including deleting the backup on Google Drive, with no trash to restore it from. It moves to a private socket the next time it starts.", - DriveBufferService.legacyRCAddress) - ) - .font(.system(size: 13)) - .foregroundStyle(RenaCodeTheme.textMain) - .fixedSize(horizontal: false, vertical: true) - - Text( - L10n.tr( - "Restart the mount: the image is detached, everything waits until it has reached Google Drive, the mount starts again and the image is attached. It takes a few minutes, during which Time Machine has no destination. Or prepare the shutdown and restart the Mac:" + "127.0.0.1:5572 answers on the Mac: Lima forwards the machine's rclone remote control, which has no password. Any web page open in a browser can send it commands - including deleting the backup on Google Drive. Add the portForwards rule from vm/cmsmb.yaml to the machine and restart it (see vm/README.md)." ) ) .font(.system(size: 13)) - .foregroundStyle(RenaCodeTheme.textMuted) + .foregroundStyle(RenaCodeTheme.textMain) .fixedSize(horizontal: false, vertical: true) - - commandBox("\(CMPaths.agentBinaryPath?.path ?? "cloudmachine-agent") prepare-shutdown") - - HStack(spacing: 12) { - Button(action: { confirmingMountRestart = true }) { - HStack(spacing: 6) { - Image(systemName: "arrow.triangle.2.circlepath") - Text(L10n.tr("Restart the mount")) - } - } - .buttonStyle(PrimaryGradientButtonStyle()) - .disabled(!status.canRestartMount) - - if status.backupProgress != nil { - Text(L10n.tr("Wait until the backup has finished.")) - .font(.system(size: 12)) - .foregroundStyle(RenaCodeTheme.textMuted) - } - Spacer() - } } .toneCard(.danger) - .confirmationDialog( - L10n.tr("Restart the Google Drive mount now?"), isPresented: $confirmingMountRestart - ) { - Button(L10n.tr("Restart the mount"), role: .destructive) { - Task { await controller.restartMount() } - } - Button(L10n.tr("Cancel"), role: .cancel) {} - } message: { - Text( - L10n.tr( - "Time Machine loses its destination for a few minutes. Nothing is stopped until everything waiting has reached Google Drive; if that does not happen, the mount is left as it is." - )) - } } // MARK: - Backups @@ -665,16 +636,6 @@ private struct DashboardContent: View { HStack(spacing: 12) { backupButton - - Button(action: { Task { await controller.verifyImage() } }) { - HStack(spacing: 6) { - Image(systemName: "checkmark.shield") - Text(L10n.tr("Check image consistency")) - } - } - .buttonStyle(SecondaryGlassButtonStyle()) - .disabled(status.buffer.imageAttached || status.isBusy) - Spacer() } } @@ -796,7 +757,7 @@ private struct DashboardContent: View { private var storageSection: some View { VStack(alignment: .leading, spacing: 16) { uploadStateCard - bufferCard + machineCard // This Mac's space limit on Google Drive if status.remoteConfigured { budgetCard @@ -817,26 +778,14 @@ private struct DashboardContent: View { private var setupSteps: [SetupStep] { controller.setupPlan } /// One button per step the app can do itself. Disabled while anything runs: - /// image operations share one lock, and a second click would only report - /// "another operation is in progress". - @ViewBuilder + /// a second click while the machine is being set up would only collide. private func setupActionButton(_ action: SetupStep.Action) -> some View { - HStack(spacing: 8) { - if action == .createImage { - Text(L10n.tr("Size (GB)")) - .font(.system(size: 12)) - .foregroundStyle(RenaCodeTheme.textMain) - TextField("", text: $imageSizeGB) - .textFieldStyle(.roundedBorder) - .frame(width: 80) - } - Button(action: { Task { await perform(action) } }) { - Text(setupActionTitle(action)) - .font(.system(size: 12, weight: .semibold)) - } - .buttonStyle(SecondaryGlassButtonStyle()) - .disabled(status.isBusy || (action == .createImage && parsedImageSize == nil)) + Button(action: { Task { await perform(action) } }) { + Text(setupActionTitle(action)) + .font(.system(size: 12, weight: .semibold)) } + .buttonStyle(SecondaryGlassButtonStyle()) + .disabled(status.isBusy) } private var effectiveDriveFolder: String { @@ -844,7 +793,7 @@ private struct DashboardContent: View { } /// Each Mac on the same Google account needs its own folder, otherwise - /// they would share one backup image. Chosen once: a Mac that already has a + /// they would share one backup. Chosen once: a Mac that already has a /// folder refuses to switch, because the new one would be an empty backup. private var folderField: some View { VStack(alignment: .leading, spacing: 4) { @@ -873,32 +822,25 @@ private struct DashboardContent: View { } } - private var parsedImageSize: Int? { - guard let value = Int(imageSizeGB.trimmingCharacters(in: .whitespaces)), value >= 100 - else { return nil } - return value - } - private func setupActionTitle(_ action: SetupStep.Action) -> String { switch action { case .installRclone: return L10n.tr("Install rclone") - case .installFuse: return L10n.tr("Install FUSE-T") + case .setUpMachine: return L10n.tr("Set up the machine") + case .startMachine: return L10n.tr("Start the machine") case .grantFullDiskAccess: return L10n.tr("Open System Settings") case .installAgents: return L10n.tr("Install agents") - case .createImage: return L10n.tr("Create image") - case .attachImage: return L10n.tr("Attach image") + case .prepareBundle: return L10n.tr("Convert the image") } } private func perform(_ action: SetupStep.Action) async { switch action { case .installRclone: await controller.installRclone() - case .installFuse: await controller.installFuse() + case .setUpMachine: await controller.setUpMachine() + case .startMachine: await controller.startMachine() case .grantFullDiskAccess: controller.openFullDiskAccessSettings() case .installAgents: await controller.installAgents() - case .createImage: - if let size = parsedImageSize { await controller.createImage(sizeGB: size) } - case .attachImage: await controller.attachImage() + case .prepareBundle: await controller.prepareBundle() } } @@ -936,7 +878,7 @@ private struct DashboardContent: View { if let command = step.choosesFolder ? controller.connectDriveCommand(folder: effectiveDriveFolder) : step.command { - commandBox(command) + commandBox(command, shown: step.displayedCommand) } } } @@ -945,10 +887,12 @@ private struct DashboardContent: View { .toneCard(.warning) } - /// A command the user has to run themselves, with a Copy button. - private func commandBox(_ command: String) -> some View { + /// A command the user has to run themselves, with a Copy button. `shown` + /// replaces the text on screen when `command` holds a secret; Copy still + /// copies `command`. + private func commandBox(_ command: String, shown: String? = nil) -> some View { HStack { - Text(command) + Text(shown ?? command) .font(.system(size: 12, design: .monospaced)) .foregroundStyle(RenaCodeTheme.textMain) .textSelection(.enabled) @@ -1062,73 +1006,80 @@ private struct DashboardContent: View { } } - // MARK: - Buffer and Upload Details + // MARK: - Machine and Upload Details - private var bufferCard: some View { + private var machineCard: some View { VStack(alignment: .leading, spacing: 10) { - cardTitle(L10n.tr("Local Buffer & Upload Status"), systemImage: "server.rack") + cardTitle(L10n.tr("Machine & Upload Status"), systemImage: "server.rack") .padding(.bottom, 4) - row( - L10n.tr("Google Drive mount (FUSE-T)"), - status.buffer.mounted ? L10n.tr("Mounted") : L10n.tr("Inactive"), - tone: status.buffer.mounted ? .success : .danger) + row(L10n.tr("Machine %@", VMLayout.instanceName), machineText, tone: machineTone) + if let address = status.machine.address { + Divider().background(RenaCodeTheme.borderGlass) + row(L10n.tr("Address"), address, tone: .neutral) + } - Divider().background(RenaCodeTheme.borderGlass) + ForEach(VMLayout.services, id: \.self) { service in + Divider().background(RenaCodeTheme.borderGlass) + let state = status.machine.services[service] + row( + L10n.tr("Service %@", service), state ?? L10n.tr("not read"), + tone: state == "active" + ? .success + : (state == nil + ? .neutral : (service == "tm-bands-bind" && state == "inactive" ? .warning : .danger)) + ) + } if status.remoteConfigured { - row(L10n.tr("Folder on Google Drive"), status.driveFolderPath, tone: .neutral) - Divider().background(RenaCodeTheme.borderGlass) + row(L10n.tr("Folder on Google Drive"), status.driveFolderPath, tone: .neutral) } - row( - L10n.tr("Backup disk image (.sparsebundle)"), - status.buffer.imageAttached ? L10n.tr("Attached to the system") : L10n.tr("Detached"), - tone: status.buffer.imageAttached ? .success : .danger) - Divider().background(RenaCodeTheme.borderGlass) - + // The tone of the upload verdict, not its own. row( - L10n.tr("Buffer allocated on the SSD"), - L10n.tr("%@ GB", "\(status.buffer.sizeGB)"), - tone: status.buffer.outOfSpace ? .danger : .neutral) + L10n.tr("To upload"), status.machine.queueSummary, + tone: uploadTone(status.machine.uploadState)) Divider().background(RenaCodeTheme.borderGlass) - - // A missing measurement MUST look different from "0 GB" - see - // `BufferGuardService.freeGB()`. A failed statfs is a failure of the buffer - // guard, not information about an empty disk. row( - L10n.tr("Free space on the local volume"), - status.buffer.freeDiskGB.map { L10n.tr("%@ GB", "\($0)") } ?? L10n.tr("not measured"), - tone: freeSpaceTone) + L10n.tr("Cache on the machine's disk"), + status.machine.cacheGiB.map { L10n.tr("%@ GiB", "\($0)") } ?? L10n.tr("not measured"), + tone: .neutral) Divider().background(RenaCodeTheme.borderGlass) - - // The tone of the upload verdict, not its own: a queue at zero with the - // daily limit used up is not green - nothing is going out. + // A missing measurement MUST look different from "0 GB". row( - L10n.tr("Cloud sync queue"), status.buffer.queueSummary, - tone: uploadTone(status.buffer.uploadState)) + L10n.tr("Free in the machine"), + status.machine.freeGiB.map { L10n.tr("%@ GiB", "\($0)") } ?? L10n.tr("not measured"), + tone: freeSpaceTone) - if status.buffer.erroredFiles > 0 { + if let open = status.machine.punchfsOpenFiles { Divider().background(RenaCodeTheme.borderGlass) row( - L10n.tr("File upload errors"), - L10n.tr("%@ files", "\(status.buffer.erroredFiles)"), - tone: .danger) + L10n.tr("punchfs open files"), + L10n.tr("%@ of %@", "\(open)", status.machine.punchfsFileLimit.map(String.init) ?? "?"), + tone: status.machine.punchfsNearLimit ? .danger : .neutral) } - if status.buffer.driveFull { + if status.machine.erroredFiles > 0 { Divider().background(RenaCodeTheme.borderGlass) - row(L10n.tr("Space on Google Drive"), L10n.tr("Out of space"), tone: .danger) + row( + L10n.tr("File upload errors"), + L10n.tr("%@ files", "\(status.machine.erroredFiles)"), + tone: .danger) } - if status.buffer.dailyQuotaExhausted { - Divider().background(RenaCodeTheme.borderGlass) - // Amber, not red: the limit passes by itself - see `UploadState`. - row(L10n.tr("Google Drive limit"), L10n.tr("Daily 750 GB exhausted"), tone: .warning) + if status.canStartMachine { + Button(action: { Task { await controller.startMachine() } }) { + HStack(spacing: 6) { + Image(systemName: "power.circle") + Text(L10n.tr("Start the machine")) + } + } + .buttonStyle(PrimaryGradientButtonStyle()) + .padding(.top, 4) } } .toneCard(.neutral) @@ -1225,7 +1176,7 @@ private struct DashboardContent: View { /// Red - you need to act. Without the middle state an exhausted limit /// would have to pretend to be either a failure or all-clear, and it is neither. private var uploadStateCard: some View { - let state = status.buffer.uploadState + let state = status.machine.uploadState let tone = uploadTone(state) return VStack(alignment: .leading, spacing: 12) { diff --git a/mac-app/Sources/CloudMachineApp/Views/MenuBarContentView.swift b/mac-app/Sources/CloudMachineApp/Views/MenuBarContentView.swift index a97f680..f65541f 100644 --- a/mac-app/Sources/CloudMachineApp/Views/MenuBarContentView.swift +++ b/mac-app/Sources/CloudMachineApp/Views/MenuBarContentView.swift @@ -42,21 +42,7 @@ private struct MenuBarPanel: View { /// Green only for a really good state - the same rule as the window's /// health card. - private var tone: StatusTone { - if status.healthy { return .success } - if case .missing = status.dependencyState { return .danger } - // Detached on purpose before a restart - amber, not "action needed". - if status.detachedForRestart && status.remoteConfigured && status.buffer.mounted { - return .warning - } - if !status.remoteConfigured || !status.buffer.mounted || !status.buffer.imageAttached { - return .danger - } - if case .notRegistered = status.timeMachineState { return .danger } - if status.buffer.uploadState.needsAttention { return .danger } - if status.backupCycle.known && !status.backupCycle.isFresh() { return .danger } - return .warning - } + private var tone: StatusTone { status.overallTone } private var cycleTone: StatusTone { if status.backupCycle.isFresh() { return .success } @@ -86,7 +72,7 @@ private struct MenuBarPanel: View { // The one-sentence answer to "is my data safe", in the verdict's color. HStack(alignment: .firstTextBaseline, spacing: 8) { - StatusDot(tone, live: status.buffer.uploadState.isMovingData) + StatusDot(tone, live: status.machine.uploadState.isMovingData) Text(status.headline) .font(.system(size: 13, weight: .medium)) .foregroundStyle(tone == .success ? RenaCodeTheme.textMain : tone.color) @@ -97,11 +83,11 @@ private struct MenuBarPanel: View { // The headline says it only while nothing worse is going on; this row // stays as long as the port is open. if status.remoteControlOpen, - status.headline != L10n.tr("Drive remote control is open - restart the mount") + status.headline != L10n.tr("rclone's remote control is open on this Mac") { HStack(alignment: .firstTextBaseline, spacing: 8) { StatusDot(.danger) - Text(L10n.tr("Drive remote control is open - restart the mount")) + Text(L10n.tr("rclone's remote control is open on this Mac")) .font(.system(size: 12, weight: .medium)) .foregroundStyle(StatusTone.danger.color) .fixedSize(horizontal: false, vertical: true) @@ -129,21 +115,21 @@ private struct MenuBarPanel: View { .accessibilityElement(children: .combine) } - // Queue and buffer state, as two tiles like the window's cards + // The machine's upload buffer, as two tiles like the window's cards HStack(spacing: 8) { tile( L10n.tr("Waiting to upload"), - !status.buffer.queueKnown - ? "?" - : (status.buffer.draining - ? L10n.tr("%@ files", "\(status.buffer.uploadsQueued)") : L10n.tr("nothing")), + status.machine.unsentGiB.map { + status.machine.draining ? L10n.tr("%@ GiB", "\($0)") : L10n.tr("nothing") + } ?? "?", // Cyan while draining - the normal state during a backup; amber only // when the queue could not be read. - valueColor: !status.buffer.queueKnown + valueColor: !status.machine.queueKnown ? RenaCodeTheme.colorWarning - : (status.buffer.draining ? RenaCodeTheme.colorCyan : RenaCodeTheme.textMain)) + : (status.machine.draining ? RenaCodeTheme.colorCyan : RenaCodeTheme.textMain)) tile( - L10n.tr("SSD buffer"), L10n.tr("%@ GB", "\(status.buffer.sizeGB)"), + L10n.tr("Free in the machine"), + status.machine.freeGiB.map { L10n.tr("%@ GiB", "\($0)") } ?? "?", valueColor: RenaCodeTheme.textMain) } @@ -208,7 +194,7 @@ private struct MenuBarPanel: View { if confirmingRestartPrep { Text( L10n.tr( - "The backup is paused, the image is detached and everything waits until it has reached Google Drive. It can take several minutes." + "The backup is stopped, the upload gets up to 30 minutes, then the machine shuts down cleanly. It starts again at the next login." ) ) .font(.system(size: 11)) @@ -231,6 +217,15 @@ private struct MenuBarPanel: View { } .buttonStyle(SecondaryGlassButtonStyle()) } + } else if status.canStartMachine { + Button(action: { Task { await controller.startMachine() } }) { + HStack { + Image(systemName: "power.circle") + Text(L10n.tr("Start the machine")) + Spacer() + } + } + .buttonStyle(SecondaryGlassButtonStyle()) } else { Button(action: { confirmingRestartPrep = true }) { HStack { @@ -240,10 +235,10 @@ private struct MenuBarPanel: View { } } .buttonStyle(SecondaryGlassButtonStyle()) - .disabled(status.isBusy) + .disabled(status.isBusy || !status.machine.isRunning) .help( L10n.tr( - "Prepares for a restart: pauses the backup, detaches the image and waits for the upload." + "Prepares for a restart: stops the backup, waits a while for the upload, then stops the machine cleanly." )) } diff --git a/mac-app/Sources/CloudMachineCore/AgentRepair.swift b/mac-app/Sources/CloudMachineCore/AgentRepair.swift index 293ade3..8f78053 100644 --- a/mac-app/Sources/CloudMachineCore/AgentRepair.swift +++ b/mac-app/Sources/CloudMachineCore/AgentRepair.swift @@ -6,25 +6,32 @@ import Foundation /// leaves the loaded jobs pointing at the old code signature of that path. /// From then on launchd refuses to start them: `job state = spawn failed`, /// `last exit reason = OS_REASON_CODESIGNING`, and nothing in any log. Seen on -/// 6 Oct 2026 after the 1.3.0 -> 1.3.1 upgrade: the backup watchdog and the -/// image attach stopped running, which looks exactly like a quiet, healthy day. -/// Only `bootout` + `bootstrap` clears it. +/// 6 Oct 2026 after the 1.3.0 -> 1.3.1 upgrade: the backup watchdog stopped +/// running, which looks exactly like a quiet, healthy day. Only `bootout` + +/// `bootstrap` clears it. /// -/// Two moments: +/// Three moments: /// - after a version change, the agents that are safe to restart are reloaded /// at once, so they run the new code; -/// - at any time, an agent that launchd failed to spawn is reloaded. This is -/// what covers `gdrive-buffer`: restarting it while it runs would drop the -/// Google Drive mount, so it is left alone until it has actually died - and -/// then reloading costs nothing, because the mount is already gone. +/// - at any time, an agent that launchd failed to spawn is reloaded; +/// - always, the agents of the retired disk-image layout are removed +/// (`removeRetiredAgents`). public enum AgentRepair { static let prefix = "com.renacode.cloudmachine." - /// Restarting these does not touch the mount (measured 26 Sep 2026; the - /// mount lives in the rclone process of `gdrive-buffer`). - public static let safeToReload = ["backup-health", "gdrive-attach", "buffer-guard"] - /// Every agent that runs our binary. - public static let all = ["gdrive-buffer"] + safeToReload + /// One-shot checks: reloading them only runs a check early. + public static let safeToReload = ["backup-health", "vm-guard"] + /// Every agent launchd should be able to start. `vm-start` is not reloaded + /// after an upgrade: a reload starts the machine, and someone may have + /// stopped it on purpose (`prepare-shutdown`). It is reloaded only when + /// launchd cannot start it at all. + public static let all = safeToReload + ["vm-start"] + + /// Agents of the disk-image layout (an rclone mount on the Mac, the image + /// attached in /Volumes/CloudMachine, the buffer guard) and the even older + /// `mount` agent. Their commands no longer exist in this binary, so a + /// loaded one only fails, over and over. + public static let retired = ["gdrive-buffer", "gdrive-attach", "buffer-guard", "mount"] // MARK: - Reading launchd @@ -100,6 +107,114 @@ public enum AgentRepair { return repaired } + // MARK: - The retired layout + + /// What removing the retired agents touches - replaceable, so the tests + /// run the whole migration without launchd and without the real + /// ~/Library/LaunchAgents. + struct RetiredProbes: Sendable { + var plistExists: @Sendable (String) -> Bool + var isLoaded: @Sendable (String) async -> Bool + var bootout: @Sendable (String) async -> Void + var removePlist: @Sendable (String) -> Bool + var leftovers: @Sendable () -> [String] + var log: @Sendable (String) -> Void + + static let live = RetiredProbes( + plistExists: { FileManager.default.fileExists(atPath: plist($0).path) }, + isLoaded: { await printOutput(label: $0) != nil }, + bootout: { name in + _ = try? await ProcessRunner.run( + "/bin/launchctl", ["bootout", "gui/\(getuid())/\(prefix)\(name)"], timeout: 30) + }, + removePlist: { name in + let url = plist(name) + guard FileManager.default.fileExists(atPath: url.path) else { return true } + return (try? FileManager.default.removeItem(at: url)) != nil + }, + leftovers: { retiredLeftovers() }, + log: { CMLogger.log($0) }) + } + + /// Unloads and deletes the retired agents. Idempotent: an agent that is + /// neither loaded nor on disk is skipped silently, so this runs on every + /// launch and costs a few `launchctl print` calls. Nothing else is deleted - + /// what the old layout left behind is only listed in the log, once, when + /// an agent was actually removed. + @discardableResult + static func removeRetiredAgents(_ probes: RetiredProbes) async -> [String] { + var removed: [String] = [] + for name in retired { + let onDisk = probes.plistExists(name) + let loaded = await probes.isLoaded(name) + guard onDisk || loaded else { continue } + if loaded { await probes.bootout(name) } + if onDisk && !probes.removePlist(name) { + probes.log("Retired launchd agent \(name): unloaded, but its plist could not be deleted") + continue + } + probes.log("Removed the retired launchd agent \(name) (disk-image layout)") + removed.append(name) + } + if !removed.isEmpty { + for line in probes.leftovers() { + probes.log("Left from the disk-image layout, remove by hand when no longer needed: \(line)") + } + } + return removed + } + + @discardableResult + public static func removeRetiredAgents() async -> [String] { + await removeRetiredAgents(.live) + } + + /// What the disk-image layout may have left on this Mac. Listed, never + /// deleted: the cache can hold bands that never reached Drive, and the old + /// image on Drive is a backup someone may still want to restore from. + static func retiredLeftovers( + home: URL = FileManager.default.homeDirectoryForCurrentUser, + mounted: [String] = mountedPaths(), + exists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) -> [String] { + var lines: [String] = [] + for path in ["/Volumes/CloudMachine", home.appendingPathComponent(".cloudmachine/drive").path] + where mounted.contains(path) { + lines.append( + "\(path) is still mounted - detach it (hdiutil detach / umount) before deleting anything") + } + for name in ["cache", "drive", "fuse", "run", "rclone.log", "rclone.log.1"] { + let path = home.appendingPathComponent(".cloudmachine/\(name)").path + if exists(path) { lines.append(path) } + } + let known = CMPaths.appSupportDir.appendingPathComponent("known-image").path + if exists(known) { lines.append(known) } + if exists("/usr/local/lib/libfuse-t.dylib") { + lines.append("/usr/local/lib/libfuse-t.dylib (FUSE-T, only if nothing else uses it)") + } + lines.append( + "\(DriveFolder.remoteName):\(DriveFolder.remoteRoot)/\(DriveFolder.name) on Google Drive (the old disk image; the machine uses \(VMLayout.remote))" + ) + return lines + } + + /// Mount points from the kernel's table, without waiting on any of them + /// (`MNT_NOWAIT`): a dead FUSE mount must not hang this check. + static func mountedPaths() -> [String] { + let count = getfsstat(nil, 0, MNT_NOWAIT) + guard count > 0 else { return [] } + var buffer = [Darwin.statfs](repeating: Darwin.statfs(), count: Int(count)) + let size = Int32(MemoryLayout.stride * Int(count)) + let filled = getfsstat(&buffer, size, MNT_NOWAIT) + guard filled > 0 else { return [] } + return buffer.prefix(Int(filled)).map { entry in + var mountPoint = entry.f_mntonname + return withUnsafePointer(to: &mountPoint) { + $0.withMemoryRebound(to: CChar.self, capacity: Int(MAXPATHLEN)) { String(cString: $0) } + } + } + } + // MARK: - After an upgrade static var stampFile: URL { @@ -114,10 +229,12 @@ public enum AgentRepair { current != stamp?.trimmingCharacters(in: .whitespacesAndNewlines) } - /// Called when the app starts: after an upgrade, reload the agents that are - /// safe to restart, then repair any that cannot start. Does nothing for a - /// build run outside a bundle (`swift run`), which has no version to compare. + /// Called when the app starts: remove the retired layout's agents, after an + /// upgrade reload the agents that are safe to restart, then repair any that + /// cannot start. The reload is skipped for a build run outside a bundle + /// (`swift run`), which has no version to compare. public static func afterLaunch() async { + await removeRetiredAgents() guard let version = AppVersionReader.current() else { return } let current = version.summary let stamp = try? String(contentsOf: stampFile, encoding: .utf8) diff --git a/mac-app/Sources/CloudMachineCore/AttachStall.swift b/mac-app/Sources/CloudMachineCore/AttachStall.swift deleted file mode 100644 index 155755f..0000000 --- a/mac-app/Sources/CloudMachineCore/AttachStall.swift +++ /dev/null @@ -1,156 +0,0 @@ -import Foundation - -/// The way out of an attach that `hdiutil` keeps turning away with "Resource -/// temporarily unavailable" - by restarting the Drive mount, which until now -/// only a restart of the Mac did. -/// -/// 24-26 Oct 2026: the mount wedged during a backup, and from then on every -/// attach of the `gdrive-attach` agent (every 15-20 min, 5 `hdiutil` attempts -/// each) failed with EAGAIN for 26 h. rclone was idle (queue 0, nothing in -/// use) and answered about its queue, so neither "rclone busy" nor "rclone -/// does not answer" describes it - the only signal was the series of EAGAINs. -/// After the restart of the Mac a fresh rclone let the image attach within -/// minutes. 37 h without a backup. -/// -/// The restart itself is `MountRestart` - the same path as the button in the -/// window, with all its refusals: not while Time Machine is backing up, not -/// before the queue has drained. This type only decides WHEN: -/// - a series: `minFailures` EAGAINs in a row, spanning at least -/// `minDuration`, in the same boot session - one or two refusals just after -/// a start-up are normal (26.10 08:27, a minute after rclone came up); -/// - at most one restart per `cooldown`: if a fresh rclone does not help, a -/// second one will not either, and restarting it hourly would only cut -/// the upload short. -public enum AttachStall { - - public static let minFailures = 3 - public static let minDuration: TimeInterval = 30 * 60 - public static let cooldown: TimeInterval = 6 * 3600 - - public static var recordURL: URL { - CMPaths.appSupportDir.appendingPathComponent("attach-stall") - } - - /// The series so far. `restartedAt == 0` = no restart in this boot session. - public struct Record: Equatable { - public var bootTime: TimeInterval - public var firstFailureAt: TimeInterval - public var failures: Int - public var restartedAt: TimeInterval - - public var text: String { - "\(Int(bootTime)) \(Int(firstFailureAt)) \(failures) \(Int(restartedAt))" - } - - public init( - bootTime: TimeInterval, firstFailureAt: TimeInterval, failures: Int, - restartedAt: TimeInterval - ) { - self.bootTime = bootTime - self.firstFailureAt = firstFailureAt - self.failures = failures - self.restartedAt = restartedAt - } - - public init?(text: String) { - let parts = text.split(whereSeparator: \.isWhitespace).compactMap { TimeInterval($0) } - guard parts.count == 4 else { return nil } - self.init( - bootTime: parts[0], firstFailureAt: parts[1], failures: Int(parts[2]), - restartedAt: parts[3]) - } - } - - /// A record from another boot session describes a mount that no longer - /// exists - it starts from scratch, like `ShutdownPrep.isActive`. - static func current(_ record: Record?, bootTime: TimeInterval) -> Record? { - guard let record, abs(record.bootTime - bootTime) <= 1 else { return nil } - return record - } - - /// One more EAGAIN in the series. - static func addingFailure(to record: Record?, bootTime: TimeInterval, now: TimeInterval) - -> Record - { - guard var series = current(record, bootTime: bootTime) else { - return Record(bootTime: bootTime, firstFailureAt: now, failures: 1, restartedAt: 0) - } - if series.failures == 0 { series.firstFailureAt = now } - series.failures += 1 - return series - } - - /// Any other outcome of the attach ends the series; the time of the last - /// restart stays, so the cooldown holds across a short success. - static func endingSeries(_ record: Record?, bootTime: TimeInterval, now: TimeInterval) - -> Record? - { - guard let series = current(record, bootTime: bootTime), series.restartedAt > 0 else { - return nil - } - return Record( - bootTime: bootTime, firstFailureAt: now, failures: 0, restartedAt: series.restartedAt) - } - - static func shouldRestart(_ record: Record?, bootTime: TimeInterval, now: TimeInterval) -> Bool { - guard let series = current(record, bootTime: bootTime) else { return false } - guard series.failures >= minFailures, now - series.firstFailureAt >= minDuration else { - return false - } - return series.restartedAt == 0 || now - series.restartedAt >= cooldown - } - - /// After a restart that ran (whatever its outcome) a new series starts, and - /// the cooldown with it. A refusal (`didNotRun`) touched nothing and counts - /// for nothing - the next failure asks again. - static func afterRestart(_ record: Record, restart: CMActionResult, now: TimeInterval) -> Record { - guard !restart.didNotRun else { return record } - return Record(bootTime: record.bootTime, firstFailureAt: now, failures: 0, restartedAt: now) - } - - /// What the attach agent does with the outcome of `BackupImageService.attach`. - /// The time and the restart are replaceable so the decision can be tested - /// without launchd. - static func handle( - _ attach: CMActionResult, record: Record?, bootTime: TimeInterval, now: TimeInterval, - restart: () async -> CMActionResult - ) async -> (record: Record?, restart: CMActionResult?) { - // "Busy with another operation" says nothing about the mount - the - // series neither grows nor ends. - if attach.didNotRun { return (record, nil) } - guard attach.attachTemporarilyUnavailable else { - return (endingSeries(record, bootTime: bootTime, now: now), nil) - } - let series = addingFailure(to: record, bootTime: bootTime, now: now) - guard shouldRestart(series, bootTime: bootTime, now: now) else { return (series, nil) } - let restarted = await restart() - return (afterRestart(series, restart: restarted, now: now), restarted) - } - - // MARK: - Live - - /// Called by `attach-image` after every attach. Returns the restart's - /// result when one was attempted. - public static func afterAttach(_ attach: CMActionResult) async -> CMActionResult? { - // No boot time - no way to tell this session's series from an old one. - // Doing nothing is the old behaviour, not a new risk. - guard let boot = ShutdownPrep.bootTime() else { return nil } - let stored = (try? String(contentsOf: recordURL, encoding: .utf8)).flatMap(Record.init(text:)) - let outcome = await handle( - attach, record: stored, bootTime: boot, now: Date().timeIntervalSince1970, - restart: { - CMLogger.log( - "Attach refused with EAGAIN \(minFailures)+ times over \(Int(minDuration / 60))+ min - restarting the Drive mount" - ) - let result = await MountRestart.perform() - CMLogger.log("Mount restart after the stalled attach: \(result.message)") - return result - }) - if let record = outcome.record { - try? record.text.write(to: recordURL, atomically: true, encoding: .utf8) - } else { - try? FileManager.default.removeItem(at: recordURL) - } - return outcome.restart - } -} diff --git a/mac-app/Sources/CloudMachineCore/BackupHealth.swift b/mac-app/Sources/CloudMachineCore/BackupHealth.swift index 0d70e1a..dff0aec 100644 --- a/mac-app/Sources/CloudMachineCore/BackupHealth.swift +++ b/mac-app/Sources/CloudMachineCore/BackupHealth.swift @@ -2,10 +2,10 @@ import Foundation /// Answers one question: is the hourly cycle STILL working. /// -/// The rest of this project measures the momentary state - whether the mount -/// is up, whether the image is attached, how much is waiting in the queue. None -/// of these measurements detects the most dangerous failure of this system: -/// everything looks mounted and attached, and Time Machine has not finished a +/// The rest of this project measures the momentary state - whether the +/// machine runs, whether its services are up, how much is waiting in the +/// queue. None of these measurements detects the most dangerous failure of +/// this system: everything looks up, and Time Machine has not finished a /// single backup for two days. The interface then shows a green badge and the /// word "Ready". /// @@ -18,18 +18,12 @@ import Foundation /// are looking for. /// /// We read the local file, not `tmutil latestbackup`. This is not an -/// optimization: `tmutil latestbackup` mounts a snapshot on the volume that -/// lives on Google Drive and, with a sick mount, can hang in uninterruptible -/// I/O. A watchdog that hangs exactly when it should raise the alarm is worse -/// than none. -/// -/// Until 23.09.2026 that sentence was a declaration, not a fact: -/// `currentReport()` calls `tmutil destinationinfo` (for the Time Machine -/// destination), and that read reaches the mount and hung WITHOUT A TIME LIMIT -/// in uninterruptible I/O exactly like `latestbackup`, which this comment warns -/// against. Since that date every tmutil call has a hard limit -/// (`TimeMachineStatus.commandTimeout`), and no answer is reported as a FAILURE -/// - not as "destination changed" and not as silence. +/// optimization: `tmutil latestbackup` mounts the backup volume and, with a +/// sick destination, can hang in uninterruptible I/O. A watchdog that hangs +/// exactly when it should raise the alarm is worse than none. Every tmutil +/// call has a hard limit (`TimeMachineStatus.commandTimeout`), and so does +/// every question to the machine; no answer is reported as a FAILURE - not as +/// "destination changed" and not as silence. public enum BackupHealth { public static let preferencesPath = "/Library/Preferences/com.apple.TimeMachine.plist" @@ -39,28 +33,26 @@ public enum BackupHealth { /// /// The cycle is hourly, so three hours are three missed runs in a row - too /// many to be chance. At the same time it leaves headroom for a backup that - /// takes long, and for the buffer guard, which deliberately pauses Time + /// takes long, and for the machine's guard, which deliberately pauses Time /// Machine while the upload catches up. public static let maxAgeHours = 3.0 - /// For this many minutes after system startup "the mount / image / - /// destination is not up yet" is NOT a failure. + /// For this many minutes after system startup "the machine / its services / + /// the destination is not up yet" is NOT a failure. /// /// The `backup-health` agent has `RunAtLoad`, so it starts together with the - /// session - a few seconds after rclone has only just started. After every - /// restart (21.09, 25.09, 01.10.2026) the watchdog then reported "BACKUP - /// FAILURE: Google Drive mount is not working" 4-8 s after login, before - /// anything had a chance to come up. An alarm that fires on every startup - /// teaches people to ignore it - and then the one real alarm is lost. + /// session - the same moment `vm-start` boots the machine. With the old + /// layout the watchdog reported a failure 4-8 s after every login (21.09, + /// 25.09, 01.10.2026), before anything had a chance to come up. An alarm + /// that fires on every startup teaches people to ignore it - and then the + /// one real alarm is lost. /// - /// 20 min, because that is what was measured in the worst case: on - /// 01.10.2026, after a restart with ~19 GB of unsent bands, rclone was loading - /// and uploading the backlog until 15:42, and the image attached at 15:49 - - /// 17 min after the agents started. ONLY the devices' `false` states are - /// deferred. The age of the last successful backup, RESULT, upload errors and - /// "unknown" (a hung read) alarm from the first second, and the next - /// watchdog run (every 30 min) already falls after this window and will - /// report every state that did not fix itself. + /// 20 min: a boot of the machine with a backlog in rclone's buffer takes + /// minutes before Samba answers. ONLY the `false` states are deferred. The + /// age of the last successful backup, RESULT, upload errors and "unknown" + /// (a hung read) alarm from the first second, and the next watchdog run + /// (every 30 min) already falls after this window and will report every + /// state that did not fix itself. public static let startupGraceMinutes = 20.0 /// How many seconds have passed since system startup (`kern.boottime`). @@ -131,23 +123,29 @@ public enum BackupHealth { // MARK: - Reading the counter of successful backups - /// Dates from the Time Machine preferences for the destination at the given - /// mount point. Pure function - takes an already read dictionary, so it can + /// Dates from the Time Machine preferences for this Mac's CloudMachine + /// destination. Pure function - takes an already read dictionary, so it can /// be tested without the system file and without Time Machine. /// + /// The destination is the one named after the share (`LastKnownVolumeName`), + /// else the one whose `NetworkURL` is the machine's share, else the only one + /// there is. + /// /// `result` is the `RESULT` field from the same block: 0 means the last run /// ended well, anything else - that it did not. public static func dates( - inPreferences plist: [String: Any], volumeNamed volumeName: String + inPreferences plist: [String: Any], volumeNamed volumeName: String = VMLayout.shareName ) -> (lastSuccess: Date?, lastAttempt: Date?, result: Int?) { guard let destinations = plist["Destinations"] as? [[String: Any]] else { return (nil, nil, nil) } - // We pick the destination by volume name, not by index 0 - a Mac can have - // several Time Machine destinations registered, and we care only about - // this one. + // We pick the destination, not index 0 - a Mac can have several Time + // Machine destinations registered, and we care only about this one. let destination = destinations.first { ($0["LastKnownVolumeName"] as? String) == volumeName } + ?? destinations.first { + ($0["NetworkURL"] as? String).map { VMStatus.isMachineShare($0, address: nil) } ?? false + } ?? (destinations.count == 1 ? destinations[0] : nil) guard let destination else { return (nil, nil, nil) } @@ -160,25 +158,26 @@ public enum BackupHealth { /// Assessment of the state. Pure function - every input is passed in /// directly, so injecting a KNOWN BAD sample (an old date, a non-zero - /// RESULT, a dead mount) is one call in a test, not breaking production. + /// RESULT, a stopped machine) is one call in a test, not breaking production. + /// + /// `nil` means "no answer" for every three-state input: it is reported as + /// its own problem, never folded into "fine" or into "broken". public static func evaluate( lastSuccess: Date?, lastAttempt: Date?, result: Int?, now: Date, - mounted: Bool?, - attached: Bool?, + machineRunning: Bool?, + servicesHealthy: Bool?, destinationRegistered: Bool?, erroredFiles: Int, outOfSpace: Bool, queueReadable: Bool, driveFreeBytes: UInt64? = nil, localFreeGB: Int? = nil, - imageDeadErrno: Int32? = nil, - // The image IS in the mount table, but the readability probe did not - // return. Goes together with `attached: nil` and serves ONLY to tell the - // person exactly what we do not know - the decision is the same. - imageProbeTimedOut: Bool = false, + machineFreeGB: Int? = nil, + punchfsNearFileLimit: Bool = false, + remoteControlOpen: Bool = false, maxAgeHours: Double = BackupHealth.maxAgeHours, // Whether the startup grace period is in progress - see `startupGraceMinutes`. withinStartupGrace: Bool = false, @@ -187,99 +186,57 @@ public enum BackupHealth { ) -> Report { var problems: [Problem] = [] var deferred: [Problem] = [] - // A device state that is NORMAL right after startup, because nothing has - // had time to come up yet. After the grace period it is an ordinary - // failure. + // A state that is NORMAL right after startup, because nothing has had + // time to come up yet. After the grace period it is an ordinary failure. func notReadyYet(_ problem: Problem) { if withinStartupGrace { deferred.append(problem) } else { problems.append(problem) } } - // Order from cause to effect: if the mount is down, the backup age will - // grow anyway, but it is the mount that has to be fixed. - // - // `mounted` and `attached` are THREE-STATE for the same reason as - // `destinationRegistered` below: reading the mount table can fail, and - // then we know neither that it is there nor that it is not. Collapsing - // that into a `Bool` ended in one of two ways and both were wrong - - // `?? false` produced an alarm about an unmounted Drive that may be - // mounted, and `!= .detached` produced SILENCE about an image we know - // nothing about. - switch mounted { - case .some(true): - break - case .some(false): - notReadyYet( - Problem( - summary: L10n.tr("Google Drive mount is not working"), - detail: L10n.tr( - "Without it the backup image is unreachable and Time Machine has nowhere to write."), - code: "drive-not-mounted")) - case .none: - problems.append( - Problem( - summary: L10n.tr("Unknown whether the Google Drive mount is working"), - detail: L10n.tr( - "Could not read the mount table. That does not mean the Drive is unmounted - it means nobody has checked. Without this answer there is no way to tell whether backups have anywhere to go." - ), - code: "drive-mount-unknown")) - } - - switch attached { + // Order from cause to effect: if the machine is down, the backup age will + // grow anyway, but it is the machine that has to be fixed. + switch machineRunning { case .some(true): - if let errno = imageDeadErrno { - // Attached but dead - a state that until 22 Sep 2026 did not exist for - // any sensor and therefore lasted 15 hours. See `ImageProbe`. + switch servicesHealthy { + case .some(true): + break + case .some(false): + notReadyYet( + Problem( + summary: L10n.tr("A service in the CloudMachine machine is not running"), + detail: L10n.tr( + "Samba, rclone or punchfs in the machine is not active, so Time Machine has nowhere to write or the bands do not reach Google Drive. Check: cloudmachine-agent vm-status" + ), + code: "machine-service-down")) + case .none: problems.append( Problem( - summary: L10n.tr("The backup image is attached, but DEAD (errno %@)", "\(errno)"), + summary: L10n.tr("The CloudMachine machine does not answer"), detail: L10n.tr( - "The image device stopped returning data - Time Machine sees it as a disconnected disk. Fix: cloudmachine-agent attach-image (force-detaches and attaches again)." + "Lima says the machine is running, but a command in it did not return. Without it nothing is known about Samba, rclone or the upload. Check: cloudmachine-agent vm-status" ), - code: "image-dead")) + code: "machine-no-answer")) } case .some(false): notReadyYet( Problem( - summary: L10n.tr("The backup image is not attached"), + summary: L10n.tr("The CloudMachine machine is not running"), detail: L10n.tr( - "Time Machine cannot see the destination %@.", BackupImageService.targetPath.path), - code: "image-detached")) + "Time Machine backs up to the machine, so without it no backup is made. Start it: limactl start %@ (or the Start button in the app).", + VMLayout.instanceName), + code: "machine-stopped")) case .none: - // THAT silence. Until 23.09.2026 the caller passed `attachment != - // .detached` here, so the new `.unknown` case ("the mount table could - // not be read") fell into `true` - i.e. "attached". The watchdog, whose - // ONLY job is not to claim things it does not know, stayed silent about - // a state it did not know. The message must be DIFFERENT from a real - // detachment: "is not attached" sends the person off to attach an image - // that may be attached correctly. - // Two causes of "I do not know" and TWO different messages, because they - // send the person to two different places. The third point at which the - // same distinction saves this report - see `mounted` above and - // `destinationRegistered` below. - if imageProbeTimedOut { - problems.append( - Problem( - summary: L10n.tr("Unknown whether the backup image returns data"), - detail: L10n.tr( - "The image %@ is listed in the mount table, but the readability probe did not answer within %@ s - that is how a read blocked on a dead FUSE-T mount behaves. This is NOT proof that the image is dead, so do NOT force-detach it: `attach-image` deliberately does nothing in that case, because detaching a live device abandons data waiting to be uploaded. First check whether rclone responds (cloudmachine-agent drive-status) and whether the gdrive-buffer agent is alive.", - BackupImageService.targetPath.path, "\(Int(ImageProbe.probeTimeout))"), - code: "image-probe-timed-out")) - } else { - problems.append( - Problem( - summary: L10n.tr("Unknown whether the backup image is attached"), - detail: L10n.tr( - "Could not read the mount table, so the state of the image %@ is UNKNOWN. Do not attach it blindly - first check whether `mount` responds at all (with a dead FUSE-T mount it can hang).", - BackupImageService.targetPath.path), - code: "image-attachment-unknown")) - } + problems.append( + Problem( + summary: L10n.tr("Unknown whether the CloudMachine machine is running"), + detail: L10n.tr( + "limactl did not answer (or is not installed). That does not mean the machine is down - it means nobody has checked." + ), + code: "machine-unknown")) } - // `nil` is NOT the same as `false`. Since 23.09.2026 `tmutil` has a time - // limit (see `TimeMachineStatus.commandTimeout`), so with a dead mount the - // watchdog comes back with no answer instead of hanging. No answer is a - // FAILURE - but a different one from a changed destination, and it has to - // sound different, so as not to send the person off to change something - // that is set correctly. + + // `nil` is NOT the same as `false`: no answer from tmutil has to sound + // different from a changed destination, so as not to send the person off + // to change something that is set correctly. switch destinationRegistered { case .some(true): break @@ -288,14 +245,15 @@ public enum BackupHealth { Problem( summary: L10n.tr("Time Machine does not point to CloudMachine"), detail: L10n.tr( - "The backup destination was changed or unregistered - backups are not being made."), + "No Time Machine destination is the machine's share (smb://tm@
/CloudMachine) - backups are not being made there. The command is printed by: cloudmachine-agent vm-setup" + ), code: "destination-not-registered")) case .none: problems.append( Problem( summary: L10n.tr("tmutil is not responding - unknown where the backup goes"), detail: L10n.tr( - "Reading the Time Machine destination did not return within %@ s. That is how tmutil behaves when blocked on a dead Google Drive mount. Fix: cloudmachine-agent attach-image, and if that does not help - restart the gdrive-buffer agent.", + "Reading the Time Machine destination did not return within %@ s. That is how tmutil behaves when its destination hangs.", "\(Int(TimeMachineStatus.commandTimeout))"), code: "tmutil-no-answer")) } @@ -326,12 +284,10 @@ public enum BackupHealth { // The age of the last success alone will not show that until it crosses // the threshold. // - // Exception: a run that is STILL IN PROGRESS. After a restart Time Machine - // can walk the whole disk (01.10.2026: 882 GB, 3.6 million files, ~4 h), - // and after an hour the watchdog then reported "the attempt did not end - // in a backup" about an attempt that simply had not finished yet. A run - // stuck forever will be caught by the last-successful-backup age - // threshold above anyway. + // Exception: a run that is STILL IN PROGRESS. A first or a long backup can + // take hours, and after an hour the watchdog then reported "the attempt did + // not end in a backup" about an attempt that simply had not finished yet. + // A run stuck forever will be caught by the age threshold above anyway. if let lastAttempt, let lastSuccess, lastAttempt > lastSuccess, now.timeIntervalSince(lastAttempt) > 3600, backupRunning != true { @@ -359,7 +315,7 @@ public enum BackupHealth { Problem( summary: L10n.tr("rclone failed to upload %@ files", "\(erroredFiles)"), detail: L10n.tr( - "These image bands exist only locally. The backup on Google Drive is INCOMPLETE and may not open." + "These bands exist only in the machine's buffer. The backup on Google Drive is INCOMPLETE and may not open." ), code: "upload-errors")) } @@ -372,24 +328,41 @@ public enum BackupHealth { ), code: "buffer-out-of-space")) } - // `mounted == true`, not `mounted != false`: when there is no mount OR it - // is unknown whether there is one, harder messages above already say so, - // and a second message about the same thing only dilutes the first. - if !queueReadable && mounted == true { + // Only for a machine whose services answered: when it is down or silent, + // harder messages above already say so, and a second message about the + // same thing only dilutes the first. + if !queueReadable && machineRunning == true && servicesHealthy == true { problems.append( Problem( summary: L10n.tr("The rclone control interface is not responding"), detail: L10n.tr( - "Without it there is no way to check whether anything reached the Drive - the buffer guard is blind then." + "Without it there is no way to check whether anything reached the Drive - the machine's guard is blind then." ), code: "rclone-rc-no-answer")) } + if punchfsNearFileLimit { + problems.append( + Problem( + summary: L10n.tr("punchfs is close to its open-file limit"), + detail: L10n.tr( + "At the limit every new band fails with EMFILE and the backup stops. Check: cloudmachine-agent vm-status" + ), + code: "punchfs-file-limit")) + } + if remoteControlOpen { + problems.append( + Problem( + summary: L10n.tr("rclone's remote control is open on this Mac"), + detail: L10n.tr( + "127.0.0.1:5572 answers on the Mac: Lima forwards the machine's rclone remote control, which has no password. Any web page open in a browser can send it commands - including deleting the backup on Google Drive. Add the portForwards rule from vm/cmsmb.yaml to the machine and restart it (see vm/README.md)." + ), + code: "rc-port-open")) + } - // Space on the Drive. Running out of it is a FATAL error for rclone, so - // the mount disappears and Time Machine loses its destination - this has - // to be known EARLIER, not from a failure. The threshold is counted in - // cycles, not in percent: at a growth of ~600 MB per hour, 30 GB is about - // two weeks of headroom. + // Space on the Drive. Running out of it stops every upload, and the + // machine's buffer then fills up - this has to be known EARLIER, not from + // a failure. The threshold is counted in cycles, not in percent: at a + // growth of ~600 MB per hour, 30 GB is about two weeks of headroom. if let driveFreeBytes { let freeGB = Int(driveFreeBytes / 1_073_741_824) if freeGB < driveFreeWarningGB { @@ -397,18 +370,29 @@ public enum BackupHealth { Problem( summary: L10n.tr("Google Drive is running out of space (%@ GB)", "\(freeGB)"), detail: L10n.tr( - "Once it runs out, rclone exits with a storageQuotaExceeded error, the mount disappears and backups stop being made. At a growth of ~600 MB per hourly cycle that is about %@ days.", + "Once it runs out, rclone stops uploading, the machine's buffer fills up and backups stop being made. At a growth of ~600 MB per hourly cycle that is about %@ days.", "\(max(1, freeGB * 1024 / 600 / 24))"), code: "drive-low-space")) } } + if let machineFreeGB, machineFreeGB < machineFreeWarningGB { + problems.append( + Problem( + summary: L10n.tr( + "The machine's disk is running out of space (%@ GB)", "\(machineFreeGB)"), + detail: L10n.tr( + "The upload buffer lives on this disk. The guard pauses Time Machine before it fills; if the upload does not catch up, backups stay paused." + ), + code: "machine-low-space")) + } + if let localFreeGB, localFreeGB < localFreeWarningGB { problems.append( Problem( summary: L10n.tr("The Mac's disk is running out of space (%@ GB)", "\(localFreeGB)"), detail: L10n.tr( - "The upload buffer lives on this disk. When it fills up, the guard pauses Time Machine, and with no space left at all rclone has nowhere to put data waiting to be uploaded." + "The machine's disk is a file on this disk and grows as the upload buffer fills. With no space left the machine cannot write the buffer, and Time Machine's local snapshots have nowhere to go either." ), code: "local-low-space")) } @@ -419,10 +403,13 @@ public enum BackupHealth { /// Below this many GB free on Google Drive we report a problem. public static let driveFreeWarningGB = 30 - /// Below this many GB free locally we report a problem. Higher than the - /// buffer guard's pause threshold - the watchdog should warn before the - /// guard starts braking. + /// Below this many GB free on the Mac we report a problem: the machine's + /// disk file can still grow by most of its 200 GiB buffer. public static let localFreeWarningGB = 120 + /// Below this many GB free in the machine we report a problem. Above the + /// guard's pause threshold (`VMGuard.Thresholds.minFreeGiB`), so the + /// watchdog warns before the guard starts braking. + public static let machineFreeWarningGB = 50 // MARK: - Live reading @@ -535,71 +522,48 @@ public enum BackupHealth { ], lastSuccess: nil, lastAttempt: nil, preferencesReadable: false) } - let (lastSuccess, lastAttempt, result) = dates( - inPreferences: plist, volumeNamed: BackupImageService.volumeName) - - let stats = await DriveBufferService.queueStats() - // `attachmentReading()`, not `attachment()`: the readability probe has a - // time limit and, once it is exceeded, returns `.unknown`. The watchdog - // then FINISHES the run and reports the lack of knowledge - that is the - // whole difference compared with the state until 26.09.2026, in which - // this read had no limit, and `StartInterval 1800` without `KeepAlive` - // means launchd will NOT start a second instance while the first one is - // alive. A single hang therefore silenced the watchdog PERMANENTLY, and - // silence in this system looks identical to health. - let reading = await BackupImageService.attachmentReading() - let attachment = reading.attachment - var deadErrno: Int32? - if case .dead(let errno) = attachment { deadErrno = errno } - - // Three states, just like for the Time Machine destination below. - // - // We derive them from `attachment`, not from a second call to - // `BackupImageService.attachedState()` - the same mount-table read already - // gave `deadErrno` above, and two separate reads could diverge and - // produce a report describing two different moments. - let attached: Bool? - switch attachment { - // `.dead` is still an ATTACHED image - just a dead one, and that is a - // separate problem reported via `imageDeadErrno`. - case .attached, .dead: attached = true - case .detached: attached = false - case .unknown: attached = nil + let (lastSuccess, lastAttempt, result) = dates(inPreferences: plist) + + // The machine first: everything below it is asked through it. + let machineRunning: Bool? + do { + machineRunning = (try await VMLayout.instanceStatus()) == "Running" + } catch { + machineRunning = nil } - // Three states, not two: `noAnswer` (a hung tmutil) has no right to - // pretend to be "destination changed" - see `evaluate`. - let registered: Bool? - switch await TimeMachineStatus.destinationReading() { - case .mountPoint(let path): registered = (path == BackupImageService.targetPath.path) - case .none: registered = false - case .noAnswer: registered = nil + let report = machineRunning == true ? await VMStatus.currentReport() : nil + let reading = report?.reading + let servicesHealthy = report.flatMap { VMStatus.servicesHealthy($0.services) } + let address = machineRunning == true ? await VMLayout.currentAddress() : nil + let registered = await VMStatus.timeMachineTargetsMachine(address: address) + + var nearFileLimit = false + if let open = report?.punchfsOpenFiles, let limit = report?.punchfsFileLimit { + nearFileLimit = Double(open) > Double(limit) * VMStatus.fileWarningRatio } // Measuring free space can FAIL (statfs returns an error), and then - // `freeGB()` returns `nil`, not a made-up zero - see the comment on it. - let localFree = BufferGuardService.freeGB() + // `macFreeGB()` returns `nil`, not a made-up zero. + let localFree = macFreeGB() - var report = evaluate( + var health = evaluate( lastSuccess: lastSuccess, lastAttempt: lastAttempt, result: result, now: now, - // `mountedState()`, and NOT `isMounted` - the latter is - // `mountedState() ?? false`, i.e. it turns "I do not know" into "not - // working" and tells the person to fix a mount that may be fine. - mounted: DriveBufferService.mountedState(), - attached: attached, + machineRunning: machineRunning, + servicesHealthy: machineRunning == true ? servicesHealthy : nil, destinationRegistered: registered, - erroredFiles: stats?.erroredFiles ?? 0, - outOfSpace: stats?.outOfSpace ?? false, - queueReadable: stats != nil, + erroredFiles: reading?.buffer.erroredFiles ?? 0, + outOfSpace: reading?.buffer.outOfSpace ?? false, + queueReadable: reading?.buffer.unsentBytes != nil, // An unreadable Drive quota is NOT a separate alarm here: when rclone - // does not respond, harder signals above already say so, and a second - // message about the same thing only dilutes the first. - driveFreeBytes: (await DriveBufferService.remoteQuota())?.free, + // does not respond, harder signals above already say so. + driveFreeBytes: await driveFreeBytes(), localFreeGB: localFree, - imageDeadErrno: deadErrno, - imageProbeTimedOut: reading.probeTimedOut, + machineFreeGB: reading.map { Int($0.freeBytes / VMGuard.gib) }, + punchfsNearFileLimit: nearFileLimit, + remoteControlOpen: VMStatus.remoteControlOpenOnMac(), maxAgeHours: maxAgeHours, // The REAL clock, not `now`: tests substitute a `now` from the past, and // uptime computed from it would come out negative, i.e. "startup in @@ -607,9 +571,43 @@ public enum BackupHealth { withinStartupGrace: (systemUptime() ?? .infinity) < startupGraceMinutes * 60, backupRunning: await TimeMachineStatus.runningState()) - report.problems.append(contentsOf: unmeasuredLocalDiskProblems(localFreeGB: localFree)) - report.problems.append(contentsOf: await MachineBudget.currentProblems()) - return report + health.problems.append(contentsOf: unmeasuredLocalDiskProblems(localFreeGB: localFree)) + health.problems.append(contentsOf: await MachineBudget.currentProblems()) + return health + } + + /// Free space on the Mac's data volume, in GB. `nil` = NOT MEASURED (statfs + /// failed) - never a made-up zero, which would read as "the disk is full". + public static func macFreeGB() -> Int? { + var stats = statfs() + guard statfs("/System/Volumes/Data", &stats) == 0 else { return nil } + let available = UInt64(stats.f_bavail) * UInt64(stats.f_bsize) + return Int(available / 1_073_741_824) + } + + /// Free space on the Google account (`rclone about`), through the Mac's + /// rclone. `nil` = no answer, or an account without a limit. + static func driveFreeBytes() async -> UInt64? { + guard + let result = try? await CMTooling.runRclone( + ["about", "--json", "\(DriveFolder.remoteName):"], timeout: 120), + result.succeeded + else { return nil } + return driveFreeBytes(aboutJSON: result.stdout) + } + + static func driveFreeBytes(aboutJSON: String) -> UInt64? { + guard let data = aboutJSON.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] + else { return nil } + func bytes(_ key: String) -> UInt64? { + if let v = json[key] as? NSNumber, v.int64Value >= 0 { return UInt64(v.int64Value) } + return nil + } + // `total` is sometimes absent (accounts without a limit) - then there is nothing to watch. + guard let total = bytes("total"), total > 0 else { return nil } + let used = bytes("used") ?? 0 + return bytes("free") ?? (total > used ? total - used : 0) } /// Problem reported when the free-space measurement CANNOT be made. @@ -617,23 +615,18 @@ public enum BackupHealth { /// `evaluate` treats `localFreeGB: nil` as "not asked" (that has been its /// contract from the start, and a dozen or so tests rely on it), but /// `currentReport` KNOWS it asked and it did not work. That is a separate - /// failure: the buffer guard decides on pausing Time Machine precisely on - /// this number, so when it is missing, it no longer protects the disk from - /// filling up. + /// failure: the machine's disk is a file on this disk, and nobody then + /// knows whether it still has room to grow. /// /// Split out of `currentReport()` SOLELY so that it can be tested: - /// `currentReport()` touches rclone, tmutil and hdiutil, so this branch - /// would otherwise be untestable - and an "I do not know" branch nobody has - /// checked is exactly the kind of dead code the review asked about (the - /// compiler warned earlier that an `Int` compared to `nil` always yields - /// false, i.e. that the branch was dead). + /// `currentReport()` touches rclone, tmutil and the machine. static func unmeasuredLocalDiskProblems(localFreeGB: Int?) -> [Problem] { guard localFreeGB == nil else { return [] } return [ Problem( summary: L10n.tr("Cannot measure free space on the Mac's disk"), detail: L10n.tr( - "statfs('/System/Volumes/Data') returned an error. The buffer guard will then not pause Time Machine before the disk fills up, because it does not know the number it bases that decision on." + "statfs('/System/Volumes/Data') returned an error, so nobody knows whether the machine's disk can still grow on this Mac." ), code: "local-space-unmeasured") ] diff --git a/mac-app/Sources/CloudMachineCore/BackupImageService.swift b/mac-app/Sources/CloudMachineCore/BackupImageService.swift deleted file mode 100644 index 9d65be9..0000000 --- a/mac-app/Sources/CloudMachineCore/BackupImageService.swift +++ /dev/null @@ -1,946 +0,0 @@ -import Foundation - -/// The backup image living on Google Drive - a port of `gdrive/create-image.sh`, -/// `attach-image.sh` and `verify-image.sh`. -/// -/// Time Machine is handed an attached, ordinary APFS volume and does not know -/// that the image's bands live in the cloud. Thanks to that there is no network -/// file system in the write path - SMB is gone, and with it a whole class of -/// failures that plague network backups. -public enum BackupImageService { - - // MARK: - Paths - - /// Named after this Mac's Drive folder (`mac-studio` on installations that - /// predate per-Mac folders, where that is the image that already exists). - public static var imageName: String { DriveFolder.name } - public static let volumeName = "CloudMachine" - - public static var imagePath: URL { - DriveBufferService.mountPoint.appendingPathComponent("\(imageName).sparsebundle") - } - - /// The destination's mount point. - /// - /// `/Volumes` is a path Time Machine is sure to accept - and that is the only - /// reason it is here. The price: after an unclean detach the - /// `/Volumes/` directory is left behind orphaned and blocks the next - /// attach. It belongs to the user, but lives in root-owned `/Volumes`, so - /// `rmdir` refuses - an agent running as the user cannot clean up after - /// itself. The alternative (a directory wholly ours, self-healing) is - /// untested: it is unknown whether `tmutil setdestination` accepts a - /// destination outside `/Volumes`. - public static var targetPath: URL { - URL(fileURLWithPath: "/Volumes/\(volumeName)") - } - - /// 32 MB per band, in 512 B sectors. Chosen by measurement - see - /// `cloudmachine-poc amplification` and the table in `gdrive/README.md`. - /// - /// Two forces pull in opposite directions. Google Drive lets through about - /// two operations per file per second, so small bands lengthen the first - /// upload. But every change dirties the whole band, so large bands multiply - /// the transfer on every increment - measured 768 MB at 64 MB versus 384 MB - /// at 8 MB for the same 300 MB of real change. 32 MB is the point where the - /// first upload stops being limited by the operation rate and starts being - /// limited by link bandwidth. - /// - /// Only takes effect when the image is created - changing it later requires a - /// backup from scratch. - public static let bandSectors = 65536 - - private static let fsckPath = - "/System/Library/Filesystems/apfs.fs/Contents/Resources/fsck_apfs" - - // MARK: - Mutual exclusion - - /// Name of the lock under which ALL operations that change the image's state - /// run: `create`, `attach`, `detach`, `verify`. - /// - /// Until 23 September 2026 they did not exclude each other at all - `CMLock` - /// existed, but `withCMLock` was not called from a single place in the repo. - /// The real run that exposed it: `detach` waits for the drain (35 s pause for - /// the queue to fill + up to 600 s for quiet, i.e. a window of up to 10.5 - /// minutes), and the `gdrive-attach` agent ticks every 900 s. The agent - /// regularly hit that window, saw the image as detached - because - /// `hdiutil detach` had already gone through - and attached it back in the - /// middle of someone else's detach. - /// - /// A second variant of the same race: `purgeStaleDevices()` from the agent's - /// tick runs `hdiutil detach -force` on the device that `fsck_apfs` from - /// `verify` is running on at that moment - and `verify` reported "Image - /// INCONSISTENT" about the whole backup (see the comment at `verify`). - public static let lockName = "image" - - /// Result of an operation that DID NOT HAPPEN, because another operation - /// holds the image. - /// - /// `withCMLock` then returns `nil`, and that `nil` must not pass as success: - /// an `attach` that never happened would report "Attached", and a `detach` - /// that never happened - "everything uploaded to Google Drive". - /// - /// `operation` goes to the log, `displayName` to the person. - private static func busyResult(_ operation: String, displayName: String) -> CMActionResult { - CMLogger.log( - "\(operation): lock '\(lockName)' is held by another image operation - doing nothing") - return CMActionResult( - succeeded: false, - message: L10n.tr( - "%@: another image operation is in progress (creating, attaching, detaching or verifying) - NOTHING was done. Try again shortly.", - displayName), - // Not a failure, just "not now" - see `CMActionResult.Disposition`. - didNotRun: true) - } - - // MARK: - State - - public static var exists: Bool { - var isDir: ObjCBool = false - let ok = FileManager.default.fileExists(atPath: imagePath.path, isDirectory: &isDir) - return ok && isDir.boolValue - } - - /// Whether the volume is in the mount table. - /// - /// NOTE: this only says that `hdiutil` attached the image at some point - NOT - /// that the image returns data. A dead device (see `ImageProbe`) sits in that - /// table just like a live one. The question "does Time Machine have somewhere - /// to write" is answered by `attachment`; `isAttached` stays where only the - /// detach itself matters. - public static var isAttached: Bool { attachedState() ?? false } - - /// Like `isAttached`, but `nil` = the mount table COULD NOT be read. - /// - /// The same change as in `DriveBufferService.mountPoints()` and for the same - /// reason: until now this went through `/sbin/mount` without a time limit, - /// and it asks about a volume that is sometimes DEAD - that is, exactly the - /// one on which such a read can hang. Now it goes through the kernel table, - /// without a process and without touching the file system. - public static func attachedState() -> Bool? { - guard let points = DriveBufferService.mountPoints() else { return nil } - return points.contains(targetPath.path) - } - - public enum Attachment: Equatable, Sendable { - case detached - case attached - /// In the mount table, but reads fail with the given `errno`. Time Machine - /// sees this state as "disk disconnected" and will not make a single - /// backup until the image is detached and attached again. - case dead(errno: Int32) - /// NOTHING is known about the image's state. This is not `.detached`: - /// `.detached` is a claim ("I checked, it is not there"), while here there - /// was nothing to check. - /// - /// Two causes, both leading to the same decision (hold off, do not touch - /// the image): - /// - /// 1. The mount table could not be read. Since the switch to - /// `getmntinfo(MNT_NOWAIT)` extremely unlikely - it is a read from - /// kernel memory that has no way to hang or go to the network. - /// 2. The image IS in the mount table, but the readability probe did not - /// answer within `ImageProbe.probeTimeout` (since 26.09.2026 - before - /// that it did not answer forever and took the caller down with it). - /// - /// There is deliberately NO separate, fifth state for the second case: - /// every decision made on this type is identical in both cases, and - /// separating them would invite those paths to drift apart. Whoever writes - /// to a person and has to give the cause takes it from - /// `attachmentReading()`. - case unknown - - /// Whether Time Machine has somewhere to write. `.unknown` deliberately - /// gives `false` - the question is "CAN I rely on this", and you cannot - /// rely on an unknown. - public var isUsable: Bool { self == .attached } - } - - /// Attachment state taking into account whether the device is ALIVE. - /// - /// `async` rather than a computed property ever since the readability probe - /// got a time limit: waiting for it must not block the calling thread (see - /// `ImageProbe` - the panel's `@MainActor` and the monitor without - /// `KeepAlive` paid for that with a frozen interface and silence). - public static func attachment() async -> Attachment { - await attachmentReading().attachment - } - - /// Like `attachment()`, but ALSO says whether "I do not know" came from a - /// probe that did not answer in time. - /// - /// For the decision the difference does not matter (both cases hold off), - /// but for the MESSAGE it matters a lot: "could not read the mount table" - /// tells a person to check something entirely different than "the image is - /// in the table, but reads from it do not come back". - public static func attachmentReading() async -> (attachment: Attachment, probeTimedOut: Bool) { - switch attachedState() { - case .none: return (.unknown, false) - case .some(false): return (.detached, false) - case .some(true): - switch await ImageProbe.probe(volume: targetPath) { - case .dead(let errno): return (.dead(errno: errno), false) - case .readable, .nothingToProbe: return (.attached, false) - // NOT `.dead`: `.dead` means "the device answered with a device error", - // while here the device did not answer at all. - case .timedOut: return (.unknown, true) - } - } - } - - public static func describe(_ attachment: Attachment, probeTimedOut: Bool = false) -> String { - switch attachment { - case .detached: return L10n.tr("NOT ATTACHED") - case .attached: return "OK (\(targetPath.path))" - case .dead(let errno): - return L10n.tr( - "DEAD - in the mount table, but reads fail (errno %@); attach-image attaches it again", - "\(errno)") - case .unknown where probeTimedOut: - return L10n.tr( - "UNKNOWN - in the mount table, but the readability probe did not answer within %@ s", - "\(Int(ImageProbe.probeTimeout))") - case .unknown: - return L10n.tr("UNKNOWN - could not read the mount table") - } - } - - /// Mount points of browsed backup snapshots. - /// - /// A pure version, so that it can be tested without mounting anything - - /// previously the same thing came from parsing `/sbin/mount` output - /// (`" on "` ... `" ("`), so there was nothing to substitute a sample for. - static func browsedSnapshotMounts(_ points: [String]) -> [String] { - points.filter { $0.hasPrefix("/Volumes/.timemachine/") } - } - - // MARK: - Hung devices - - /// `/dev/diskN` devices attached to the given image. - /// - /// After a forced detach a device can remain in the system as a zombie. - /// A new attach then ends with a "no mountable file systems" error, or - - /// worse - returns a dead handle on which `fsck_apfs` reports "failed to read - /// container superblock" with an all-zero UUID. It looks like a deleted - /// backup, but is only an unreadable device: an earlier version of the - /// pull-the-floor test concluded three times in a row, on that basis, that - /// data was lost which was in fact intact. - public static func devicesForImage(_ image: URL = imagePath) async -> [String] { - guard let result = try? await ProcessRunner.run("/usr/bin/hdiutil", ["info"], timeout: 60), - result.succeeded - else { return [] } - return parseDevices(hdiutilInfo: result.stdout, imagePath: image.path) - } - - /// Pure version of the parser - `hdiutil info` groups entries into blocks, - /// where all `/dev/diskN` lines after an `image-path` line belong to it. - public static func parseDevices(hdiutilInfo: String, imagePath: String) -> [String] { - var devices: [String] = [] - var currentImage: String? - for line in hdiutilInfo.components(separatedBy: .newlines) { - if line.hasPrefix("image-path") { - currentImage = - line - .drop(while: { $0 != ":" }) - .dropFirst() - .trimmingCharacters(in: .whitespaces) - continue - } - guard line.hasPrefix("/dev/disk"), currentImage == imagePath else { continue } - let device = String(line.prefix(while: { !$0.isWhitespace })) - // We want the parent device (/dev/disk7), not the partition - // (/dev/disk7s1) - detaching the parent takes the partitions with it. - // - // NOTE: the tempting `!device.contains("s")` is WRONG, because "disk" - // contains an "s" too and it rejects everything. We check whether only - // digits are left after the prefix. - let suffix = device.dropFirst("/dev/disk".count) - guard !suffix.isEmpty, suffix.allSatisfy(\.isNumber) else { continue } - if !devices.contains(device) { - devices.append(device) - } - } - return devices - } - - public static func purgeStaleDevices(_ image: URL = imagePath) async { - for device in await devicesForImage(image) { - _ = try? await ProcessRunner.run( - "/usr/bin/hdiutil", ["detach", device, "-force", "-quiet"], timeout: 60) - } - } - - // MARK: - Creation - - /// Creates the image IN PLACE, on the mounted Drive. - /// - /// Creating it locally and moving it gives an image that `hdiutil` later - /// does not open ("CBSDBackingStore::newProbe stat() failed"), even though - /// all files and bands are in place and readable. - public static func create(sizeGB: Int) async -> CMActionResult { - await withCMLock(lockName) { await createLocked(sizeGB: sizeGB) } - ?? busyResult("Image creation", displayName: L10n.tr("Image creation")) - } - - private static func createLocked(sizeGB: Int) async -> CMActionResult { - switch DriveBufferService.mountedState() { - case .some(true): - break - case .some(false): - return CMActionResult( - succeeded: false, message: L10n.tr("Drive is not mounted - start the buffer first.")) - case .none: - // Not `isMounted`: creating the image is IRREVERSIBLE, so "I do not - // know" must not pass here as "not mounted", let alone go any further. - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not read the mount table - it is UNKNOWN whether the buffer is mounted. NOT creating the image." - )) - } - - // The "image already exists" guard reads the FUSE cache, and rclone - // exposes the mount BEFORE it loads the directory contents from Drive - in - // that window `exists` says "no image" about an image that is there. - // `attach-image` has waited here on `BufferReadiness.wait` since - // 13 September 2026, `create` did not wait at all. For `attach` missing the - // window costs one failed attach; for `create` - `hdiutil create` goes to - // the path of an existing backup, and with five retries at that. - // - // We wait for a SUCCESSFUL listing of the mount point, not just for - // `isMounted`: the guard above has already answered the latter, so the - // probe would pass immediately and the wait would do NOTHING. Listing the - // root with a cold `--dir-cache-time` goes to Google for data, so its - // success means "rclone is actually serving this directory"; until FUSE - // has started serving, it ends with a device error. - // - // NOTE on scope: this wait removes the "FUSE is not answering yet" window, - // but does NOT prove the image is absent - an empty listing looks the same - // for an empty account and for a directory not yet loaded. The proof is - // `remoteImagePresence()` below, and it, not this wait, holds back the - // irreversible operation. - let ready = await BufferReadiness.wait( - sleep: { seconds in - try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) - }, - probe: { - DriveBufferService.isMounted - && (try? FileManager.default.contentsOfDirectory( - atPath: DriveBufferService.mountPoint.path)) != nil - }) - guard ready else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The buffer did not come up within %@ min - NOT creating the image.", - "\(Int(BufferReadiness.defaultTimeout / 60))")) - } - - guard !exists else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The image already exists. Deleting it erases the whole backup - do it deliberately.")) - } - - // The FUSE cache has already lied once, so we ask the REMOTE again, - // bypassing the mount. This is an IRREVERSIBLE operation: lack of certainty - // must ABORT it, not just print a warning that nobody reads before - // confirming anyway. - switch await remoteImagePresence() { - case .absent: - if let refusal = absentImageVerdict(trace: InstallationTraces.knownImageTrace()) { - return refusal - } - case .present: - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The image already exists on Google Drive (the mount cache did not show it, but the remote has it). Deleting it erases the whole backup - do it deliberately." - )) - case .unknown(let why): - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not confirm on Google Drive that the image is not there yet (%@) - ABORTING. Creating the image over an existing backup is irreversible, so I do not start without that answer.", - why)) - } - - await DriveBufferService.waitUntilIdle(timeout: 180) - - let args = [ - "create", "-type", "SPARSEBUNDLE", - "-size", "\(sizeGB)g", - "-fs", "Case-sensitive APFS", - "-volname", volumeName, - "-imagekey", "sparse-band-size=\(bandSectors)", - imagePath.path, - ] - - let result = await retryingFlakyMount(attempts: 5) { - try? await ProcessRunner.run("/usr/bin/hdiutil", args, timeout: 600) - } - - guard result?.succeeded == true else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not create the image: %@", result?.stderr ?? L10n.tr("unknown error"))) - } - InstallationTraces.rememberImage() - return CMActionResult( - succeeded: true, - message: L10n.tr( - "Created a %@ GB image, band size %@ MB.", "\(sizeGB)", - "\(bandSectors * 512 / 1024 / 1024)")) - } - - /// What "Google Drive has no image" means when this Mac remembers one. - /// - /// The listing can be wrong without failing: a token with the `drive.file` - /// scope sees only what its own `client_id` created, so after the remote is - /// replaced (the panel used to suggest `configure-remote - /// --replace-existing` after every credential change) the existing image - - /// 632 GB on this project's own Mac - is simply not there. `create` then - /// made a second image next to it, Drive allowing duplicate names, and the - /// old backup sat orphaned, still using space. A remembered image that - /// cannot be seen is a question about the token, not a reason to start - /// over. `nil` = no trace, go ahead. - static func absentImageVerdict(trace: String?) -> CMActionResult? { - guard let trace else { return nil } - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Google Drive does not show the image %@, but this Mac has already used it (trace: %@). NOT creating a new one - that would start an empty backup and orphan the existing one. Most likely the token cannot see it: a connection with the 'drive.file' scope sees only files created under its own client_id. Check the token's permissions (scope and client_id of the remote in rclone.conf). Only if the image really was deleted on purpose: remove %@ and run again.", - "\(DriveBufferService.remoteName):\(DriveBufferService.remotePath)/\(imageName).sparsebundle", - trace, trace)) - } - - // MARK: - Image on the remote - - /// Whether the image is on Google Drive - asked WITHOUT going through the mount. - public enum RemotePresence: Equatable { - case present - case absent - /// Unknown. `why` goes into the message, so that the user knows what - /// exactly was missing. - case unknown(String) - } - - /// Asks `rclone lsf` directly about the remote backup directory. - /// - /// The point is that it bypasses the FUSE cache - and it is precisely the - /// FUSE cache that says "no image" during the first seconds after the mount - /// is exposed. - public static func remoteImagePresence() async -> RemotePresence { - let remote = "\(DriveBufferService.remoteName):\(DriveBufferService.remotePath)" - guard - let result = try? await CMTooling.runRclone(["lsf", "--dirs-only", remote], timeout: 120) - else { - return .unknown(L10n.tr("rclone did not answer")) - } - return classifyRemoteListing( - succeeded: result.succeeded, stdout: result.stdout, stderr: result.stderr) - } - - /// Pure version - so that it can be tested without a network and without an - /// account. - /// - /// A failed `lsf` with the message "directory not found" is NOT a missing - /// answer, but the answer "there is nothing there": that is what the first - /// run looks like, before anything has been uploaded to Drive. If we counted - /// it as `.unknown`, `create` could not run EVEN ONCE - the guard would block - /// exactly the case it exists for. - static func classifyRemoteListing(succeeded: Bool, stdout: String, stderr: String) - -> RemotePresence - { - if succeeded { - return listingContainsImage(stdout) ? .present : .absent - } - if stderr.lowercased().contains("directory not found") { return .absent } - let reason = stderr.trimmingCharacters(in: .whitespacesAndNewlines) - return .unknown( - reason.isEmpty ? L10n.tr("rclone lsf ended with an error") : String(reason.suffix(200))) - } - - /// `rclone lsf --dirs-only` ends directory names with a slash, but we do not - /// rely on it - we accept both forms. - static func listingContainsImage(_ listing: String) -> Bool { - let wanted = "\(imageName).sparsebundle" - return listing.components(separatedBy: .newlines) - .map { $0.trimmingCharacters(in: .whitespaces) } - .contains { $0 == wanted || $0 == wanted + "/" } - } - - // MARK: - Attaching - - public static func attach() async -> CMActionResult { - await withCMLock(lockName) { await attachLocked() } - ?? busyResult("Image attach", displayName: L10n.tr("Image attach")) - } - - static func attachLocked() async -> CMActionResult { - switch DriveBufferService.mountedState() { - case .some(true): - break - case .some(false): - return CMActionResult(succeeded: false, message: L10n.tr("Drive is not mounted.")) - case .none: - // Attaching the image on an UNMOUNTED buffer ends with an image hanging - // on an empty directory, so "I do not know" must hold off here, not let - // it through. The agent's tick will try again in 900 s. - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not read the mount table - it is UNKNOWN whether the buffer is mounted. NOT attaching the image." - )) - } - guard exists else { - return CMActionResult( - succeeded: false, message: L10n.tr("No image - create it first.")) - } - let reading = await attachmentReading() - switch reading.attachment { - case .attached: - InstallationTraces.rememberImage() - return CMActionResult( - succeeded: true, message: L10n.tr("Already attached: %@", targetPath.path)) - case .unknown: - // Not `.detached`, because the next step would be `hdiutil attach` on an - // image that may already be attached - and before that - // `purgeStaleDevices()`, i.e. `detach -force` on someone else's live - // device. "I do not know" must not trigger either of them. - // - // This ALSO applies to a probe that did not answer in time. The price is - // real and chosen deliberately: if the image really is dead and reads - // from it hang, this function will not fix it, and the agent's tick will - // try again in 900 s. The opposite mistake, however, is irreversible - - // `detach -force` on a slow but LIVE device drops writes that have not - // reached Drive. The `backup-health` monitor reports that the state is - // unknown; there is no silence here. - if reading.probeTimedOut { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The image is in the mount table, but the readability probe did not answer within %@ s - it is UNKNOWN whether the device is alive. NOT force-detaching and NOT attaching.", - "\(Int(ImageProbe.probeTimeout))")) - } - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not read the mount table - it is UNKNOWN whether the image is attached. NOT attaching." - )) - case .dead(let errno): - // The image is in the mount table, but returns no data. Until 22 Sep 2026 - // this function then said "Already attached" and returned - the attaching - // agent repeated that every 15 minutes for 15 hours, and Time Machine had - // no destination. The only way is to detach (it has to be `-force`, a - // plain one refuses on a dead device) and attach again. The wait for the - // upload stays: whatever made it into the rclone buffer still has to - // reach Drive. - CMLogger.log("Image dead (errno \(errno)) - force-detaching and attaching again") - // `detachLocked`, not `detach`: we already hold the 'image' lock, and - // `CMLock` is not reentrant - going through the public `detach` would see - // our own live lock and refuse itself. - let detached = await detachLocked(force: true) - CMLogger.log("Detaching the dead image: \(detached.message)") - guard !isAttached else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Image dead (errno %@) and could not be detached: %@", "\(errno)", detached.message)) - } - case .detached: - break - } - - await purgeStaleDevices() - - // An orphaned mount point blocks the attach. If it is in /Volumes, removing - // it needs root - so we say exactly what to run, instead of retrying - // forever. - if FileManager.default.fileExists(atPath: targetPath.path) { - do { - try FileManager.default.removeItem(at: targetPath) - } catch { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "An orphaned mount point blocks the attach: %@\nRemove it and try again: sudo rmdir '%@'", - targetPath.path, targetPath.path)) - } - } - - // A quiet queue is not a convenience here but a condition for success: - // `hdiutil` on a FUSE-T volume rejects the mount the more often, the busier - // rclone is (see `retryingFlakyMount`). With `writeBackSeconds` counted in - // minutes the queue will not empty by itself within the time limit below, - // so we force the upload first - otherwise attaching after every start-up - // would be a lottery. - // - // We wait as long as the upload makes progress, not a fixed 120 s - after a - // restart without `prepare-shutdown` the backlog reaches a dozen or more GB - // (see `UploadDrain`). - let drain = await UploadDrain.wait( - sleep: { try? await Task.sleep(nanoseconds: UInt64($0 * 1_000_000_000)) }, - expire: { _ = await DriveBufferService.expireQueuedUploads() }, - unsent: { await DriveBufferService.queueStats()?.unsentItems }) - switch drain { - case .idle: - break - case .stalled(let unsent): - CMLogger.log("Attach: upload stalled (\(unsent) items queued) - attaching anyway") - case .timedOut(let unsent): - CMLogger.log( - "Attach: backlog did not drain within \(Int(UploadDrain.defaultMaxTotal / 60)) min (\(unsent) items) - attaching anyway" - ) - case .noAnswer: - CMLogger.log("Attach: rclone does not answer about the queue state - attaching blind") - } - - let result = await retryingFlakyMount(attempts: 5) { - try? await ProcessRunner.run( - "/usr/bin/hdiutil", - ["attach", imagePath.path, "-nobrowse", "-mountpoint", targetPath.path], - timeout: 300) - } - - guard result?.succeeded == true else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not attach the image: %@", result?.stderr ?? L10n.tr("unknown error")), - attachTemporarilyUnavailable: isTemporarilyUnavailable(stderr: result?.stderr ?? "")) - } - InstallationTraces.rememberImage() - return CMActionResult(succeeded: true, message: L10n.tr("Attached: %@", targetPath.path)) - } - - /// `hdiutil attach` turned away with EAGAIN. On 25-26 Oct 2026 every attach - /// ended like this for 26 h, with rclone idle (queue 0) and answering; only - /// a fresh rclone - a restart of the Mac - let the image attach. The text is - /// `strerror(EAGAIN)`, which `hdiutil` prints in English whatever the system - /// language. - static func isTemporarilyUnavailable(stderr: String) -> Bool { - stderr.contains("Resource temporarily unavailable") - } - - /// Detaches the image and WAITS until everything has reached Google Drive. - /// - /// The wait is not extra caution. The detach itself writes APFS metadata to - /// the bands, and `--vfs-write-back` delays uploading them by tens of - /// seconds. Losing the buffer in that window does not cost "the latest - /// changes" - it takes the volume's root directory. Observed live: 367 MiB of - /// bands were already on Drive, and the image was empty after re-attaching, - /// because three bands with metadata were killed in the queue. - /// - /// That is why every shutdown path - detach, stopping the buffer, turning off - /// the Mac - must let the drain run to the end. - public static func detach(force: Bool = false, waitForUpload: Bool = true) async -> CMActionResult - { - await withCMLock(lockName) { await detachLocked(force: force, waitForUpload: waitForUpload) } - ?? busyResult("Image detach", displayName: L10n.tr("Image detach")) - } - - static func detachLocked(force: Bool = false, waitForUpload: Bool = true) async - -> CMActionResult - { - let stillMounted = await unmountBrowsedSnapshots() - - var args = ["detach", targetPath.path, "-quiet"] - if force { args.append("-force") } - let result = try? await ProcessRunner.run("/usr/bin/hdiutil", args, timeout: 120) - guard result?.succeeded == true else { - // We give the reason if we know it. `hdiutil` only says "resource busy" - // and not a word about what holds the device - and that is almost always - // a browsed backup snapshot. - guard stillMounted.isEmpty else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not detach - the image is held by browsed backup snapshots that could not be unmounted:\n%@\nClose the Time Machine / Finder window on the backup and try again.", - stillMounted.joined(separator: "\n"))) - } - return CMActionResult(succeeded: false, message: L10n.tr("Could not detach.")) - } - - guard waitForUpload else { - return CMActionResult( - succeeded: true, - message: L10n.tr( - "Detached (without waiting for the upload - the data may be local only).")) - } - - // Writes from the detach must reach the queue first - without this pause - // it would look empty, because it would not have had time to fill yet. - // - // NOTE on the mechanism: an item appears in the queue RIGHT after the - // write, only with an upload deadline `writeBackSeconds` ahead (visible in - // `vfs/queue` as a positive `expiry`). So this pause waits for the queuing - // itself, NOT for that deadline to pass - the earlier comment here claimed - // the opposite. - try? await Task.sleep(nanoseconds: 35_000_000_000) - - // We move the deadlines only now, when the queue is complete. Without this - // the drain would take as long as `writeBackSeconds` (ten minutes), i.e. - // longer than the time limit below - and the detach would report failure - // every time. - CMLogger.log(expiryLogLine(await DriveBufferService.expireQueuedUploads())) - return detachVerdict(settled: await DriveBufferService.statsWhenIdle(timeout: 600)) - } - - /// What the detach writes to the log after trying to speed up the queue. - /// - /// THREE different things looked like two here. "We got no answer" was told - /// apart from "the queue was empty" on 23.09.2026, but the third case - the - /// queue FULL and every `vfs/queue-set-expiry` failed - still came out of - /// `expireQueuedUploads` as `0`, and the log reported "queue empty". The - /// measured state of this machine at the time of the audit: 462 items queued. - /// - /// The failure mode is worse than just an untruth in the log: a person reads - /// this line exactly when deciding whether the buffer may be deleted. "Queue - /// empty" reads as "nothing is waiting to be uploaded", while it meant "462 - /// items are waiting and not one of them could be moved". - /// - /// Split out and PURE, so that these three cases can be tested without - /// rclone. The function is only a description: waiting for the drain and the - /// verdict are decided by `detachLocked`/`detachVerdict`, and this fix does - /// not touch them. - static func expiryLogLine(_ outcome: DriveBufferService.ExpiryOutcome?) -> String { - let drain = "the drain may take up to \(DriveBufferService.writeBackSeconds / 60) min" - guard let outcome else { - // No answer is NOT an empty queue - see `expireQueuedUploads`. - return "Detach: rclone did not answer the question about the queue - upload deadlines were" - + " NOT moved, \(drain)" - } - if outcome.queued == 0 { - return "Detach: queue empty - nothing to speed up" - } - if outcome.moved == 0 { - return "Detach: WARNING - the queue has \(outcome.queued) items and NOT ONE could be" - + " sped up (rclone rejected every vfs/queue-set-expiry), \(drain)" - } - if outcome.moved < outcome.queued { - return "Detach: forced upload of \(outcome.moved) of \(outcome.queued) queued items -" - + " the deadline of the remaining \(outcome.queued - outcome.moved) was NOT moved, \(drain)" - } - return "Detach: forced upload of \(outcome.moved) queued items" - } - - /// Pure version of the detach verdict - `settled` is the queue reading from - /// the moment it went quiet (`nil` = it did not go quiet in time, or rclone - /// did not answer). - /// - /// An empty queue is not yet complete data on Drive. Bands that rclone - /// ABANDONED drop out of the queue exactly like uploaded ones and remain only - /// in `erroredFiles`. Until 23 September 2026 the detach looked only at - /// `uploadsInProgress`/`uploadsQueued`, so it reported "Detached, everything - /// uploaded to Google Drive" with data existing ONLY on this Mac - while - /// `UploadState`, from the same counters, already derived `.failedFiles(...)` - /// with the label "ACTION NEEDED". The CLI and the GUI said two different - /// things about the same moment. - static func detachVerdict(settled: DriveBufferService.QueueStats?) -> CMActionResult { - guard let settled else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Detached, but the upload did NOT finish in time - do not delete the buffer.")) - } - guard settled.erroredFiles == 0 else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Detached, but rclone ABANDONED %@ backup fragments - they exist only on this Mac and are not on Google Drive. Do not delete the buffer.", - "\(settled.erroredFiles)")) - } - return CMActionResult( - succeeded: true, message: L10n.tr("Detached, everything uploaded to Google Drive.")) - } - - /// Unmounts backup snapshots mounted under `/Volumes/.timemachine/`. - /// Returns the paths that could NOT be unmounted. - /// - /// Browsing a backup - in Finder or with a plain `ls` on a path from - /// `tmutil listbackups` - mounts its snapshot read-only. Such a mount keeps - /// the image's device busy and `hdiutil detach` refuses, while the message - /// says not a word about what is blocking it. - /// - /// We USE `diskutil unmount`, NOT `/sbin/umount`. Measured on a working - /// installation: `umount` on such a snapshot ends with - /// `Operation not permitted` for the user (the system manages the mount), - /// while `diskutil unmount` on the same path succeeds without root. - /// The previous version called `umount` via `try?` and logged "Unmounted" - /// REGARDLESS of the result - so with 18 mounted snapshots the log reported - /// 18 successes, none was unmounted, and `hdiutil detach` right after refused - /// with no connection visible in the log. - @discardableResult - public static func unmountBrowsedSnapshots() async -> [String] { - guard let points = DriveBufferService.mountPoints() else { return [] } - var failed: [String] = [] - for path in browsedSnapshotMounts(points) { - let result = try? await ProcessRunner.run( - "/usr/sbin/diskutil", ["unmount", path], timeout: 60) - if result?.succeeded == true { - CMLogger.log("Unmounted browsed backup snapshot: \(path)") - } else { - failed.append(path) - CMLogger.log("FAILED to unmount backup snapshot: \(path)") - } - } - return failed - } - - // MARK: - Verification - - /// Checks the image's consistency. - /// - /// NOTE: `hdiutil verify` on a sparsebundle does NOT work - such an image has - /// no checksum and the tool ends with the message "has no checksum". The - /// device has to be attached without mounting and `fsck_apfs` run on it. - public static func verify() async -> CMActionResult { - await withCMLock(lockName) { await verifyLocked() } - ?? busyResult("Image verify", displayName: L10n.tr("Image verification")) - } - - private static func verifyLocked() async -> CMActionResult { - guard exists else { - return CMActionResult(succeeded: false, message: L10n.tr("No image.")) - } - if isAttached { - return CMActionResult( - succeeded: false, - message: L10n.tr("The image is attached - detach it before verifying.")) - } - - guard - let attachResult = try? await ProcessRunner.run( - "/usr/bin/hdiutil", ["attach", imagePath.path, "-nomount"], timeout: 300), - attachResult.succeeded, - let device = attachResult.stdout - .components(separatedBy: .newlines) - .first(where: { $0.contains("41504653") })? - .prefix(while: { !$0.isWhitespace }) - else { - return CMActionResult( - succeeded: false, message: L10n.tr("Could not find an APFS device in the image.")) - } - - // NO timeout. `fsck_apfs` reads metadata through the rclone mount, so its - // time depends on the link and on the number of snapshots - measured on a - // 210 GiB image with 18 snapshots: a single snapshot takes minutes. - // The earlier one-hour limit protected against nothing, but turned "the - // check is still running" into "Image INCONSISTENT", because a killed - // `fsck` returns a non-zero code just like an `fsck` that found damage. - // A false alarm about losing the backup is more dangerous here than a long - // wait. - let fsck = try? await ProcessRunner.run(fsckPath, ["-n", String(device)]) - - // Was the device still ours when `fsck` finished? - // - // `fsck_apfs` returns a non-zero code both when it found damage and when - // someone pulled the device out from under it - and - // `purgeStaleDevices()` (`hdiutil detach -force`) can do exactly that on - // the `gdrive-attach` agent's tick every 900 s. Without this check `verify` - // then reported "Image INCONSISTENT", i.e. a false alarm about losing the - // whole backup. The 'image' lock already closes that window, but the - // message has to be honest also when the device disappears for another - // reason. - let deviceSurvived = await devicesForImage().contains(parentDevice(of: String(device))) - - // We detach WITH waiting, not via `defer { Task { ... } }`. That version - // returned from the function before the detach happened - and the caller - // usually attaches the image back right away, so the attach raced with a - // pending detach of the same device. - _ = try? await ProcessRunner.run( - "/usr/bin/hdiutil", ["detach", String(device), "-force", "-quiet"], timeout: 120) - - // "Could not check" is NOT the same as "inconsistent" - one means no - // result, the other a damaged backup. Merging them into one message would - // make the user restore the whole backup because of a failed tool run. - guard let fsck else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not run %@ - the image's consistency REMAINS UNCHECKED.", fsckPath)) - } - if !fsck.succeeded && !deviceSurvived { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Check INTERRUPTED - device %@ disappeared midway (someone force-detached the image). This is not a result about the backup's state: the image's consistency REMAINS UNCHECKED. Repeat the check.", - String(device))) - } - return CMActionResult( - succeeded: fsck.succeeded, - message: fsck.succeeded - ? L10n.tr("Image consistent.") - : L10n.tr("Image INCONSISTENT: %@", String(fsck.stdout.suffix(500)))) - } - - /// `/dev/disk7s1` -> `/dev/disk7`. - /// - /// `fsck_apfs` gets the APFS partition, while `hdiutil info` - and therefore - /// `devicesForImage()` - lists the PARENT device. Comparing them directly - /// would never match, so the "did the device survive" check would silently - /// answer "no" every time. - static func parentDevice(of device: String) -> String { - let prefix = "/dev/disk" - guard device.hasPrefix(prefix) else { return device } - let digits = device.dropFirst(prefix.count).prefix(while: \.isNumber) - return digits.isEmpty ? device : prefix + digits - } - - // MARK: - Restart readiness - - /// Whether the Mac can be safely shut down without `prepare-shutdown`. - /// - /// The risk when shutting down is not constant - it exists only when data - /// not yet uploaded is waiting in the buffer. macOS gives agents a dozen or - /// so seconds to quit, which with an empty queue is plenty, and with a full - /// one not enough at all. - /// - /// Measured: the queue returns to zero within a few minutes after every - /// hourly backup, so for most of the day a restart is simply safe. Instead of - /// making the user remember a command before every restart, we tell them - /// when it is really needed. - public static func safeToRebootNow() async -> Bool { - guard let stats = await DriveBufferService.queueStats() else { - // Without a reading of the queue state we have no grounds to claim it is - // safe - and for such a question silence must mean "no". - return false - } - // `isQuiet`, NOT `isIdle`: an empty queue is not enough, because bands - // abandoned by rclone (`erroredFiles`) drop out of the queue just like - // uploaded ones. A restart in that state does not destroy anything extra, - // but the answer "YES - queue empty" read as "the backup on Drive is - // complete", and it was not - and the same sentence appeared in - // `drive-status` next to `UploadState.failedFiles` with the label - // "ACTION NEEDED". - return stats.isQuiet - } - - // MARK: - Retrying - - /// Retries `hdiutil` operations on a FUSE-T mount. - /// - /// FUSE-T mounts via NFS, and `hdiutil` on such a volume is sometimes - /// rejected with "RPC version wrong". Measured: the error does not depend on - /// the image size or the data (one run failed for 100 GB and 400 GB, and - /// passed for 600, 1000 and 1500 GB), only on the moment - with an empty - /// upload queue 5 of 5 attempts succeeded, with rclone busy it was random. - /// When creating the production image, the first attempt failed and the - /// second passed. - private static func retryingFlakyMount( - attempts: Int, _ operation: () async -> ProcessResult? - ) async -> ProcessResult? { - var last: ProcessResult? - for attempt in 1...attempts { - last = await operation() - if last?.succeeded == true { return last } - guard attempt < attempts else { break } - CMLogger.log("hdiutil: attempt \(attempt) failed, retrying") - try? await Task.sleep(nanoseconds: 5_000_000_000) - await DriveBufferService.waitUntilIdle(timeout: 60) - } - return last - } -} diff --git a/mac-app/Sources/CloudMachineCore/BufferGuardService.swift b/mac-app/Sources/CloudMachineCore/BufferGuardService.swift deleted file mode 100644 index 784827a..0000000 --- a/mac-app/Sources/CloudMachineCore/BufferGuardService.swift +++ /dev/null @@ -1,813 +0,0 @@ -import Foundation - -/// Keeps the buffer from eating the disk - a port of `gdrive/buffer-guard.sh`. -/// -/// Time Machine writes to the attached image at SSD speed (measured 267 MB/s), -/// while rclone uploads at link speed (~41 MB/s with a 332 Mb/s upload). The -/// difference lands in the buffer. -/// -/// `--vfs-cache-max-size` is a SOFT limit: rclone evicts from the buffer only -/// data already uploaded, so when everything is waiting in the queue the -/// buffer keeps growing and can fill the disk. With a first backup measured in -/// terabytes this is not theory - the measured net growth at the start was -/// 32 MB/s. -/// -/// The watchdog pauses Time Machine when the UNSENT BACKLOG exceeds a -/// threshold, and resumes it when the upload catches up. The backup becomes -/// slower, but it finishes instead of crashing the machine. -/// -/// Three things you need to know here, because each was once done the other -/// way round and each cost the whole protection: -/// -/// 1. We measure the BACKLOG, not the cache size. The cache size sits at the -/// limit permanently and does not answer whether the upload is keeping up -/// (see `backlogGB` and `Thresholds.init`). -/// 2. "I do not know" is neither a pause nor a resume. No answer from rclone -/// is not turned into a number, and an unreadable rclone log is not turned -/// into "no problem". -/// 3. A pause lasts as long as we keep it up. `tmutil stopbackup` cancels the -/// RUNNING backup and does not touch the schedule, so macOS starts another -/// one in its hourly cycle - which is why the pause is repeated on every -/// tick, not only on a state change (see `keepPaused`). -public actor BufferGuardService { - - public struct Thresholds: Sendable { - /// Above this many GB of UNSENT BACKLOG we pause Time Machine. - /// - /// Backlog, not cache size - see `backlogGB` for the reason. - public var highGB: Int - /// Below this many GB of backlog we resume. - public var lowGB: Int - /// Below this many GB free on disk we pause regardless of the buffer. - public var minFreeGB: Int - /// Below this many GB free ON GOOGLE DRIVE a pause put in place because of - /// lack of space on Drive must not be lifted. - /// - /// The same number that `BackupHealth` considers the warning threshold for - /// Drive - one source of truth. With growth of around 600 MB per hourly - /// cycle, 30 GB is about two weeks of headroom, i.e. enough for the resumed - /// backup to have room, rather than hitting the wall again in the next hour. - public var minDriveFreeGB: Int - - /// Thresholds derived from the buffer size, not typed in by hand - but - /// computed ANEW since the watchdog measures the unsent backlog rather than - /// the cache size. The old 1.5x and 0.4x of `cacheSizeGB` are not carried - /// over, because they referred to a different quantity and mean nothing in - /// this one. - /// - /// WHAT WAS WRONG - /// - /// The old pair (150 GB / 40 GB) referred to `bytesUsed`, i.e. the size of - /// the WHOLE cache. With `--vfs-cache-max-size 100G` and - /// `--vfs-cache-max-age 9999h` that sits at the limit permanently: 281 - /// measurements in the journal, minimum 99 GB. The 40 GB resume threshold - /// was therefore UNREACHABLE, and the 150 GB pause threshold reachable only - /// through the result of the directory walk, i.e. through a DIFFERENT - /// measure. The log shows it plainly: ONE PAUSE line (23.09.2026 03:34, - /// "buffer 155 GB") and ZERO RESUME lines. - /// - /// WHY THE THRESHOLDS ARE A FRACTION OF `cacheSizeGB`, BUT BELOW IT - /// - /// The unsent backlog is exactly the part of the cache that rclone CANNOT - /// evict - it only evicts what it has already uploaded. As long as the - /// backlog is smaller than `cacheSizeGB`, the cache has something to shrink - /// from and the limit works. When the backlog reaches `cacheSizeGB`, there is - /// no headroom and every further gigabyte written goes OVER the limit, - /// straight into free disk space. So the pause threshold must lie BELOW the - /// cache size - unlike the old 150 GB, which lay above it. - /// - /// `highGB` = half the buffer, 50 GB today: - /// - leaves 50 GB of evictable headroom, i.e. about 26 minutes at the - /// measured net growth of 32 MB/s - with margin for a tick every 30 s and - /// for `tmutil stopbackup` to take effect; - /// - lies more than three times above the highest backlog seen in normal - /// operation (462 items, i.e. about 15 GB), so neither an ordinary backup - /// nor a jam on Google's daily limit pauses the backups. The latter is - /// intended and described below in `step()`. - /// - /// `lowGB` = a tenth of the buffer, 10 GB today: - /// - must be REACHABLE, because that is what the previous version tripped - /// over. After a pause no new bands are created, the `writeBackSeconds` - /// delay passes and the queue drains at link speed (measured 23.09: - /// 96 Mb/s, i.e. about 43 GB/h), so the way from 50 -> 10 GB is about an - /// hour; - /// - a 40 GB hysteresis is, at the measured speed difference (267 MB/s of - /// Time Machine writes, 41 MB/s of upload, 226 MB/s net), about three - /// minutes of work between consecutive pauses. A resume threshold close to - /// the pause threshold would give start/stop on almost every tick. - /// - /// Disk protection does NOT depend on these two numbers: `minFreeGB` and the - /// `outOfSpace` reported by rclone work independently of the backlog and in - /// EVERY watchdog state (see `step()`). - public init( - highGB: Int = DriveBufferService.cacheSizeGB / 2, - lowGB: Int = DriveBufferService.cacheSizeGB / 10, - minFreeGB: Int = 80, - minDriveFreeGB: Int = BackupHealth.driveFreeWarningGB - ) { - self.highGB = highGB - self.lowGB = lowGB - self.minFreeGB = minFreeGB - self.minDriveFreeGB = minDriveFreeGB - } - } - - public enum State: String, Sendable { - /// We are supervising a running backup. - case running - case pausedForBuffer - case pausedForQuota - /// No backup running - we keep watch until the next one. - /// - /// The watchdog does NOT exit after a finished backup. It runs under - /// launchd with KeepAlive, so exiting would mean an immediate restart, and - /// with Time Machine not working - a tight restart loop limited only by - /// ThrottleInterval. - case idle - } - - public struct Snapshot: Sendable { - public var state: State - /// Unsent backlog in GB. `nil` = rclone did not answer, i.e. UNKNOWN - and - /// then the watchdog NEITHER pauses NOR resumes the backup. - public var backlogGB: Int? - /// `nil` = there was NO measurement (statfs failed), not "zero gigabytes". - public var freeGB: Int? - /// `nil` = tmutil did not answer, i.e. unknown. - public var backupRunning: Bool? - public var percent: Double - } - - /// Sources of measurements and control. - /// - /// The defaults (`live`) read the real system. A test substitutes its own and - /// thanks to that walks the WHOLE decision path of the watchdog - pause, state - /// change, resume - without tmutil, rclone and a real backup. The same pattern - /// as `preferencesFile` in `BackupHealth.currentReport`: there is no other way - /// to inject a KNOWN BAD sample, and it is precisely in the watchdog's - /// decisions (not in parsing) that the silent failures lived here. - public struct Probes: Sendable { - public var queueStats: @Sendable () async -> DriveBufferService.QueueStats? - /// Cache size on disk - ONLY for one line in the log. - /// - /// A separate probe rather than a call in place, for two reasons. First: - /// we call it only when reporting that rclone is not answering, because in - /// the `live` version it is a walk of 6504 files on the disk the backup is - /// going to. Second: the test must be able to show that this number takes - /// part in NO decision - it feeds it the 155 GB from the production run of - /// 23.09 and checks that the watchdog still does not pause Time Machine. - public var cacheSizeGB: @Sendable (DriveBufferService.QueueStats?) -> Int? - /// `nil` = not measured. - public var freeGB: @Sendable () -> Int? - /// `nil` = tmutil did not answer. - public var backupRunning: @Sendable () async -> Bool? - public var progressPercent: @Sendable () async -> Double - /// Whether Google Drive has run out of space. `nil` = THE RCLONE LOG CANNOT - /// BE READ, i.e. unknown - not "no problem". - public var hitStorageQuota: @Sendable () -> Bool? - /// Whether the upload is stuck on the daily limit. `nil` as above. - public var uploadStalled: @Sendable () -> Bool? - /// Free bytes on Google Drive. `nil` = UNKNOWN (rclone did not answer) - - /// and that is not consent to resume. - public var driveFreeBytes: @Sendable () async -> UInt64? - /// `true` ONLY when tmutil confirmed that the command was carried out. - public var stopBackup: @Sendable () async -> Bool - public var startBackup: @Sendable () async -> Bool - /// Reporting an upload jam. `nil` ("I do not know") HAS NO RIGHT to clear - /// the jam marker - see `reportUploadStall`. - public var reportStall: @Sendable (Bool?) async -> Void - /// Split out so that a test does not append its made-up "PAUSE (threshold)" - /// to the production `cloudmachine.log` - that log serves to diagnose real - /// failures and must not contain events that did not happen. - public var log: @Sendable (String) -> Void - - public init( - queueStats: @escaping @Sendable () async -> DriveBufferService.QueueStats?, - cacheSizeGB: @escaping @Sendable (DriveBufferService.QueueStats?) -> Int?, - freeGB: @escaping @Sendable () -> Int?, - backupRunning: @escaping @Sendable () async -> Bool?, - progressPercent: @escaping @Sendable () async -> Double, - hitStorageQuota: @escaping @Sendable () -> Bool?, - uploadStalled: @escaping @Sendable () -> Bool?, - driveFreeBytes: @escaping @Sendable () async -> UInt64?, - stopBackup: @escaping @Sendable () async -> Bool, - startBackup: @escaping @Sendable () async -> Bool, - reportStall: @escaping @Sendable (Bool?) async -> Void, - log: @escaping @Sendable (String) -> Void - ) { - self.queueStats = queueStats - self.cacheSizeGB = cacheSizeGB - self.freeGB = freeGB - self.backupRunning = backupRunning - self.progressPercent = progressPercent - self.hitStorageQuota = hitStorageQuota - self.uploadStalled = uploadStalled - self.driveFreeBytes = driveFreeBytes - self.stopBackup = stopBackup - self.startBackup = startBackup - self.reportStall = reportStall - self.log = log - } - - public static let live = Probes( - queueStats: { await DriveBufferService.queueStats() }, - cacheSizeGB: { BufferGuardService.cacheSizeGB(stats: $0) }, - freeGB: { BufferGuardService.freeGB() }, - backupRunning: { await TimeMachineStatus.runningState() }, - progressPercent: { (await TimeMachineStatus.currentProgress())?.percent ?? 0 }, - // The `...State()` versions, not `hitStorageQuota()`/`uploadStalled()`: - // the latter are for display and turn "I do not know" into `false`. - hitStorageQuota: { DriveBufferService.hitStorageQuotaState() }, - uploadStalled: { DriveBufferService.uploadStalledState() }, - driveFreeBytes: { await DriveBufferService.remoteQuota()?.free }, - stopBackup: { await BufferGuardService.tmutil("stopbackup") }, - startBackup: { await BufferGuardService.tmutil("startbackup") }, - reportStall: { await BufferGuardService.reportUploadStall($0) }, - log: { CMLogger.log($0) }) - } - - private let thresholds: Thresholds - private let probes: Probes - private var state: State = .idle - /// Whether since the last transition to idle we have seen a running backup - - /// so that the end is reported once, not on every tick. - private var sawBackupRunning = false - /// Whether the previous step already reported a failed pause - so that with a - /// failure lasting hours the log does not grow by a line every 30 seconds, - /// but the event itself does NOT disappear (see `EdgeTriggeredLog`, same - /// reason). - private var reportedStopFailure = false - /// The same for a missing free-space measurement. - private var reportedFreeUnknown = false - /// The same for no answer about the unsent backlog. - private var reportedBacklogUnknown = false - /// The same for an unreadable rclone log. - private var reportedLogUnreadable = false - /// The same for Time Machine starting during a pause. - private var reportedRestop = false - /// The same for a pause for lack of space on Drive held without proof. - private var reportedQuotaHold = false - - public init(thresholds: Thresholds = Thresholds(), probes: Probes = .live) { - self.thresholds = thresholds - self.probes = probes - } - - // MARK: - Measurements - - /// UNSENT BACKLOG in GB - the only measure on which the watchdog decides to - /// pause and to resume. `nil` = rclone did not answer, i.e. WE DO NOT KNOW. - /// - /// WHY NOT THE CACHE SIZE - /// - /// Until 25.09.2026 the watchdog looked at `stats.bytesUsed`, i.e. the size of - /// the whole rclone cache. With `--vfs-cache-max-size 100G` and - /// `--vfs-cache-max-age 9999h` that number always sits at the limit - rclone - /// also keeps in the cache what it uploaded long ago. So the watchdog measured - /// an almost CONSTANT state and asked it about a CHANGING thing: whether the - /// upload is keeping up with the writes. Result in the journal: 281 - /// measurements, minimum 99 GB, one pause and not a single resume. The unsent - /// backlog is the same quantity that decides whether the cache can shrink at - /// all - see `Thresholds.init`. - /// - /// WHY FROM THE ITEM COUNT, NOT FROM BYTES - /// - /// `vfs/stats` does not report the number of unsent BYTES: it has item - /// counters (`uploadsQueued`, `uploadsInProgress`) and `bytesUsed` of the - /// whole cache. Item sizes are exposed by `vfs/queue`, but that is a SECOND rc - /// interface call on every tick, and `vfs/stats` alone was measured here at - /// 36.7 s with a clogged buffer (see `DriveBufferService.queueStats`) - - /// doubling that cost lengthens the reaction exactly when the backlog grows - /// fastest. And it would give almost nothing: every item in this queue is a - /// sparsebundle band of a FIXED size `BackupImageService.bandSectors` - /// (32 MiB), so the sum of sizes is almost exactly the item count times - /// 32 MiB. Check against real numbers: the 462 items of 23.09 give 14 GB from - /// here, and the owner estimated "about 15 GB". - /// - /// THIS IS AN ESTIMATE, not a measurement - and it is described as such in the - /// log (the "~" sign). The error goes ONE way: partial bands and small metadata - /// files are SMALLER than 32 MiB, so the estimate overstates the backlog, and an - /// overstated backlog pauses the backup earlier. For disk protection that is - /// the right direction to err in. - public static func backlogGB(stats: DriveBufferService.QueueStats?) -> Int? { - guard let stats else { return nil } - let bandBytes = UInt64(BackupImageService.bandSectors) * 512 - return Int(UInt64(max(0, stats.unsentItems)) * bandBytes / 1_073_741_824) - } - - /// rclone cache size in GB - for THE VIEW AND THE LOG, never for decisions. - /// `nil` = not measured either way. - /// - /// The two sources of this number are NOT EQUIVALENT, and that is why they - /// must not be mixed in a decision: rclone reports the size of its own cache, - /// while the directory walk reports DISK SPACE TAKEN, which can exceed the - /// `--vfs-cache-max-size` limit (hence "155 GB" with a 100 GB limit). For a - /// status line both are as good as any approximation; for pausing Time - /// Machine neither is - see `DriveBufferService.cacheSizeBytesByWalk`. - public static func cacheSizeGB(stats: DriveBufferService.QueueStats? = nil) -> Int? { - if let bytes = stats?.bytesUsed, bytes > 0 { return Int(bytes / 1_073_741_824) } - guard let walked = DriveBufferService.cacheSizeBytesByWalk() else { return nil } - return Int(walked / 1_073_741_824) - } - - /// Cache size for an interface that has nowhere to show "I do not know" - /// (`BufferStatus.sizeGB`). Behaves exactly as the old `bufferGB` did - - /// substituted zero included. - /// - /// NO DECISION may call this; the watchdog uses `backlogGB`. The zero for a - /// missing measurement stays here until it is removed together with - /// `BufferStatus` and `CloudMachineController`, which have to learn a third - /// state - that is a separate change, outside this branch. - public static func bufferGB(stats: DriveBufferService.QueueStats? = nil) -> Int { - cacheSizeGB(stats: stats) ?? 0 - } - - /// Free space counted the way `df` counts it - i.e. PESSIMISTICALLY. - /// - /// It is tempting to use `volumeAvailableCapacityForImportantUsageKey`, but - /// that is an optimistic measure: it includes space taken by local snapshots - /// that the system only COULD free. On this machine it showed 1202 GB while - /// `df` said 427 GB. The watchdog is supposed to pause the backup before the - /// disk fills up, so it has to look at the space actually available now, not - /// at a promise. - /// - /// `nil` means "NOT MEASURED", and that is not cosmetic. Previously a failed - /// `statfs` returned `0`, i.e. a number - and then the pause condition - /// (`free <= minFreeGB`) was met immediately, the resume condition - /// (`free > minFreeGB`) NEVER, and the watchdog paused Time Machine forever. - /// In parallel the monitor reported "The Mac's disk is running out of space - /// (0 GB)" - an alarm about a state nobody measured. The same distinction that - /// `BufferStatus.queueKnown` already introduced for the upload queue. - public static func freeGB() -> Int? { - var stats = statfs() - guard statfs("/System/Volumes/Data", &stats) == 0 else { return nil } - let available = UInt64(stats.f_bavail) * UInt64(stats.f_bsize) - return Int(available / 1_073_741_824) - } - - /// Whether Time Machine may be resumed, looking ONLY at local measurements. - /// - /// A pure function - the decision can be tested without a disk and without - /// tmutil. `free == nil` does not resume: a missing measurement is not proof - /// that space is there. Resuming checks free space THE SAME WAY as pausing, - /// because a pause protecting the disk must not be lifted by a condition that - /// knows nothing about the disk. - /// - /// `backlog == nil` does not resume either, and that is the same rule applied - /// to the second number. Previously no answer from rclone ended in a directory - /// walk, and a failed walk - in zero; and zero meets the resume condition - /// immediately, i.e. it LIFTED a pause put in place because the buffer was - /// full. - static func canResumeLocally(backlog: Int?, free: Int?, thresholds: Thresholds) -> Bool { - guard let backlog, let free else { return false } - return backlog <= thresholds.lowGB && free > thresholds.minFreeGB - } - - /// Whether there is PROVEN room on Google Drive for further work. - /// - /// `nil` (rclone did not answer) is NOT consent - see `step()`. - static func driveHasRoom(freeBytes: UInt64?, minGB: Int) -> Bool { - guard let freeBytes else { return false } - return freeBytes / 1_073_741_824 >= UInt64(max(0, minGB)) - } - - // MARK: - One step - - /// Performs one supervision step and returns the state. Split out of the loop - /// so that decisions can be tested without waiting in real time. - /// - /// THE LAYOUT OF THIS FUNCTION IS PART OF THE FIX. Until 25.09.2026 the checks - /// of `stats?.outOfSpace` and free space sat ONLY in the `.running` branch, - /// and the `.pausedForBuffer` branch looked at nothing except the resume - /// condition. After one pause the watchdog therefore stopped guarding the - /// disk - i.e. the protection this process exists for switched off until the - /// agent restarted. Measured: 53 hours in this state (pause 23.09.2026 03:34 - /// -> process restart 25.09.2026 08:46). That is why disk protection and - /// `outOfSpace` NOW stand BEFORE `switch state`, and no path through this - /// function can skip them. - @discardableResult - public func step() async -> Snapshot { - let stats = await probes.queueStats() - let backlog = Self.backlogGB(stats: stats) - let free = probes.freeGB() - let running = await probes.backupRunning() - let percent = await probes.progressPercent() - // Both questions go to the rclone log and both are three-state: an - // unreadable file is "I do not know", not "no problem". - let quota = probes.hitStorageQuota() - let stalled = probes.uploadStalled() - - func snapshot() -> Snapshot { - Snapshot( - state: state, backlogGB: backlog, freeGB: free, backupRunning: running, percent: percent) - } - - // A failed free-space measurement must NOT pass silently: the only - // protection of the disk against filling up depends on this number, and - // without it the watchdog will not pause Time Machine (and rightly so - it - // does not guess). A person must learn about it from the log, not from a - // full disk. - if free == nil, !reportedFreeUnknown { - probes.log( - "WARNING: cannot measure free disk space (statfs failed) - the watchdog will not pause Time Machine because of the disk, as it has nothing to base the decision on." - ) - } - reportedFreeUnknown = (free == nil) - - // No answer from rclone must not pass silently either - but this time we do - // NOT TURN it into a number. On 23.09.2026 in this situation the watchdog - // fell back to the directory walk, got 155 GB (disk space taken - a measure - // not comparable with the 100 GB limit), crossed the threshold with it and - // paused Time Machine; an hour later the monitor wrote "rclone remote - // control is not answering". So the pause rested on a number taken from the - // fact that there was no measurement. We still print the cache size - but AS - // SOMETHING ELSE, once per episode and with no effect on decisions. - if backlog == nil, !reportedBacklogUnknown { - let cache = probes.cacheSizeGB(stats) - probes.log( - "WARNING: rclone remote control is not answering - unknown how much is left to upload. The watchdog will NEITHER pause NOR resume Time Machine on that basis. The cache takes \(cache.map { "\($0) GB" } ?? "an unknown amount") on disk - that is DISK SPACE, not a backlog to upload, and it is no basis for a pause. Disk protection keeps working: free \(describe(free)), threshold \(thresholds.minFreeGB) GB." - ) - } - reportedBacklogUnknown = (backlog == nil) - - // An unreadable rclone log means "I do not know" for BOTH questions asked - // of this file. Previously it meant "no problem", and that had two effects: - // the watchdog did not pause the backup when Drive was out of space, and - // `reportStall(false)` DELETED the jam marker and reported "Upload to Google - // Drive has resumed" - a claim about an event nobody checked. The log has - // `-rw-r-----` permissions, and at rclone start-up it is moved to `.1`, so - // an unreadable log is an expected state, not a hypothesis. - let logUnreadable = (quota == nil || stalled == nil) - if logUnreadable, !reportedLogUnreadable { - probes.log( - "WARNING: cannot read the rclone log (\(DriveBufferService.logFile.path)) - the watchdog will recognise neither lack of space on Google Drive nor an upload jam. The jam marker stays unchanged, because 'I do not know' does not clear it." - ) - } - reportedLogUnreadable = logUnreadable - - // The daily upload limit is SOMETHING ELSE and deliberately does NOT pause - // the backup. - // - // Measured on two episodes (12 and 15 September 2026): during a jam lasting - // several hours the buffer did not budge - 99-103 GB, exactly as usual - - // and the queue cleared by itself once the rolling 24 h window moved - // forward. A pause would then have cost backups and given nothing in - // return. The thresholds below protect against filling the disk, and they - // work regardless of what causes the jam. - // - // We do, however, ALWAYS report it, because the jam of 12 September went - // completely unnoticed - three hours without uploads and not a single trace - // except the raw rclone log. `nil` is passed on as `nil`: the report itself - // knows that "I do not know" clears nothing. - await probes.reportStall(stalled) - - // Lack of SPACE on Drive takes precedence and will not pass by itself: - // until the user deletes something, the upload will not move, and further - // Time Machine work only pumps up the buffer. `nil` (unreadable log) does - // NOT pause - no answer is not proof of a failure, just as it is not proof - // of its absence; we reported it in the log above. - if quota == true { - if state == .pausedForQuota { - await keepPaused(backupRunning: running) - } else { - await pause( - reason: - "PAUSE (no space on Google Drive): backlog \(describeBacklog(backlog)), free \(describe(free))", - into: .pausedForQuota, backupRunning: running) - } - return snapshot() - } - - // DISK PROTECTION - IN EVERY STATE, not only in `.running`. - // - // `outOfSpace` comes from rclone and means "I have nowhere left to put - // data" - a harder fact than any threshold of ours, and it does not stop - // being a fact because the watchdog happens to be paused. - // - // A missing free-space measurement (`free == nil`) does NOT pause the - // backup. Previously a failed `statfs` gave zero, zero met the pause - // condition and the watchdog paused Time Machine based on a number it - // never measured - and then could not resume it, because the resume - // condition never holds at zero. - let lowDisk = free.map { $0 <= thresholds.minFreeGB } ?? false - let bufferFull = stats?.outOfSpace == true - if bufferFull || lowDisk { - let why = - bufferFull ? "rclone reports no space in the buffer" : "little free disk space" - switch state { - case .pausedForBuffer, .pausedForQuota: - // Already paused, so there is nothing to announce - but Time Machine may - // have started by itself in its hourly cycle, so we repeat the pause. - // IMPORTANT: we do not go on to resuming from here. As long as the disk - // is against the wall, no resume condition may lift the pause. - await keepPaused(backupRunning: running) - case .running, .idle: - // Also from `.idle`: the disk fills up regardless of whether a backup - // is running this second, and macOS starts another one every hour. - // Entering the pause makes the next tick stop it. - await pause( - reason: - "PAUSE (\(why)): backlog \(describeBacklog(backlog)), free \(describe(free)) - waiting for the upload", - into: .pausedForBuffer, backupRunning: running) - } - return snapshot() - } - - switch state { - case .idle: - // Only an EXPLICIT "yes". `nil` (tmutil did not answer) leaves the state - // unchanged - we do not start supervising something we know nothing about. - if running == true { - probes.log("Backup started - supervising the upload backlog") - sawBackupRunning = true - state = .running - } - - case .running: - // `backlog == nil` does NOT pause: the same pattern as with `free`. - // No answer from rclone is not a number and has no right to trigger an - // irreversible pause. - if let backlog, backlog >= thresholds.highGB { - await pause( - reason: - "PAUSE (backlog threshold \(thresholds.highGB) GB): backlog \(describeBacklog(backlog)), free \(describe(free)) - waiting for the upload", - into: .pausedForBuffer, backupRunning: running) - } else if running == false { - if sawBackupRunning { - probes.log("Time Machine finished. Backlog \(describeBacklog(backlog))") - sawBackupRunning = false - } - state = .idle - } - - case .pausedForBuffer: - // We resume only when the upload has actually caught up - otherwise we - // would fall into start/stop oscillation at the threshold. Until it has - // caught up, we KEEP UP the pause: the `tmutil stopbackup` from the moment - // of pausing applied only to that one run. - if Self.canResumeLocally(backlog: backlog, free: free, thresholds: thresholds) { - await resume(backlogGB: backlog, freeGB: free) - } else { - await keepPaused(backupRunning: running) - } - - case .pausedForQuota: - // A pause for LACK OF SPACE on Drive needs POSITIVE proof that space is - // there before it is lifted. This is not extra caution: - // - // `hitStorageQuota()` reads entries from the last 30 minutes of the rclone - // log. After Time Machine is paused no new bands are created, rclone stops - // trying to upload, the entries age and the function starts returning - // `false` - even though Drive has as little space as before. With a quiet - // buffer (and after a pause the buffer does empty) the shared resume - // condition was then met immediately: the watchdog let Time Machine go, it - // wrote more bands that cannot be uploaded, and the whole pause ended after - // a few dozen minutes without anything changing on the Drive side. - // `UploadState` says plainly that this state does NOT pass by itself. - guard Self.canResumeLocally(backlog: backlog, free: free, thresholds: thresholds) else { - await keepPaused(backupRunning: running) - break - } - let driveFree = await probes.driveFreeBytes() - guard Self.driveHasRoom(freeBytes: driveFree, minGB: thresholds.minDriveFreeGB) else { - // No answer from rclone KEEPS the pause - "I do not know" is never - // consent to resume something that fills up the disk. - if !reportedQuotaHold { - let amount = - driveFree.map { "\($0 / 1_073_741_824) GB" } ?? "unknown (rclone did not answer)" - probes.log( - "PAUSE (no space on Drive) kept: free on Google Drive \(amount), required at least \(thresholds.minDriveFreeGB) GB." - ) - reportedQuotaHold = true - } - await keepPaused(backupRunning: running) - break - } - reportedQuotaHold = false - await resume(backlogGB: backlog, freeGB: free) - } - - return snapshot() - } - - public func currentState() -> State { state } - - /// Free space for the log. A missing measurement MUST look different from - /// zero, otherwise the log line lies just as the number itself used to. - private func describe(_ freeGB: Int?) -> String { - freeGB.map { "\($0) GB" } ?? "not measured" - } - - /// Backlog for the log. The "~" sign is not decoration: this number is - /// ESTIMATED from the number of items in the queue (see `backlogGB`), and a - /// log that gives an estimate as a measurement lies about how solid the basis - /// of the decision is. - private func describeBacklog(_ gb: Int?) -> String { - gb.map { "~\($0) GB" } ?? "unknown (rclone did not answer)" - } - - // MARK: - Controlling Time Machine - - /// Pauses Time Machine and moves to `into` ONLY when that succeeded. - /// - /// THIS is the fix. Previously the result of `tmutil stopbackup` was thrown - /// away (`_ = try? await ...`), and the state changed UNCONDITIONALLY. When - /// the command failed - no permissions, time limit exceeded - the watchdog - /// considered the pause done, and since `stopBackup()` was called only on a - /// state CHANGE, it never retried. Time Machine kept writing, the watchdog - /// waited for the drain, the disk filled up completely, and the log said - /// "PAUSE ... waiting for the upload". - /// - /// On failure the state stays at `.running`, so the pause condition (still - /// met) triggers another attempt on the next tick - i.e. in 30 seconds, - /// without any extra retry mechanism. - /// - /// `backupRunning == false` (tmutil says EXPLICITLY that no backup is - /// running) is a separate path: there is nothing to pause then, and calling - /// `stopbackup` without a running backup can return an error - and the - /// watchdog would then report "Time Machine KEEPS WRITING", i.e. a claim - /// about an event nobody checked. `nil` ("tmutil did not answer") takes the - /// strict path, because no answer is not proof of quiet. - private func pause(reason: String, into paused: State, backupRunning: Bool?) async { - probes.log(reason) - if backupRunning == false { - state = paused - reportedStopFailure = false - return - } - if await probes.stopBackup() { - state = paused - reportedStopFailure = false - return - } - if !reportedStopFailure { - probes.log( - "FAILED to pause Time Machine (tmutil stopbackup). State stays at '\(state.rawValue)', retrying on every subsequent check. Time Machine KEEPS WRITING - the disk may fill up." - ) - reportedStopFailure = true - } - } - - /// KEEPS UP the pause while in a paused state - on every tick. - /// - /// THIS is the fix. `stopBackup()` was called ONLY on a state change, and - /// `tmutil stopbackup` cancels only the RUNNING backup and does not touch the - /// schedule (`tmutil disable` does not appear in this repo even once). An - /// hour after the pause macOS therefore started another backup: the watchdog - /// neither stopped nor supervised it, because the `.pausedForBuffer` branch - /// looked at nothing except the resume condition - and the log said "waiting - /// for the upload". The pause held back writes for one run, even though the - /// state itself lasted 53 hours. `pause` describes and fixes exactly this - /// flaw for the FAILURE branch of `stopbackup`; for success it stayed - /// untouched until 25.09.2026. - /// - /// We repeat only when tmutil does not say plainly "no backup running": - /// "I do not know" (`nil`) counts here as "running", because no answer is not - /// proof of quiet. With Time Machine idle this saves two processes (sudo + - /// tmutil) every 30 seconds for the whole pause - in the 23.09 episode it - /// would have been over 12 thousand of them. - /// - /// REJECTED ALTERNATIVE: `tmutil disable`. It turns the schedule off once and - /// for good, so the pause would hold without repeating - but the watchdog - /// keeps the pause state IN PROCESS MEMORY, and it runs under launchd with - /// `KeepAlive`. After its death (or after a Mac restart) nobody would know - /// that Time Machine had been disabled and has to be turned back on - a - /// silent loss of backups forever instead of a slower backup. A repeated - /// `stopbackup` is reversible by itself: when the watchdog stops working, - /// Time Machine goes back to work in its hourly cycle. - private func keepPaused(backupRunning: Bool?) async { - guard backupRunning != false else { - reportedRestop = false - return - } - if !reportedRestop { - probes.log( - "Time Machine is running during a pause (state '\(state.rawValue)') - repeating the pause. `tmutil stopbackup` cancels only the running run, and macOS starts another one in its hourly cycle." - ) - reportedRestop = true - } - if await probes.stopBackup() { - reportedStopFailure = false - return - } - if !reportedStopFailure { - probes.log( - "FAILED to repeat the Time Machine pause (tmutil stopbackup) in state '\(state.rawValue)'. Time Machine KEEPS WRITING to the buffer we are waiting to drain - the disk may fill up." - ) - reportedStopFailure = true - } - } - - /// Resumes Time Machine. - /// - /// The asymmetry with `pause` is deliberate. A failed `stopbackup` risks - /// filling the disk, so we must not pretend the pause happened. A failed - /// `startbackup` risks nothing: Time Machine will start by itself in its - /// hourly cycle anyway, and `startbackup` only speeds that up. If we stayed - /// paused when it failed, the watchdog would be stuck in a state whose only - /// exit is exactly what is not working. - private func resume(backlogGB: Int?, freeGB: Int?) async { - probes.log("RESUME: backlog \(describeBacklog(backlogGB)), free \(describe(freeGB))") - reportedRestop = false - if await probes.startBackup() == false { - probes.log( - "tmutil startbackup failed - Time Machine will start by itself in its hourly cycle.") - } - state = .running - } - - /// What to do with the jam marker. A pure function, so that "I do not know" - /// can be tested without a marker file, without a notification and without - /// a log. - /// - /// `stalled == nil` is `.doNothing`, and that is the whole fix. Previously an - /// unreadable rclone log came out of `uploadStalled()` as `false`, `false` - /// meant "the jam is over" - so the marker was DELETED, and the log got - /// "Upload to Google Drive has resumed". A claim about an event nobody - /// checked, and the report cleared in exactly the case it exists for. - enum StallAction: Equatable { - case raise - case clear - case doNothing - } - - static func stallAction(stalled: Bool?, markerExists: Bool) -> StallAction { - guard let stalled else { return .doNothing } - guard stalled != markerExists else { return .doNothing } - return stalled ? .raise : .clear - } - - /// Reports the start and end of an upload jam - once per state change. - /// - /// We keep the state in a file, not a field, because `buffer-guard` runs - /// under launchd with `KeepAlive`: after every resurrection of the process the - /// field would start from zero and the same jam would be reported anew every - /// 30 seconds. - /// - /// `nil` = "I do not know", and then we touch NOTHING - see `stallAction`. - static func reportUploadStall(_ stalled: Bool?) async { - let marker = CMPaths.appSupportDir.appendingPathComponent(".upload-stalled") - let reported = FileManager.default.fileExists(atPath: marker.path) - let action = stallAction(stalled: stalled, markerExists: reported) - guard action != .doNothing else { return } - - if action == .raise { - CMLogger.log( - "Upload to Google Drive is stalled - daily limit exhausted. Backups continue, the jam passes by itself within a few hours." - ) - // The marker is created ONLY after the notification is delivered. Created - // earlier, it closed the matter even when the notification did not get - // through - i.e. it cleared the report in exactly the case it exists for - // (the same bug as in `HealthAlert.report`, see the comment there). - if await HealthAlert.notify( - title: L10n.tr("CloudMachine: upload to Drive is stalled"), - message: L10n.tr("Upload to Google Drive is stalled - daily limit exhausted.")) - { - FileManager.default.createFile(atPath: marker.path, contents: nil) - } else { - CMLogger.log( - "The upload jam notification was NOT delivered - will retry on the next check." - ) - } - } else { - try? FileManager.default.removeItem(at: marker) - CMLogger.log("Upload to Google Drive has resumed.") - } - } - - /// Calls `tmutil ` and says whether it REALLY succeeded. - /// - /// First via `sudo -n`: `tmutil stopbackup` and `startbackup` need root - /// privileges, and the watchdog runs under launchd in the user session. The - /// existing `runTmutilUnattended` was unused here, even though it was made - /// exactly for this. The NOPASSWD rule in `/etc/sudoers.d/cloudmachine` is not - /// set up by anything in this repo (checked: no installer writes it), so - /// `sudo -n` refuses immediately today - and that is exactly why, on an - /// AUTHORIZATION refusal, we still try without sudo instead of giving up. - /// `isSudoAuthFailure` tells "sudo did not let us in" apart from "the command - /// ran and returned an error". - static func tmutil(_ command: String) async -> Bool { - if let viaSudo = try? await ProcessRunner.runTmutilUnattended([command], timeout: 120) { - if viaSudo.succeeded { return true } - if !viaSudo.isSudoAuthFailure { - CMLogger.log( - "sudo tmutil \(command): exit code \(viaSudo.exitCode) \(shortError(viaSudo))") - return false - } - } - guard let direct = try? await ProcessRunner.run("/usr/bin/tmutil", [command], timeout: 120) - else { - CMLogger.log("tmutil \(command): NO ANSWER within the time limit.") - return false - } - if !direct.succeeded { - CMLogger.log("tmutil \(command): exit code \(direct.exitCode) \(shortError(direct))") - } - return direct.succeeded - } - - private static func shortError(_ result: ProcessResult) -> String { - let text = (result.stderr + " " + result.stdout).trimmingCharacters(in: .whitespacesAndNewlines) - return text.isEmpty ? "(no message)" : text.replacingOccurrences(of: "\n", with: " ") - } -} diff --git a/mac-app/Sources/CloudMachineCore/BufferReadiness.swift b/mac-app/Sources/CloudMachineCore/BufferReadiness.swift deleted file mode 100644 index 215e59e..0000000 --- a/mac-app/Sources/CloudMachineCore/BufferReadiness.swift +++ /dev/null @@ -1,51 +0,0 @@ -import Foundation - -/// When the buffer is ready enough to attach the image on it. -/// -/// Split out of `attach-image` so that it can be tested without mounting -/// anything - just like `CooldownGate` and `TimeMachineStatus`. -public enum BufferReadiness { - /// How long to wait for a ready buffer before we treat it as a failure. - /// - /// Two minutes were enough as long as the buffer started empty. After an - /// unclean shutdown it is different: on 13 Sep 2026 rclone read 225 dirty - /// items (9.4 GB) after start-up, the mount came up at 07:56:30, and the wait - /// gave up at 07:56:20 - ten seconds too early. Time Machine was left without - /// a destination until the next launchd tick, i.e. for 15 minutes, and nobody - /// found out except through an exit code that nobody reads. - /// - /// Waiting is free - `attach` on an already attached image only says - /// "Already attached" - while a missed window costs a quarter of an hour - /// without a backup. - public static let defaultTimeout: TimeInterval = 900 - - /// How often to poll. - public static let defaultPoll: TimeInterval = 2 - - /// The mount alone is not enough. - /// - /// rclone exposes the mount BEFORE it reads the dirty cache, so for a while - /// the directory is empty. The attach would then fail with "No image" - that - /// is, on the same race, just one step later. - public static func isReady(mounted: Bool, imageVisible: Bool) -> Bool { - mounted && imageVisible - } - - /// Waits for the buffer to become ready. Returns `true` if it did. - /// - /// The clock and sleep are injected so that a test does not have to really wait. - public static func wait( - timeout: TimeInterval = defaultTimeout, - poll: TimeInterval = defaultPoll, - now: () -> Date = Date.init, - sleep: (TimeInterval) async -> Void, - probe: () -> Bool - ) async -> Bool { - let deadline = now().addingTimeInterval(timeout) - while true { - if probe() { return true } - if now() >= deadline { return false } - await sleep(poll) - } - } -} diff --git a/mac-app/Sources/CloudMachineCore/CMActionResult.swift b/mac-app/Sources/CloudMachineCore/CMActionResult.swift index 1607d41..2099711 100644 --- a/mac-app/Sources/CloudMachineCore/CMActionResult.swift +++ b/mac-app/Sources/CloudMachineCore/CMActionResult.swift @@ -13,52 +13,9 @@ public struct CMActionResult { /// handing the user a dead end. public var isSudoAuthFailure: Bool - /// `true` if the operation did NOT START at all - not because something went - /// wrong, but because the resource was held by another operation (see - /// `BackupImageService.busyResult`). - /// - /// A separate field rather than recognizing it by the content of `message`: - /// matching on text breaks with every change of the message, and does so - /// silently - meaning nobody notices until something fails. - public var didNotRun: Bool - - /// `true` if `hdiutil attach` was turned away with "Resource temporarily - /// unavailable" on every attempt - the failure that the mount restart in - /// `AttachStall` answers. Recognized once, where `hdiutil` runs, so callers - /// do not match on `message`. - public var attachTemporarilyUnavailable: Bool - - public init( - succeeded: Bool, message: String, isSudoAuthFailure: Bool = false, didNotRun: Bool = false, - attachTemporarilyUnavailable: Bool = false - ) { + public init(succeeded: Bool, message: String, isSudoAuthFailure: Bool = false) { self.succeeded = succeeded self.message = message self.isSudoAuthFailure = isSudoAuthFailure - self.didNotRun = didNotRun - self.attachTemporarilyUnavailable = attachTemporarilyUnavailable - } - - /// What the caller should do with this result - in particular, which exit - /// code a CLI command running under launchd should end with. - /// - /// It exists because "busy with another operation" is NOT a failure, yet - /// with exit code 1 it ended up in `launchd-gdrive-attach.err.log` - exactly - /// where a person looks when asking "is the backup working". It is the same - /// bug this code fights in the other direction ("no answer read as an - /// answer"), only reversed: a normal state read as a failure. - public enum Disposition: Equatable { - /// It worked. - case ok - /// Nothing happened and nothing broke. A periodic tick should simply try - /// again on the next run. - case skipped - /// A real failure - this one must be visible. - case failed - } - - public var disposition: Disposition { - if succeeded { return .ok } - return didNotRun ? .skipped : .failed } } diff --git a/mac-app/Sources/CloudMachineCore/CMLock.swift b/mac-app/Sources/CloudMachineCore/CMLock.swift deleted file mode 100644 index 45889e8..0000000 --- a/mac-app/Sources/CloudMachineCore/CMLock.swift +++ /dev/null @@ -1,280 +0,0 @@ -import Foundation - -/// A lock based on an atomic `mkdir` (not `flock`, which macOS does not have -/// by default), with orphan detection by the PID stored in the lock (not by -/// the directory's age - the same motivation as in the bash `cm_acquire_lock`: -/// an operation under the lock can legitimately take long, e.g. catching up -/// with a backlog of uploads, so a fixed time threshold would falsely mark a -/// live process as orphaned). -public final class CMLock { - private let lockDir: URL - private var acquired = false - - public init(name: String) { - lockDir = CMPaths.logDir.appendingPathComponent("\(name).lock.d") - } - - /// For tests only: a lock in a directory the test cleans up itself. Without - /// this, every lock test would leave directories in the real `~/Library/Logs/ - /// CloudMachine`, i.e. where the running installation keeps its production - /// locks - a test could block the agent. - init(directory: URL) { - lockDir = directory - } - - /// Tries to take the lock. Returns `false` if another live process already holds it. - public func acquire() -> Bool { - if (try? FileManager.default.createDirectory(at: lockDir, withIntermediateDirectories: false)) - != nil - { - // Verification by reading back is needed here for EXACTLY the same - // reason as on the takeover path below - until 23 September 2026 it was - // only there. `writePid()` swallows a write error (`try?`), and a - // process killed between `createDirectory` and the write leaves the - // lock directory WITHOUT a `pid` file. A competitor then reads an empty - // directory, finds no PID, considers the lock orphaned and takes it over - // - while we have already returned `true` and are acting in the belief - // that we have exclusivity. Two owners of the same "image" lock means a - // parallel `detach` and `attach` on the same image, i.e. exactly what - // the lock is meant to protect against. - guard writeAndVerifyPid() else { - try? FileManager.default.removeItem(at: lockDir) - return false - } - acquired = true - return true - } - // The directory already exists - check whether the owner is still alive. - let pidFile = lockDir.appendingPathComponent("pid") - if let content = try? String(contentsOf: pidFile, encoding: .utf8), - let pid = Self.parsePid(from: content), - isProcessAlive(pid, recordedStartTime: Self.parseStartTime(from: content)) - { - return false - } - // The owner process is no longer alive (or the lock was interrupted before - // it managed to write its PID) - orphaned, we take it over. - try? FileManager.default.removeItem(at: lockDir) - guard - (try? FileManager.default.createDirectory(at: lockDir, withIntermediateDirectories: false)) - != nil - else { - return false - } - guard writeAndVerifyPid() else { return false } - acquired = true - return true - } - - /// Writes our own PID and CONFIRMS it by reading it back. `false` means "I - /// cannot prove the lock is mine" - and that must end in giving up, not in - /// optimism. - /// - /// IMPORTANT: `removeItem` + `createDirectory` on the takeover path is NOT - /// atomic - two processes taking over the same orphaned lock in an - /// overlapping window can both read "dead PID", both remove and recreate the - /// directory, both write their PID - and both return `true`. We read the - /// file we just wrote back: if another process managed to overwrite it with - /// its PID between the write and this read, we know we lost the race, and - /// we back off instead of acting in the false belief of exclusivity. This - /// does not eliminate the window completely (both sides can still - /// transiently "win" just before this verification), but it guarantees that - /// at least one of them will detect it and back off. - private func writeAndVerifyPid() -> Bool { - writePid() - guard - let verifyContent = try? String( - contentsOf: lockDir.appendingPathComponent("pid"), encoding: .utf8), - Self.parsePid(from: verifyContent) == getpid() - else { - return false - } - return true - } - - /// Checks whether the process with the given PID is still alive AND not - /// permanently stuck in an uninterruptible kernel wait (state 'U' from - /// `ps`). Uses a synchronous `popen("ps -p -o stat=")` because - /// `acquire()` is sync - we cannot wait for the async ProcessRunner here. - /// - /// A process in state U passes `kill(pid, 0) == 0`, but will never release - /// the lock by itself (SIGKILL does not wake it from an NFS/I-O wait in the - /// kernel). We treat it as "dead for locking purposes" after - /// `stuckThreshold` minutes - a threshold large enough not to conflict with - /// legitimate long operations (catching up with the upload queue, checksum - /// verification that can take hours). - private static let stuckLockThreshold: TimeInterval = 15 * 60 // 15 minutes - - /// Marker "since when we have continuously seen this process in state U/D" - /// - DELIBERATELY separate from the mtime of `lockDir` (the time the lock - /// was TAKEN). A legitimate long-running operation (checksum verification - /// lasting hours) can hold the same lock for a long time BEFORE it ever - /// falls into U/D - counting the threshold from the time the lock was taken - /// (as the earlier code did) kills such a legitimate, long-running process - /// at the first U/D sample after the threshold has passed, which is exactly - /// what the comment in this file's header warns against (orphan detection - /// BY PID, NOT by age). We write this marker on the FIRST observed U/D and - /// delete it when the process returns to a normal state - so it measures - /// the actual, UNINTERRUPTED duration of the hang. - private var stuckSinceFile: URL { lockDir.appendingPathComponent("stuck-since") } - - /// `recordedStartTime` is the owner process's start time WRITTEN to the - /// lock file when the lock was taken (`writePid()`) - it allows telling - /// "the same process is still alive" apart from "the PID has already been - /// reused by a completely different, newer process" (`kill(pid, 0) == 0` on - /// its own cannot tell these apart - it sees ONLY that SOMETHING is alive - /// under that PID number). The risk is very low in practice (the PID space - /// is large, watchdogs fire rarely), but it costs little to check. `nil` (a - /// lock file written before this field was introduced, or `ps` briefly did - /// not respond at write time) skips this extra verification instead of - /// falsely assuming an orphan. - private func isProcessAlive(_ pid: pid_t, recordedStartTime: String?) -> Bool { - guard kill(pid, 0) == 0 else { return false } - - if let recordedStartTime, !recordedStartTime.isEmpty, - let currentStartTime = Self.processStartTime(pid: pid), - currentStartTime != recordedStartTime - { - return false - } - - // Synchronously fetch the process state from `ps`. - // State 'U' (macOS uninterruptible NFS wait) or 'D' (Linux disk sleep) - // - both mean a hang in the kernel that SIGKILL does not wake from. - guard let psState = processState(pid: pid), - psState.uppercased().hasPrefix("U") || psState.uppercased().hasPrefix("D") - else { - // The process is running normally (or `ps` briefly did not respond) - - // delete the marker, in case the process fell into U/D transiently and - // came back. - try? FileManager.default.removeItem(at: stuckSinceFile) - return true - } - - if let stuckSinceString = try? String(contentsOf: stuckSinceFile, encoding: .utf8), - let stuckSinceEpoch = TimeInterval( - stuckSinceString.trimmingCharacters(in: .whitespacesAndNewlines)) - { - let stuckDuration = Date().timeIntervalSince1970 - stuckSinceEpoch - guard stuckDuration > Self.stuckLockThreshold else { return true } - CMLogger.log( - "[CMLock] WARNING: PID \(pid) holding the lock '\(lockDir.lastPathComponent)'" - + " has been CONTINUOUSLY stuck in state U/D (NFS hang?) for more than" - + " \(Int(Self.stuckLockThreshold/60)) min. Taking over the lock. SIGKILL to the stuck" - + " process (ignored by the kernel, but it will clean up the PID after unmounting)." - ) - kill(pid, SIGKILL) - try? FileManager.default.removeItem(at: stuckSinceFile) - return false - } - // First sample in state U/D - record the start of the window and wait for - // further samples before considering the process stuck. - try? "\(Date().timeIntervalSince1970)".write( - to: stuckSinceFile, atomically: true, encoding: .utf8) - return true - } - - /// Synchronous run of `ps -p -o ` via Process/Pipe - shared by - /// `processState` (`stat=`) and `processStartTime` (`lstart=`). The blocking - /// use of `readDataToEndOfFile()` is safe because `ps` ALWAYS finishes - /// quickly and closes its FDs - no risk of waiting forever for EOF (that - /// problem concerns only daemons such as rclone --daemon, which fork a child - /// that inherits the FDs and never closes them). - private static func runPS(pid: pid_t, format: String) -> String? { - let proc = Process() - proc.executableURL = URL(fileURLWithPath: "/bin/ps") - proc.arguments = ["-p", "\(pid)", "-o", format] - // IMPORTANT: `ps` formats dates (e.g. `lstart=`) according to the - // LC_TIME/LANG of the process that CALLS it - NOT of the one whose PID we - // check. Without forcing a fixed locale, the same live process got a - // DIFFERENT start-time text depending on who was checking it (e.g. the - // Polish "sr. 29 lip 10:01:01 2026" from the user's shell with - // LANG=pl_PL.UTF-8, but "Wed Jul 29 10:01:01 2026" from a watchdog started - // by launchd with another/default locale) - which `isProcessAlive` - // misread as "the PID was reused by another process" and which allowed - // the lock to be stolen from a still-living process. Observed live: two - // parallel `rclone copy` runs to the same destination. `LC_ALL=C` - // guarantees the same text regardless of who asks. - proc.environment = ["LC_ALL": "C"] - let pipe = Pipe() - proc.standardOutput = pipe - proc.standardError = FileHandle.nullDevice - proc.standardInput = FileHandle.nullDevice - guard (try? proc.run()) != nil else { return nil } - let data = pipe.fileHandleForReading.readDataToEndOfFile() - proc.waitUntilExit() - let text = String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) - return (text?.isEmpty == false) ? text : nil - } - - private func processState(pid: pid_t) -> String? { - Self.runPS(pid: pid, format: "stat=") - } - - /// The process start time (e.g. "Wed Jul 29 09:07:59 2026") - a unique - /// "fingerprint" of a specific process instance under a given PID, used to - /// detect PID reuse (see `isProcessAlive`). - private static func processStartTime(pid: pid_t) -> String? { - runPS(pid: pid, format: "lstart=") - } - - private func writePid() { - let pidFile = lockDir.appendingPathComponent("pid") - let pid = getpid() - let startTime = Self.processStartTime(pid: pid) ?? "" - try? "\(pid)\n\(startTime)".write(to: pidFile, atomically: true, encoding: .utf8) - } - - private static func parsePid(from content: String) -> pid_t? { - guard let firstLine = content.split(separator: "\n", maxSplits: 1).first else { return nil } - return pid_t(firstLine.trimmingCharacters(in: .whitespacesAndNewlines)) - } - - /// `nil` for lock files written before this field was introduced (only one - /// line with the PID) - `isProcessAlive` treats that as a lack of extra - /// information, not as proof of PID reuse. - private static func parseStartTime(from content: String) -> String? { - let lines = content.split(separator: "\n", maxSplits: 1) - guard lines.count == 2 else { return nil } - let trimmed = lines[1].trimmingCharacters(in: .whitespacesAndNewlines) - return trimmed.isEmpty ? nil : trimmed - } - - public func release() { - guard acquired else { return } - try? FileManager.default.removeItem(at: lockDir) - acquired = false - } - - deinit { - if acquired { - try? FileManager.default.removeItem(at: lockDir) - } - } -} - -/// Runs `body` under the lock `name`, releasing it automatically on exit -/// (also when an error is thrown) - the counterpart of `cm_acquire_lock` + -/// `trap EXIT`. Returns `nil` without calling `body` if another live instance -/// already holds the lock. -/// -/// `nil` means **"nothing happened"** and the caller MUST tell it apart from -/// the result of `body`. Code such as `(await withCMLock("image") { ... }) ?? -/// true` or `!= nil ? ... : ...` that turns not running into success is a bug: -/// an `attach` that did not happen would report "Attached", and a `detach` -/// that did not run - "everything uploaded". -@discardableResult -public func withCMLock(_ name: String, _ body: () throws -> T) rethrows -> T? { - let lock = CMLock(name: name) - guard lock.acquire() else { return nil } - defer { lock.release() } - return try body() -} - -@discardableResult -public func withCMLock(_ name: String, _ body: () async throws -> T) async rethrows -> T? { - let lock = CMLock(name: name) - guard lock.acquire() else { return nil } - defer { lock.release() } - return try await body() -} diff --git a/mac-app/Sources/CloudMachineCore/CMPaths.swift b/mac-app/Sources/CloudMachineCore/CMPaths.swift index a22cbdb..3fa7e5c 100644 --- a/mac-app/Sources/CloudMachineCore/CMPaths.swift +++ b/mac-app/Sources/CloudMachineCore/CMPaths.swift @@ -1,6 +1,6 @@ import Foundation -/// Resolves all paths used by CloudMachine - config, logs, mount points, and +/// Resolves all paths used by CloudMachine - config, logs, the state directory and /// (for the CLI/launchd) the resources directory (`launchd/`, /// `config/machines.example.json`) - regardless of whether the code runs as a /// packaged .app (Resources are read-only, next to the binary in @@ -77,6 +77,15 @@ public enum CMPaths { public static var configPath: URL { appSupportDir.appendingPathComponent("machines.json") } + /// `~/.cloudmachine`: the managed rclone (`bin/`), the machine's state + /// (`vm/`). Created on first use. + public static var stateDir: URL { + let dir = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".cloudmachine") + try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + return dir + } + /// Under XCTest, a temporary directory of this test process. `swift test` /// used to append to the real `cloudmachine.log`: lock messages from image /// tests and a "BACKUP FAILURE: ... test canary" line from HealthAlertTests, diff --git a/mac-app/Sources/CloudMachineCore/CMTooling.swift b/mac-app/Sources/CloudMachineCore/CMTooling.swift index af42974..9ac423c 100644 --- a/mac-app/Sources/CloudMachineCore/CMTooling.swift +++ b/mac-app/Sources/CloudMachineCore/CMTooling.swift @@ -1,17 +1,11 @@ import Foundation -/// Resolves the external tools the Google Drive layer needs and checks whether -/// they are usable at all. +/// Resolves the external tools the Mac side needs and checks whether they are +/// there: the official rclone (Google sign-in, `rclone.conf` for the machine, +/// usage measurements) and Lima (the machine itself). /// -/// It exists because `ProcessRunner.runRclone` calls `/usr/bin/env rclone`, -/// and that hits rclone from Homebrew - built WITHOUT FUSE support. When asked -/// to mount, it refuses outright: -/// -/// rclone mount is not supported on MacOS when rclone is installed via Homebrew -/// -/// The official binary from rclone.org is needed. We keep it in our own -/// directory so as not to clash with the Homebrew installation, which the -/// remaining, non-mounting code paths use. +/// rclone lives in our own directory, so the configuration is always written +/// and read by the same binary, whatever Homebrew has installed. public enum CMTooling { // MARK: - rclone @@ -24,7 +18,7 @@ public enum CMTooling { return dir } - /// Official rclone binary with mount support. + /// The official rclone binary, managed by CloudMachine. public static var managedRclonePath: URL { toolsDir.appendingPathComponent("rclone") } @@ -33,8 +27,8 @@ public enum CMTooling { FileManager.default.isExecutableFile(atPath: managedRclonePath.path) } - /// Runs an rclone that CERTAINLY can mount. Every code path touching - /// mounting must go through here, not through `ProcessRunner.runRclone`. + /// Runs the managed rclone. Every code path that reads or writes the + /// Google Drive configuration goes through here. public static func runRclone(_ args: [String], timeout: TimeInterval? = nil) async throws -> ProcessResult { @@ -48,8 +42,8 @@ public enum CMTooling { /// Creates a symlink to the agent binary in PATH. /// - /// Without it, every command from the documentation - `prepare-shutdown`, - /// `drive-status` - ends in "command not found", because the binary sits + /// Without it, every command from the documentation - `vm-setup`, + /// `vm-status` - ends in "command not found", because the binary sits /// inside the app bundle. Root is not needed: `/usr/local/bin` belongs to /// the user and the admin group. @discardableResult @@ -75,48 +69,6 @@ public enum CMTooling { } } - // MARK: - FUSE - - /// Our copy of FUSE-T - so as not to keep a separate application in the - /// system. See `FuseInstaller`. - public static var bundledFuseDir: URL { - let dir = FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent(".cloudmachine/fuse") - try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - return dir - } - - public static var bundledFuseLib: URL { bundledFuseDir.appendingPathComponent("libfuse-t.dylib") } - - /// The NFS server that actually holds the mount. Its path can be given via - /// the `FUSE_NFSSRV_PATH` variable, so it can live in our directory. - public static var bundledNfsServer: URL { bundledFuseDir.appendingPathComponent("go-nfsv4") } - - /// Paths where FUSE may live. One is enough. - private static let fuseCandidates = [ - "/usr/local/lib/libfuse-t.dylib", - "/usr/local/lib/libfuse.2.dylib", - "/usr/local/lib/libfuse.dylib", - "/Library/Filesystems/fuse-t.fs", - ] - - /// Whether FUSE is installed. - /// - /// BEWARE of a trap that has already sprung once: the first version of this - /// check in bash ran `ls a b c` and checked the exit code. `ls` returns an - /// error when ANY of the paths is missing, not when all of them are - so it - /// refused to start with a correctly installed FUSE-T. We check one by one. - public static var hasFuse: Bool { - // Our own copy counts the same as a system installation: we can recreate - // the symlink in /usr/local/lib ourselves (FuseInstaller.ensureSystemLink), - // so its temporary absence does not mean there is no FUSE. The FUSE-T - // uninstaller deletes that symlink when removing the separate application - // - without this condition the status would then report FUSE missing even - // though mounting works. - if FileManager.default.fileExists(atPath: bundledFuseLib.path) { return true } - return fuseCandidates.contains { FileManager.default.fileExists(atPath: $0) } - } - // MARK: - Readiness diagnostics public struct Readiness { @@ -128,21 +80,15 @@ public enum CMTooling { } public static func checkReadiness() -> Readiness { - // The check is also an opportunity to repair - the symlink is sometimes - // deleted by the FUSE-T uninstaller and there is no reason to wait with - // that until the next startup. - FuseInstaller.ensureSystemLink() - var missing: [String] = [] var remedies: [String] = [] - if !hasManagedRclone { - missing.append(L10n.tr("rclone with mount support")) + missing.append("rclone") remedies.append("cloudmachine-agent install-rclone") } - if !hasFuse { - missing.append("FUSE") - remedies.append("cloudmachine-agent install-fuse") + if VMLayout.limactlPath == nil { + missing.append("Lima") + remedies.append("brew install lima") } return Readiness(missing: missing, remedies: remedies) } diff --git a/mac-app/Sources/CloudMachineCore/DriveBufferService.swift b/mac-app/Sources/CloudMachineCore/DriveBufferService.swift deleted file mode 100644 index 9d9d132..0000000 --- a/mac-app/Sources/CloudMachineCore/DriveBufferService.swift +++ /dev/null @@ -1,959 +0,0 @@ -import Foundation - -/// The buffer between Time Machine and Google Drive - a port of `gdrive/mount-drive.sh`. -/// -/// Mounts Drive as a volume with a local write cache. A write completes the -/// moment it reaches the cache, the upload goes on in the background - which -/// is why a broken link pauses the drain instead of interrupting the backup. -/// -/// The rclone process stays in the foreground; launchd manages its life cycle -/// (KeepAlive). -public enum DriveBufferService { - - // MARK: - Paths and settings - - public static var root: URL { - let dir = FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent(".cloudmachine") - try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - return dir - } - - public static var mountPoint: URL { root.appendingPathComponent("drive") } - public static var cacheDir: URL { root.appendingPathComponent("cache") } - public static var logFile: URL { root.appendingPathComponent("rclone.log") } - - public static let remoteName = "gdrive" - /// Top-level folder on Drive; each Mac has its own subfolder in it. - public static let remoteRoot = "CloudMachine" - /// This Mac's folder - see `DriveFolder` for why the name is fixed once set. - public static var remotePath: String { "\(remoteRoot)/\(DriveFolder.name)" } - /// Buffer size. Kept as a number, because the watchdog's thresholds are - /// derived from it - otherwise changing one without the other gives - /// thresholds that never fire or fire immediately. - public static let cacheSizeGB = 100 - public static var cacheSize: String { "\(cacheSizeGB)G" } - - /// How long rclone waits after a band's last change before uploading it. - /// - /// This is NOT a caution setting, but a bumper against write amplification. - /// Time Machine rewrites the same bands throughout a run, and with a short - /// delay every touch is a full 32 MB uploaded again. Measured on our own log - /// (56,533 intervals between consecutive uploads of THE SAME band): the median - /// interval is 9.5 minutes, so 10 minutes merges about half of the repeats. - /// Going longer pays off less and less (15 min -> 57%, 30 min -> 70%), while - /// the window in which data exists ONLY locally grows. - /// - /// History: it was 30 s, and with that value the day of 14/15 September 2026 - /// pushed 823 GB to Drive for about 45 GB of real change - i.e. over Google's - /// daily limit (750 GB), which blocked the upload for several hours. - /// - /// Every shutdown path MUST force the upload via `expireQueuedUploads()`, - /// otherwise a detach would wait as long as this delay. - public static let writeBackSeconds = 600 - - /// rclone's remote control interface: a unix socket in a directory only this - /// user can enter. - /// - /// Until 09.10.2026 it was `127.0.0.1:5572` with `--rc-no-auth`. "Loopback - /// only" did not mean "only us": any web page open in a browser can send a - /// form POST there (a "simple request" goes out without a CORS preflight, - /// and rclone does not check `Origin`). The answer stays unreadable to the - /// page, but the command runs - and the interface offers `operations/purge` - /// on the backup folder, with `--drive-use-trash=false`, so nothing to undo. - /// Checked on the running mount: `config/listremotes` answered a form POST - /// with `Origin: https://evil.example`. - /// - /// A browser cannot reach a unix socket at all, and other users cannot enter - /// `runDir` (0700). `--rc-no-auth` stays: `vfs/queue-set-expiry`, which every - /// detach needs, requires it or a password. A password would add nothing - /// against the one remaining caller, a process of this same user - it can - /// read `rclone.conf`, token included, directly - and every restart that - /// rotated it would be a new way for the readers to lose the queue. - public static var runDir: URL { root.appendingPathComponent("run") } - public static var rcSocket: URL { runDir.appendingPathComponent("rc.sock") } - - /// Where a mount started by CloudMachine 1.3.7 or older still listens. Such a - /// mount survives an upgrade (`gdrive-buffer` is deliberately not - /// restarted - see `AgentRepair`), and until it is restarted it is the only - /// interface there is. Without this fallback the upgrade would leave the - /// buffer watchdog and every detach without a queue until the next restart. - public static let legacyRCAddress = "127.0.0.1:5572" - - /// Longest path a unix socket address can hold on macOS (`sun_path`, with the - /// terminating zero). rclone fails to start with `bind: invalid argument` - /// above it. - static let socketPathLimit = 103 - - public enum RCTransport: Equatable, Sendable { - /// The private socket - a mount started by this version. - case socket - /// The old TCP address - a mount from before the socket, still running. - case legacyTCP - } - - /// Which interface the running mount listens on. The socket file is there - /// exactly as long as a mount of this version has it (`prepare` removes a - /// stale one before every start). - public static var rcTransport: RCTransport { - FileManager.default.fileExists(atPath: rcSocket.path) ? .socket : .legacyTCP - } - - /// Whether the running mount still takes commands on the open TCP port - - /// a mount started by 1.3.7 or older that survived the upgrade. - /// - /// Until 11.10.2026 only `drive-status` said so, in one line; the panel and - /// the watchdog stayed silent, and such a mount runs until the next restart - /// of the Mac - on this project's own Mac, for days. All three now ask here. - /// - /// Three conditions, each needed: no socket (a mount of this version has - /// one), a mount (no mount, nothing listens), and an answer on the port. - /// The last one tells the old mount apart from a socket removed under a new - /// one, where nothing listens on TCP and nothing is exposed. - public static func legacyPortOpen( - transport: RCTransport, mounted: Bool?, portAnswers: () -> Bool - ) -> Bool { - guard transport == .legacyTCP, mounted == true else { return false } - return portAnswers() - } - - /// `legacyPortOpen` for this Mac, now. - public static func legacyPortOpenNow() -> Bool { - legacyPortOpen( - transport: rcTransport, mounted: mountedState(), - portAnswers: { tcpPortAnswers(port: legacyRCPort) }) - } - - static let legacyRCPort: UInt16 = 5572 - - /// Whether something accepts a connection on `127.0.0.1:port`. Connects - /// and closes - nothing is sent, so a listener gets no request. - static func tcpPortAnswers(port: UInt16) -> Bool { - let fd = socket(AF_INET, SOCK_STREAM, 0) - guard fd >= 0 else { return false } - defer { close(fd) } - var address = sockaddr_in() - address.sin_len = UInt8(MemoryLayout.size) - address.sin_family = sa_family_t(AF_INET) - address.sin_port = port.bigEndian - address.sin_addr.s_addr = inet_addr("127.0.0.1") - return withUnsafePointer(to: &address) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { - connect(fd, $0, socklen_t(MemoryLayout.size)) == 0 - } - } - } - - /// Arguments that point `rclone rc` at the running mount. - static func rcClientArguments(_ transport: RCTransport) -> [String] { - switch transport { - case .socket: return ["--unix-socket", rcSocket.path] - case .legacyTCP: return ["--url", legacyRCAddress] - } - } - - /// Calls the running mount's remote control interface. The ONLY way to it - - /// so that no caller is left on the old address. - static func rc(_ method: [String], timeout: TimeInterval) async throws -> ProcessResult { - try await CMTooling.runRclone( - ["rc"] + rcClientArguments(rcTransport) + method, timeout: timeout) - } - - /// Above this size the rclone log is trimmed at start-up. rclone does not - /// rotate its own log, and this project has already lost 3.3 GiB once to a log - /// that grew without limit. - public static let logSizeLimit: UInt64 = 100 * 1024 * 1024 - - // MARK: - State - - /// Mount points straight from the kernel table. `nil` = the table COULD NOT - /// be read, which is something else than "nothing is mounted". - /// - /// WHY NOT `/sbin/mount` - /// - /// Until 23 September 2026 this ran `/sbin/mount` with - /// `readDataToEndOfFile()` + `waitUntilExit()` WITHOUT a time limit. With a - /// dead FUSE-T mount (the ENXIO incident of 22.09) such a read can enter - /// uninterruptible I/O and never return - and `isMounted` reads from here, - /// i.e. the `backup-health` monitor AND the GUI refresh loop running every 10 - /// seconds. The hang thus froze both the view and the supervision, on the very - /// failure both are meant to detect. - /// - /// Adding a time limit (as in `TimeMachineStatus.commandTimeout`) would remove - /// the hang, but any such limit is pure cost here: with a refresh every 10 s - /// the calls would start to overlap, and the answer would not come anyway. - /// `getmntinfo(MNT_NOWAIT)` removes the problem at the source - it reads the - /// mount table from kernel memory and does NOT query any file system (that is - /// what `MNT_WAIT` does, which is exactly what could hang). There is no - /// process, pipe or I/O here, so there is nothing to put a limit on. Measured - /// on this machine: 19 mounts in 0.0002 s. - /// - /// Text parsing goes away as a bonus: `f_mntonname` is the path itself, - /// instead of searching for `" on "` in the output. - public static func mountPoints() -> [String]? { - var raw: UnsafeMutablePointer? - let count = getmntinfo(&raw, MNT_NOWAIT) - guard count > 0, let raw else { return nil } - return (0.. Bool? { - guard let points = mountPoints() else { return nil } - // The mount table is the source of truth - the directory merely existing - // means nothing, because the mount point stays on disk after unmounting. - return points.contains(mountPoint.path) - } - - /// Shortcut for places where a failed read and "not mounted" mean the same - /// thing - i.e. where we wait for the mount anyway or only print it. - /// Wherever a DECISION follows from the answer, use `mountedState()`. - public static var isMounted: Bool { mountedState() ?? false } - - // MARK: - Start-up - - /// Arguments for `rclone mount`. Split out so that they can be tested without - /// running anything. - public static func mountArguments() -> [String] { - [ - "mount", "\(remoteName):\(remotePath)", mountPoint.path, - "--vfs-cache-mode", "full", - "--vfs-cache-max-size", cacheSize, - // The cache must not evict data waiting to be uploaded - hence the high - // age. Size is governed by --vfs-cache-max-size. - "--vfs-cache-max-age", "9999h", - "--vfs-write-back", "\(writeBackSeconds)s", - "--vfs-cache-poll-interval", "1m", - "--cache-dir", cacheDir.path, - // ONLY this Mac writes to this folder, so change notifications from Drive - // carry only our own uploads - and each one invalidates the `bands` - // directory (18,853 files on 02.10.2026). The next `stat` reloaded it from - // Google in pages of 1000, HOLDING the directory lock: for ~42 s out of every - // minute every Getattr from FUSE (Time Machine, hdiutil), every `vfs/stats` - // and every upload completion stood still. That is where the hung - // `hdiutil attach`, `validateMountPoint timed out` in Time Machine and the red - // panel came from. Without notifications and with a long cache the directory - // loads once after start-up; rclone adds its own writes to it by itself. - "--poll-interval", "0", - "--dir-cache-time", "9999h", - "--attr-timeout", "5m", - "--transfers", "8", - // Chunk size matched to the image's band size. - "--drive-chunk-size", "32M", - // Without this, deleted bands go to Drive's trash and keep counting - // towards the storage limit. Kept after 09.10.2026, when the trash was - // weighed as a second line against a stray `operations/purge`: a - // deliberate deletion (re-creating the image) would then hold hundreds - // of GB for 30 days on an account where running out of space stops the - // mount. The private socket (see `rcSocket`) keeps such calls out; a - // process of this user can bypass the trash with its own rclone anyway. - "--drive-use-trash=false", - // After exceeding the daily 750 GB limit rclone is to stop, not spin in - // 403s until the end of the world. - "--drive-stop-on-upload-limit", - "--volname", remoteName, - "--rc", "--rc-addr", "unix://\(rcSocket.path)", "--rc-no-auth", - "--log-file", logFile.path, - "--log-level", "INFO", - ] - } - - /// Prepares the environment and returns the arguments to run with. Does not - /// start rclone itself - `cloudmachine-agent mount-drive` does that, and it - /// has to stay in the foreground under launchd. - public static func prepare() throws -> [String] { - try FileManager.default.createDirectory(at: mountPoint, withIntermediateDirectories: true) - try FileManager.default.createDirectory(at: cacheDir, withIntermediateDirectories: true) - try prepareRCSocketDirectory() - rotateLogIfLarge() - return mountArguments() - } - - public enum PrepareError: LocalizedError { - case socketPathTooLong(String) - - public var errorDescription: String? { - switch self { - case .socketPathTooLong(let path): - return L10n.tr( - "The remote control socket path is too long for macOS (%@ characters, at most %@): %@", - "\(path.utf8.count)", "\(socketPathLimit)", path) - } - } - } - - /// The socket's directory, private, and no stale socket in it. - /// - /// A socket file outlives an rclone that crashed or was killed, and rclone - /// does not remove it: the next start ends with `bind: address already in - /// use` (checked on rclone 1.75.1). Under launchd's KeepAlive that is a - /// mount that never comes back. So a socket nobody answers on is removed; - /// one that answers belongs to an rclone that is still running, and is left - /// alone - the new start then fails exactly as a second mount on the same - /// TCP port used to. - static func prepareRCSocketDirectory( - _ directory: URL = runDir, socket: URL = rcSocket - ) throws { - let path = socket.path - guard path.utf8.count <= socketPathLimit else { throw PrepareError.socketPathTooLong(path) } - let fm = FileManager.default - try fm.createDirectory( - at: directory, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700]) - // `createDirectory` leaves an existing directory's permissions as they are. - try fm.setAttributes([.posixPermissions: 0o700], ofItemAtPath: directory.path) - if fm.fileExists(atPath: path), !socketAnswers(path) { - try? fm.removeItem(atPath: path) - } - } - - /// Whether a process is listening on the unix socket at `path`. - static func socketAnswers(_ path: String) -> Bool { - let fd = socket(AF_UNIX, SOCK_STREAM, 0) - guard fd >= 0 else { return false } - defer { close(fd) } - var address = sockaddr_un() - address.sun_family = sa_family_t(AF_UNIX) - let bytes = Array(path.utf8) - guard bytes.count < MemoryLayout.size(ofValue: address.sun_path) else { return false } - withUnsafeMutableBytes(of: &address.sun_path) { raw in - raw.copyBytes(from: bytes) - raw[bytes.count] = 0 - } - let length = socklen_t(MemoryLayout.size) - return withUnsafePointer(to: &address) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { connect(fd, $0, length) == 0 } - } - } - - private static func rotateLogIfLarge() { - guard - let attrs = try? FileManager.default.attributesOfItem(atPath: logFile.path), - let size = attrs[.size] as? UInt64, size > logSizeLimit - else { return } - let rotated = previousLog(of: logFile) - try? FileManager.default.removeItem(at: rotated) - try? FileManager.default.moveItem(at: logFile, to: rotated) - } - - /// Where the older part of a log goes - at start-up and in `trimLogIfLarge`. - static func previousLog(of file: URL) -> URL { file.appendingPathExtension("1") } - - /// How much of the log `trimLogIfLarge` keeps in `.1`. Above the 4 MB the - /// stall detection reads (`uploadStalledState`), with room to spare. - public static let logKeepBytes: UInt64 = 32 * 1024 * 1024 - - /// Trims the log of a RUNNING mount: its last `keep` bytes go to `.1`, and - /// the log itself is truncated to zero. Returns whether it trimmed. - /// - /// `rotateLogIfLarge` only runs when the mount starts, and the mount runs - /// for days: on 10.10.2026 the log stood at 257 MB, growing ~60 MB a day. - /// - /// WHY COPY-AND-TRUNCATE, AND WHY THE TAIL GOES TO `.1` - /// - /// rclone keeps the file open, so renaming it would only move rclone's - /// writes to `.1`. It opens it with `O_APPEND` (checked live: `lsof +fg` - /// shows `W,AP` on both descriptors), so after a truncation its next line - /// lands at offset 0 - no hole, no garbage. Putting the kept tail back into - /// the live file instead was rejected: rclone appends at the same time, its - /// new lines could land before the old ones, and the detection reading - /// newest-to-oldest stops at the first line older than its window - an - /// out-of-order file reads as "no jam". The reader (`recentLog`) joins the - /// tail of `.1` and the live file in order instead, so a trim in the middle - /// of a jam does not hide it. - /// - /// rclone's own `--log-file-max-size` was rejected too: it applies only to - /// a mount started with it, so not to one that survives an upgrade, and its - /// backups get time-stamped names the reader would have to look for. - /// - /// Lines rclone writes between the last read and the truncation (micro- - /// seconds) are lost. A log, not a record - the price is accepted. - @discardableResult - public static func trimLogIfLarge( - _ file: URL = logFile, limit: UInt64 = logSizeLimit, keep: UInt64 = logKeepBytes - ) -> Bool { - guard - let attrs = try? FileManager.default.attributesOfItem(atPath: file.path), - let size = attrs[.size] as? UInt64, size > limit, - let handle = try? FileHandle(forUpdating: file) - else { return false } - defer { try? handle.close() } - do { - let end = try handle.seekToEnd() - try handle.seek(toOffset: end > keep ? end - keep : 0) - var tail = try handle.readToEnd() ?? Data() - // Start at a whole line, so the first line of `.1` has its timestamp. - if end > keep, let newline = tail.firstIndex(of: UInt8(ascii: "\n")) { - tail = tail[tail.index(after: newline)...] - } - try tail.write(to: previousLog(of: file), options: .atomic) - try handle.truncate(atOffset: 0) - } catch { - CMLogger.log("Could not trim \(file.path): \(error.localizedDescription)") - return false - } - CMLogger.log("Trimmed \(file.path) (\(size / 1_048_576) MB); its last part is in .1") - return true - } - - /// Excludes the buffer directory from Time Machine. The buffer holds a copy of - /// the backup data - if Time Machine covered it, it would back up its own - /// backup and grow forever. The exclusion is stored as an xattr on the - /// directory, so it disappears together with it; that is why we renew it on - /// every start-up, not once in setup. - @discardableResult - public static func excludeBufferFromTimeMachine() async -> Bool { - let result = try? await ProcessRunner.run( - "/usr/bin/tmutil", ["addexclusion", root.path], timeout: 30) - return result?.succeeded == true - } - - // MARK: - Upload queue - - public struct QueueStats { - public var uploadsInProgress: Int - public var uploadsQueued: Int - public var files: Int - public var erroredFiles: Int - /// Buffer size according to rclone itself. Computing it with our own walk of - /// the directory meant 6504 stat calls on every interface refresh, every 10 - /// seconds, on the same disk the backup is going to. - public var bytesUsed: UInt64 - /// rclone has nowhere left to put data - it has not managed to upload what - /// it holds, so there is nothing to evict. A stronger signal than any - /// threshold, because it comes from the one who really knows. - public var outOfSpace: Bool - - /// rclone is doing nothing RIGHT NOW. This is a STABILITY condition for - /// `hdiutil` on a FUSE-T mount (see `retryingFlakyMount`) and nothing more - - /// in particular it is NOT proof that the backup has reached Drive. - public var isIdle: Bool { uploadsInProgress == 0 && uploadsQueued == 0 } - - /// Nothing is waiting AND nothing was abandoned along the way. - /// - /// Until 23 September 2026 this question had only one answer - the one now - /// called `isIdle` - and it went into the detach message and into - /// `safeToRebootNow()`. A band that rclone abandoned drops out of the queue - /// exactly like an uploaded band: `uploadsQueued` returns to zero, and the - /// only trace is left in `erroredFiles`. Result: "Detached, everything - /// uploaded to Google Drive" and "Restart without asking: YES" with data - /// existing only locally - while `UploadState`, from the same counters, - /// already derived `.failedFiles(...)` and "ACTION NEEDED". - public var isQuiet: Bool { isIdle && erroredFiles == 0 } - - /// How many items are WAITING to be uploaded: the queue plus what is - /// uploading right now. - /// - /// This is the quantity from which the buffer watchdog derives its measure - /// (see `BufferGuardService.backlogGB`) - and NOT `bytesUsed`. The cache size - /// with `--vfs-cache-max-size 100G` and `--vfs-cache-max-age 9999h` sits at - /// the limit permanently (281 measurements in the journal, minimum 99 GB), - /// because rclone also keeps there what it uploaded long ago. The unsent - /// backlog is the only one of these two numbers that answers the question - /// "is the upload keeping up". - /// - /// `uploadsInProgress` is part of the sum, because an item being uploaded is - /// not on Drive yet either and also takes up the buffer. With `--transfers 8` - /// that is at most eight items, but an empty queue with eight transfers in - /// progress is not a zero backlog. - public var unsentItems: Int { uploadsQueued + uploadsInProgress } - } - - /// Reads the queue state through rclone's remote control interface. - /// - /// NOTE: `--rc-no-auth` is a SERVER flag. The `rclone rc` client does not - /// accept it and ends with an "unknown flag" error - this has already cost one - /// silent breakage of the status view. Hence one way to the interface: `rc`. - /// - /// Time limit 60 s, not 30 s: on 23.09.2026 the same call took **36.7 s** with - /// a clogged buffer (the next one 0.03 s - so sporadic, under load). With 30 s - /// it ended with `nil`, and `nil` went on as a set of zeros and the interface - /// announced "Everything uploaded" with 386 bands in the queue. Raising the - /// limit alone does not fix that - that is what `UploadState.queueUnknown` is - /// for - but it means the question usually gets an answer. - /// - /// Higher is not worth it. The interface refresh loop runs every 10 s and - /// waits for this read, and `drive-status` asks twice (the second time via - /// `safeToRebootNow`). With a dead rclone every second of the limit is a - /// second of a frozen window, and the answer will not come anyway. - public static func queueStats() async -> QueueStats? { - guard - let result = try? await rc(["vfs/stats"], timeout: 60), - result.succeeded - else { return nil } - return parseQueueStats(result.stdout) - } - - /// Pure version of parsing the `vfs/stats` response. `nil` means "I do not - /// know", never "all zeros". - /// - /// Split out so that it can be tested - until now the parsing sat in an async - /// function calling rclone and there was no way to reach it other than - /// running the whole buffer. - /// - /// Two things that were here and had to go: - /// - /// 1. `(json["diskCache"] as? [String: Any]) ?? json` - a response WITHOUT the - /// `diskCache` section (rclone built without the disk cache, a different - /// interface version, a truncated response) fell through to `json`, where - /// none of the counters are. - /// 2. `number(_:in:)` returning 0 for a missing key. - /// - /// Together they gave `QueueStats` with all zeros instead of `nil`, i.e. - /// `queueKnown == true` and again the "Everything uploaded to Google Drive" - /// screen. It is the same pattern that was fixed above by - /// `UploadState.queueUnknown`, just moved one step - into the parsing. - static func parseQueueStats(_ raw: String) -> QueueStats? { - guard - let data = raw.data(using: .utf8), - let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - // vfs/stats returns the counters nested in the "diskCache" section. Its - // absence is no answer to the question asked, not the answer "zero". - let disk = json["diskCache"] as? [String: Any] - else { return nil } - - func number(_ key: String) -> Int? { - if let v = disk[key] as? Int { return v } - if let v = disk[key] as? NSNumber { return v.intValue } - return nil - } - - guard - let inProgress = number("uploadsInProgress"), - let queued = number("uploadsQueued"), - let files = number("files"), - let errored = number("erroredFiles"), - let bytesUsed = number("bytesUsed") - else { return nil } - - return QueueStats( - uploadsInProgress: inProgress, - uploadsQueued: queued, - files: files, - erroredFiles: errored, - bytesUsed: UInt64(max(0, bytesUsed)), - // The only field whose absence may be made up with a default: it is a - // flag, not a counter - older rclone does not expose it, and its absence - // cannot be mistaken for "buffer full". - outOfSpace: (disk["outOfSpace"] as? Bool) ?? false - ) - } - - /// Google Drive account capacity, straight from rclone. - /// - /// NOBODY checked this until now. `machines.json` has the fields - /// `drive_total_gb` and `limit_gb`, but not a single line of code outside the - /// model itself uses them - it was a budget on paper. Meanwhile running out of - /// space on Drive is a FATAL error for rclone (`--drive-stop-on-upload-limit` + - /// `storageQuotaExceeded`): the mount disappears and Time Machine loses its - /// destination. With growth of around 600 MB per hourly cycle this is not a - /// distant problem, only a matter of the date. - public static func remoteQuota() async -> (used: UInt64, total: UInt64, free: UInt64)? { - guard - let result = try? await CMTooling.runRclone( - ["about", "--json", "\(remoteName):"], timeout: 120), - result.succeeded, - let data = result.stdout.data(using: .utf8), - let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] - else { return nil } - - func bytes(_ key: String) -> UInt64? { - if let v = json[key] as? NSNumber, v.int64Value >= 0 { return UInt64(v.int64Value) } - return nil - } - // `total` is sometimes absent (accounts without a limit) - then there is nothing to watch. - guard let total = bytes("total"), total > 0 else { return nil } - let used = bytes("used") ?? 0 - let free = bytes("free") ?? (total > used ? total - used : 0) - return (used, total, free) - } - - /// Waits until rclone stops uploading anything. `hdiutil` operations on a - /// FUSE-T mount are stable only with an empty queue - see - /// `BackupImageService.retryingFlakyMount`. - /// - /// The condition is `isIdle`, NOT `isQuiet`: bands abandoned by rclone stay in - /// `erroredFiles` until the process ends, so waiting for `isQuiet` would never - /// finish and every attach would pay the full time limit for nothing. - /// - /// Returns the queue reading from the moment it went quiet - `nil` when it did - /// not go quiet in time or rclone did not answer. The caller gets it so that it - /// can check `erroredFiles` without asking rclone the same question a second - /// time (costs up to 60 s - see `queueStats`). - @discardableResult - public static func statsWhenIdle(timeout: TimeInterval = 180) async -> QueueStats? { - let deadline = Date().addingTimeInterval(timeout) - while Date() < deadline { - if let stats = await queueStats(), stats.isIdle { return stats } - try? await Task.sleep(nanoseconds: 2_000_000_000) - } - return nil - } - - /// Like `statsWhenIdle`, when the caller only cares about "it got there". - @discardableResult - public static func waitUntilIdle(timeout: TimeInterval = 180) async -> Bool { - await statsWhenIdle(timeout: timeout) != nil - } - - /// Moves the upload deadline of all waiting items to "now". - /// - /// Needed on EVERY shutdown. An item enters the queue right after the write - - /// it is visible in `vfs/queue` immediately - but with a due date - /// `writeBackSeconds` ahead. Without moving it a detach would wait those whole - /// ten minutes, and `prepare-shutdown` before a Mac restart would become - /// unbearable. This is not cosmetic: a user who does not want to wait will - /// shut down the Mac without `prepare-shutdown`, and that has already once left - /// Time Machine without a destination for a whole night. - /// - /// Call AFTER the writes from the detach have had time to reach the queue - - /// items added later will not be touched. - /// - /// Returns `ExpiryOutcome` - how many items we FOUND and for how many the - /// deadline was moved - or `nil` when rclone did not answer the question about - /// the queue. - /// - /// `nil` and `0` are TWO DIFFERENT THINGS, and that is why the type is - /// optional. Until 23 September 2026 both situations - "the queue was empty" - /// and "we got no answer" - came out of here as `0`, so `detach` stayed silent - /// in the log in exactly the case where the deadlines were NOT moved and the - /// drain could take the whole `writeBackSeconds` (ten minutes) instead of a - /// moment. - /// - /// The number of moved items alone is not enough, because a THIRD case looks - /// like the first: when the queue has items but every `vfs/queue-set-expiry` - /// fails, "moved 0" was indistinguishable from "there was nothing to move". - /// That is why `queued` and `moved` are separate - see - /// `BackupImageService.expiryLogLine`. - /// - /// The limit for listing the queue itself raised from 30 s to 60 s: this same - /// file documents a measurement of **36.7 s** for the LIGHTER `vfs/stats` with - /// a clogged buffer (see `queueStats`), and `vfs/queue` lists hundreds of items - /// at such times. With 30 s the answer failed to arrive exactly when moving - /// the deadlines was needed most. - /// - /// The limit for a single `queue-set-expiry` stays at 30 s ON PURPOSE: that - /// one call decides the whole function, while this is one of hundreds and - /// losing it costs one item. With several hundred items a 60 s ceiling per - /// item would turn the detach into an operation without an upper time bound. - /// How many items to speed up were in the queue and for how many the deadline - /// was ACTUALLY moved. Two fields, not one, because "zero" means something - /// different depending on how many attempts there were - see - /// `expireQueuedUploads`. - public struct ExpiryOutcome: Sendable, Equatable { - /// Items found in the queue that could be sped up (without those already - /// uploading - see `parseQueueIDs`). - public var queued: Int - /// How many of them rclone confirmed. - public var moved: Int - - public init(queued: Int, moved: Int) { - self.queued = queued - self.moved = moved - } - } - - @discardableResult - public static func expireQueuedUploads() async -> ExpiryOutcome? { - guard - let result = try? await rc(["vfs/queue"], timeout: 60), - result.succeeded, - let ids = parseQueueIDs(result.stdout) - else { return nil } - - var moved = 0 - for id in ids { - // A large negative number instead of zero - that is how rclone itself describes it. - let response = try? await rc( - ["vfs/queue-set-expiry", "id=\(id)", "expiry=-1000000000"], timeout: 30) - if response?.succeeded == true { moved += 1 } - } - return ExpiryOutcome(queued: ids.count, moved: moved) - } - - /// Pure version: IDs of items from the `vfs/queue` response whose deadline can - /// be moved. `nil` = the response cannot be read, `[]` = the queue is empty. - /// Split out so that this distinction can be tested. - static func parseQueueIDs(_ raw: String) -> [Int]? { - guard - let data = raw.data(using: .utf8), - let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let queue = json["queue"] as? [[String: Any]] - else { return nil } - - return queue.compactMap { item in - // An item already uploading cannot be sped up - rclone ignores it, so we - // do not waste a call on it. - if (item["uploading"] as? Bool) == true { return nil } - return (item["id"] as? NSNumber)?.intValue - } - } - - /// DISK SPACE TAKEN by the buffer directory. `nil` = NOT MEASURED. - /// - /// Normally rclone itself reports the cache size (`QueueStats.bytesUsed`): - /// walking the directory is 6504 stat calls, and with a refresh every 10 - /// seconds needless load on the disk the backup is going to. - /// - /// NOTE: this is NOT a fallback for the measure the watchdog makes decisions - /// on, and must not be substituted there. Two reasons, both measured: - /// - /// 1. This function counts DISK SPACE TAKEN (`totalFileAllocatedSize`), a - /// quantity NOT COMPARABLE with the `--vfs-cache-max-size` limit - it can - /// exceed it. Hence "buffer 155 GB" with a 100 GB limit in the only PAUSE - /// line in the whole journal (23.09.2026 03:34). An hour later the monitor - /// wrote "rclone remote control is not answering": a missing answer was - /// replaced with a number from a DIFFERENT measure, and that number - /// triggered an irreversible pause. - /// 2. When the walk fails (permission denied, `~/.cloudmachine` gone), the - /// result `0` looks like an EMPTY buffer, i.e. like a met condition for - /// resuming a Time Machine that was paused because the buffer was full. - /// Exactly this pattern was closed by `BufferGuardService.freeGB()` for - /// `statfs` - hence `nil` here, not zero. - /// - /// So it is left for ONE thing: telling a person how much disk space the - /// cache takes. No decision reads it. - public static func cacheSizeBytesByWalk() -> UInt64? { - guard - let enumerator = FileManager.default.enumerator( - at: cacheDir, includingPropertiesForKeys: [.totalFileAllocatedSizeKey], - options: [.skipsHiddenFiles]) - else { return nil } - var total: UInt64 = 0 - for case let url as URL in enumerator { - let values = try? url.resourceValues(forKeys: [.totalFileAllocatedSizeKey]) - total += UInt64(values?.totalFileAllocatedSize ?? 0) - } - return total - } - - /// Whether rclone stopped on the Google Drive daily limit (750 GB/day). - /// - /// We recognise it by rclone's BEHAVIOUR, not by the error text. The reason is - /// concrete: `403 userRateLimitExceeded` is a momentary rate throttle that - /// rclone retries by itself ("will retry in 1m0s"), but it describes it with - /// the message "Received upload limit error" - indistinguishable by text alone - /// from the daily limit. The first version of this function caught exactly - /// that and paused the backup after 109 GiB uploaded, i.e. at a seventh of the - /// limit. - /// - /// The real limit is fatal for rclone (`--drive-stop-on-upload-limit` acts - /// exactly on `storageQuotaExceeded` and `teamDriveFileLimitExceeded`), so the - /// process exits and the mount disappears. - /// - /// NOTE: do NOT add a condition "only when the mount is down" here. Such a - /// version was here and it was dead: the `gdrive-buffer` agent has `KeepAlive` - /// with a `ThrottleInterval` of 30 s, so launchd brings rclone back faster than - /// the buffer watchdog manages to tick (every 30 s). The window in which the - /// mount is actually gone is shorter than the polling period - detecting the - /// limit was a coin toss, and in practice never happened at all. Recognising - /// it by a FRESH log entry works regardless of whether launchd has already - /// resurrected the mount. - /// - /// The momentary throttle (`userRateLimitExceeded`) is still NOT caught here - - /// see `logMentionsUploadLimit`. That is what once paused the backup after - /// 109 GiB, and that is what the caution was about, not the mount state. - /// - /// `nil` = THE LOG CANNOT BE READ, which is something else than "no trace of - /// the limit". Previously both cases came out of here as `false`, i.e. as the - /// answer "no problem" to a question that had no answer - and the watchdog - /// then does not pause the backup for lack of space on Drive. This is not - /// theoretical: the rclone log has `-rw-r-----` permissions, and at start-up - /// it is moved to `.1` (see `rotateLogIfLarge`). - public static func hitStorageQuotaState(logFile: URL? = nil) -> Bool? { - guard let text = recentLog(bytes: 256 * 1024, from: logFile ?? Self.logFile) else { - return nil - } - return logMentionsUploadLimit(text, now: Date(), within: 30) - } - - /// Version FOR SHOWING TO A PERSON, where there is no place to put a third - /// state (`BufferStatus.driveFull`, `UploadState.from`). - /// - /// NO DECISION may use it: `?? false` is exactly the substitution described in - /// the comment above. The buffer watchdog reads `hitStorageQuotaState()` and - /// decides itself what to do with "I do not know". A third state in the - /// interface requires changing `BufferStatus` and `CloudMachineController` - - /// that is a separate change, outside this branch. - public static func hitStorageQuota() -> Bool { hitStorageQuotaState() ?? false } - - /// Whether the upload is actually STALLED - recognised by behaviour, not by - /// text. - /// - /// Google's daily upload limit (750 GB) reports itself as `403 - /// userRateLimitExceeded`, i.e. with EXACTLY the same code as an ordinary - /// momentary rate throttle. They cannot be told apart by text and one should - /// not try - the first version of `logMentionsUploadLimit` tried and paused the - /// backup after 109 GiB of 750 GB. - /// - /// What does tell them apart is the RATIO of successes to errors in a time - /// window. Measured on our own log: - /// - throttling: 11 Sep 14h -> 4833, 12 Sep 09h -> 1.07, 15 Sep 08h -> 2.39 - /// - real jam: 12 Sep 10-12h -> 0.002-0.011, 15 Sep 09h -> 0.003 - /// TWO ORDERS OF MAGNITUDE lie between one and the other, so the 0.1 threshold - /// has margin both ways. - /// - /// `minErrors` protects against silence: a window without traffic has zero - /// errors and zero successes, and that is not a jam. - /// - /// `nil` = THE LOG CANNOT BE READ. The distinction is more dangerous here than - /// for the measurement itself: `false` went on to `reportUploadStall`, which - /// REMOVED the jam marker and wrote "Upload to Google Drive has resumed" - a - /// claim about an event nobody checked, based on a file nobody read. - public static func uploadStalledState(logFile: URL? = nil) -> Bool? { - // A bigger window than for the text test: during a jam the log grows by - // about 90 KB per minute, so 256 KB would show only the last three minutes - // and the ratio would be computed from a sample without a single success. - guard let text = recentLog(bytes: 4 * 1024 * 1024, from: logFile ?? Self.logFile) else { - return nil - } - return logShowsUploadStalled(text, now: Date(), within: 30) - } - - /// Version for showing to a person - see `hitStorageQuota()`, same reason and - /// same warning: no decision reads this version. - public static func uploadStalled() -> Bool { uploadStalledState() ?? false } - - /// Combined answer "the upload to Drive is not going" - for showing to the - /// user. The buffer watchdog does NOT use this function, because for it the - /// difference between the two is fundamental: lack of space will not pass by - /// itself, while the daily limit passes within a few hours. - public static func hitDailyQuota() -> Bool { - hitStorageQuota() || uploadStalled() - } - - /// Tail of the rclone log as text. `nil` = the file CANNOT BE READ: it does not - /// exist, there is no permission for it, or the read failed. - /// - /// The caller MUST pass that `nil` on as "I do not know". No limit entries and - /// no access to the log are two different things, and only the first means "no - /// problem". The file is taken from a parameter so that both questions asked of - /// this log can be tested on our own file - without `~/.cloudmachine` and - /// without guessing permissions. - /// - /// When the live file holds less than `bytes` - it was just trimmed or - /// rotated - the rest is taken from the end of `.1`, which holds what came - /// before it. Without that, a trim in the middle of a jam left the detection - /// a few lines to count and it answered "no jam" for the next half hour. - /// `.1` is only an addition: when it cannot be read, the live file alone is - /// still an answer. - private static func recentLog(bytes: UInt64, from file: URL) -> String? { - guard let current = tail(of: file, bytes: bytes) else { return nil } - var data = current - if UInt64(current.count) < bytes, - let older = tail(of: previousLog(of: file), bytes: bytes - UInt64(current.count)) - { - // A line cut in half at the boundary has no timestamp and is skipped. - data = older + Data("\n".utf8) + current - } - // The tail almost always starts in the middle of a multi-byte character, so - // we decode lossily - otherwise the whole read would be lost to one byte. - return String(decoding: data, as: UTF8.self) - } - - private static func tail(of file: URL, bytes: UInt64) -> Data? { - guard let handle = try? FileHandle(forReadingFrom: file) else { return nil } - defer { try? handle.close() } - let size = (try? handle.seekToEnd()) ?? 0 - try? handle.seek(toOffset: size > bytes ? size - bytes : 0) - return try? handle.readToEnd() ?? Data() - } - - /// The locale we read rclone log timestamps with. - /// - /// `en_US_POSIX`, NOT `Locale.current`. A `DateFormatter` with a fixed - /// `dateFormat` and the default locale takes the calendar from that locale: on - /// a machine with the Buddhist calendar (`th_TH`) "2026" means the Buddhist - /// year, i.e. Gregorian 1483, and with the Persian or Hijri calendar yet - /// another date comes out. The timestamp then parses WITHOUT AN ERROR and lands - /// 543 years too early, so `stamp < cutoff` ends the loop on the first line, - /// `errors` stays zero and `uploadStalled()` reports "no jam" exactly when the - /// jam is going on - and on that basis the buffer watchdog does not pause Time - /// Machine. - /// - /// The same class of bug as `LC_ALL=C` forced in `CMLock` (see the description - /// of the real incident there): machine text is read with the machine's - /// settings, not the person's. - public static let rcloneLogLocale = Locale(identifier: "en_US_POSIX") - - /// Pure version of jam detection - counts successes and errors in the window. - /// - /// A success is a `... : Copied (...)` line, an error `Received upload limit - /// error`. Both come from the same log and the same event, so the ratio needs - /// no calibration between machines. - /// - /// `locale` exists only so that a test can inject a KNOWN BAD calendar - see - /// `rcloneLogLocale`. Production code does not pass it. - public static func logShowsUploadStalled( - _ text: String, now: Date, within minutes: Int, - minErrors: Int = 300, maxSuccessRatio: Double = 0.1, - locale: Locale = DriveBufferService.rcloneLogLocale - ) -> Bool { - let formatter = DateFormatter() - formatter.locale = locale - formatter.dateFormat = "yyyy/MM/dd HH:mm:ss" - formatter.timeZone = TimeZone.current - let cutoff = now.addingTimeInterval(-Double(minutes) * 60) - - var errors = 0 - var successes = 0 - for line in text.components(separatedBy: .newlines).reversed() { - guard line.count > 19, let stamp = formatter.date(from: String(line.prefix(19))) else { - continue - } - // The log is chronological, so the first line older than the window ends - // the counting - everything further back is older still. - if stamp < cutoff { break } - let lower = line.lowercased() - if lower.contains("received upload limit error") { - errors += 1 - } else if lower.contains(": copied (") { - successes += 1 - } - } - - guard errors >= minErrors else { return false } - return Double(successes) < maxSuccessRatio * Double(errors) - } - - /// Looks for a trace of the limit only in fresh entries. Without a time bound - /// an alarm once raised would never go out, because the entry stays in the log - /// forever - the backup would fall into a pause-resume-pause cycle. - /// - /// A pure version, so that it can be tested without a file and without a clock. - /// - /// `locale` as in `logShowsUploadStalled` - for tests only. - public static func logMentionsUploadLimit( - _ text: String, now: Date, within minutes: Int, - locale: Locale = DriveBufferService.rcloneLogLocale - ) -> Bool { - let formatter = DateFormatter() - formatter.locale = locale - formatter.dateFormat = "yyyy/MM/dd HH:mm:ss" - formatter.timeZone = TimeZone.current - let cutoff = now.addingTimeInterval(-Double(minutes) * 60) - - for line in text.components(separatedBy: .newlines).reversed() { - guard line.count > 19, let stamp = formatter.date(from: String(line.prefix(19))) else { - continue - } - if stamp < cutoff { return false } - let lower = line.lowercased() - // userRateLimitExceeded deliberately SKIPPED - it is an ordinary throttle - // that rclone retries by itself. Catching it paused the backup at 109 GiB of - // 750 GB. - if lower.contains("storagequotaexceeded") || lower.contains("teamdrivefilelimitexceeded") { - return true - } - } - return false - } -} diff --git a/mac-app/Sources/CloudMachineCore/DriveFolder.swift b/mac-app/Sources/CloudMachineCore/DriveFolder.swift index 7f4b2c7..c271899 100644 --- a/mac-app/Sources/CloudMachineCore/DriveFolder.swift +++ b/mac-app/Sources/CloudMachineCore/DriveFolder.swift @@ -1,24 +1,29 @@ import Foundation -/// The folder on Google Drive that holds THIS Mac's backup image: -/// `gdrive:CloudMachine/`, with the image `.sparsebundle` inside. +/// The name of THIS Mac on Google Drive. The machine keeps the backup's bands +/// in `gdrive:CloudMachine/-tm` (`VMLayout.remote`). /// /// It used to be the constant `mac-studio` on every Mac, so two Macs on one -/// Google account would have shared one folder and one image. Each Mac now +/// Google account would have shared one folder and one backup. Each Mac now /// keeps its own name in `~/Library/Application Support/CloudMachine/drive-folder`, /// chosen once by `configure-remote` and never changed afterwards. /// /// "Never changed" is the point. A different name is a different, empty /// folder: Time Machine starts from zero and the old backup sits orphaned on /// Drive, still using quota. So the name is decided once, and an installation -/// that predates this file keeps `mac-studio` - it is where its backup lives. +/// that predates this file keeps `mac-studio`. public enum DriveFolder { + /// The rclone remote every command uses. + public static let remoteName = "gdrive" + /// Top-level folder on Drive; each Mac has its own folder in it. + public static let remoteRoot = "CloudMachine" + /// The name every installation used before names were per Mac. public static let legacyName = "mac-studio" static var file: URL { CMPaths.appSupportDir.appendingPathComponent("drive-folder") } - /// The name the mount, the image and the status use. + /// The name the machine's Drive folder and the status use. /// /// No file means an installation from before this setting existed (a new /// one gets the file from `configure-remote` before anything is mounted), @@ -43,32 +48,11 @@ public enum DriveFolder { /// /// The evidence is the rclone remote itself. `configure-remote` decides the /// folder BEFORE it creates the remote, so on a new Mac the remote is not - /// there yet; on an installation that predates `drive-folder` it is, and its - /// backup lives under the legacy name. Weaker traces were rejected: the - /// buffer directory appears after any mount attempt, and the old - /// `mount-desired.state` is no longer written by anything. The cached - /// legacy folder is not such a weak trace - rclone writes it only once the - /// remote was opened, i.e. after a mount that worked. + /// there yet; on an installation that predates `drive-folder` it is. /// /// `nil` = rclone did not answer; `decide` then refuses rather than guess. - /// - /// A cached `mac-studio` folder is evidence on its own (see - /// `InstallationTraces`): only an installation from before `drive-folder` - /// ever mounted that folder, because a new one gets its file before its - /// first mount. Until 11.10.2026 the remote was the only witness, and a Mac - /// whose `rclone.conf` was lost or replaced got a new, empty folder. static func hasLegacyInstallation() async -> Bool? { - legacyEvidence( - remoteConfigured: await RemoteConfigurer.isConfigured( - remoteName: DriveBufferService.remoteName), - cachedFolders: InstallationTraces.cachedFolders()) - } - - /// Pure: the cache trace of the legacy folder outweighs the remote's answer, - /// including "I do not know". - static func legacyEvidence(remoteConfigured: Bool?, cachedFolders: [String]) -> Bool? { - if cachedFolders.contains(legacyName) { return true } - return remoteConfigured + await RemoteConfigurer.isConfigured(remoteName: remoteName) } public enum Decision: Equatable { diff --git a/mac-app/Sources/CloudMachineCore/EdgeTriggeredLog.swift b/mac-app/Sources/CloudMachineCore/EdgeTriggeredLog.swift deleted file mode 100644 index c23c3fc..0000000 --- a/mac-app/Sources/CloudMachineCore/EdgeTriggeredLog.swift +++ /dev/null @@ -1,23 +0,0 @@ -import Foundation - -/// Logs `message` only the FIRST time a given state is met (`active == -/// true`) - later watchdog cycles in the same state stay quiet, so that a longer -/// failure does not flood the shared log with hundreds of identical lines. -/// When `active` goes back to `false`, the marker is cleared and the next -/// occurrence logs again. -/// -/// Born from a real case: `BackupWatchdogService`/`VerifyWatchdogService` were -/// silent for >2 days, because their "mount not ready yet" guards simply -/// `return`ed without a trace in the log - the diagnosis took much longer than -/// it would have if that state boundary had been visible. -public enum EdgeTriggeredLog { - public static func log(marker: URL, active: Bool, _ message: @autoclosure () -> String) { - if active { - guard !FileManager.default.fileExists(atPath: marker.path) else { return } - CMLogger.log(message()) - FileManager.default.createFile(atPath: marker.path, contents: nil) - } else { - try? FileManager.default.removeItem(at: marker) - } - } -} diff --git a/mac-app/Sources/CloudMachineCore/FuseInstaller.swift b/mac-app/Sources/CloudMachineCore/FuseInstaller.swift deleted file mode 100644 index 3640be2..0000000 --- a/mac-app/Sources/CloudMachineCore/FuseInstaller.swift +++ /dev/null @@ -1,189 +0,0 @@ -import Foundation - -/// Pulls FUSE-T into CloudMachine, so that there is no separate application in -/// the system. -/// -/// The official FUSE-T installer puts down `/Applications/fuse-t.app`, -/// libraries in `/usr/local/lib` and a server in -/// `/Library/Application Support/fuse-t`. That application hosts the FSKit -/// extension (`FskitSrvModule.appex`) - a backend we do not use anyway, -/// because we mount over NFS. So an icon and a package that do nothing are -/// left in the system. -/// -/// Exactly two files are needed, both depending only on system libraries: -/// - `libfuse-t.dylib` - rclone opens it via dlopen, -/// - `go-nfsv4` - the NFS server that actually holds the mount. -/// -/// We point at the server via the `FUSE_NFSSRV_PATH` variable, so it can live -/// anywhere. The library cannot: rclone has absolute paths baked in -/// (`/usr/local/lib/libfuse-t.dylib`, `/usr/local/lib/libfuse.2.dylib`), and -/// `dlopen` on an absolute path ignores `DYLD_*`. That is why we leave a -/// SYMLINK to our copy there - `/usr/local/lib` belongs to a user in the admin -/// group, so root is not needed for that. -/// -/// LICENSE: FUSE-T is not open-source software. The binary distribution is free -/// for non-commercial use provided the copyright notice is kept - that is why -/// we copy `LICENSE.rtf` next to the binaries. Bundling with commercial -/// software requires a separate license from the FUSE-T authors. -public enum FuseInstaller { - - private static let releasesAPI = - "https://api.github.com/repos/macos-fuse-t/fuse-t/releases/latest" - - /// Path where rclone looks for the library. It cannot be changed. - static let systemLibLink = "/usr/local/lib/libfuse-t.dylib" - - public static var isInstalled: Bool { - FileManager.default.isExecutableFile(atPath: CMTooling.bundledNfsServer.path) - && FileManager.default.fileExists(atPath: systemLibLink) - } - - /// Recreates the symlink in `/usr/local/lib` if it has disappeared. - /// - /// Needed because the FUSE-T uninstaller deletes everything under that path - /// - including our symlink. Without this, removing the separate fuse-t - /// application would take the mount down with it, even though our copy of - /// the library lies untouched in its place. - @discardableResult - public static func ensureSystemLink() -> Bool { - guard FileManager.default.fileExists(atPath: CMTooling.bundledFuseLib.path) else { - return false - } - if let target = try? FileManager.default.destinationOfSymbolicLink(atPath: systemLibLink), - target == CMTooling.bundledFuseLib.path - { - return true - } - do { - try linkSystemLibrary() - CMLogger.log("Recreated the symlink \(systemLibLink) to the copy in CloudMachine") - return true - } catch { - return false - } - } - - public static func install() async -> CMActionResult { - let workDir = URL(fileURLWithPath: NSTemporaryDirectory()) - .appendingPathComponent("cloudmachine-fuse-\(UUID().uuidString)") - defer { try? FileManager.default.removeItem(at: workDir) } - try? FileManager.default.createDirectory(at: workDir, withIntermediateDirectories: true) - - guard let (version, pkgURL) = await latestPackage() else { - return CMActionResult( - succeeded: false, message: L10n.tr("Could not determine the FUSE-T version.")) - } - - let pkgPath = workDir.appendingPathComponent("fuse-t.pkg") - guard - let download = try? await ProcessRunner.run( - "/usr/bin/curl", ["-fsSL", "-o", pkgPath.path, pkgURL], timeout: 600), - download.succeeded - else { - return CMActionResult( - succeeded: false, message: L10n.tr("Could not download the FUSE-T package.")) - } - - let expanded = workDir.appendingPathComponent("expanded") - guard - let expand = try? await ProcessRunner.run( - "/usr/sbin/pkgutil", ["--expand-full", pkgPath.path, expanded.path], timeout: 300), - expand.succeeded - else { - return CMActionResult( - succeeded: false, message: L10n.tr("Could not unpack the FUSE-T package.")) - } - - guard - let dylib = findFile( - under: expanded, matching: { $0.hasPrefix("libfuse-t") && $0.hasSuffix(".dylib") }), - let server = findFile(under: expanded, matching: { $0.hasPrefix("go-nfsv4") }) - else { - return CMActionResult( - succeeded: false, - message: L10n.tr("The FUSE-T package does not contain the expected files.")) - } - - let destination = CMTooling.bundledFuseDir - try? FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - - do { - try copy(dylib, to: CMTooling.bundledFuseLib) - try copy(server, to: CMTooling.bundledNfsServer) - // The license requires keeping the copyright notice on redistribution. - if let license = findFile(under: expanded, matching: { $0 == "LICENSE.rtf" }) { - try? copy(license, to: destination.appendingPathComponent("LICENSE.rtf")) - } - try linkSystemLibrary() - } catch { - return CMActionResult( - succeeded: false, - message: L10n.tr("Installing FUSE-T failed: %@", error.localizedDescription)) - } - - return CMActionResult( - succeeded: true, - message: L10n.tr( - "Installed FUSE-T %@ inside CloudMachine (%@).\nThe separate fuse-t application is no longer needed - you can remove it:\n sudo \"/Library/Application Support/fuse-t/uninstall.sh\"", - version, destination.path)) - } - - // MARK: - Details - - /// Replaces `/usr/local/lib/libfuse-t.dylib` with a symlink to our copy. - /// - /// Does not need root: `/usr/local/lib` belongs to the user and the admin - /// group. If a real file from the official installer lies there, we remove - /// it - our copy is bit-for-bit identical, because it comes from the same - /// package. - private static func linkSystemLibrary() throws { - let fm = FileManager.default - let linkURL = URL(fileURLWithPath: systemLibLink) - try? fm.createDirectory( - at: linkURL.deletingLastPathComponent(), withIntermediateDirectories: true) - if fm.fileExists(atPath: systemLibLink) || isSymlink(systemLibLink) { - try fm.removeItem(at: linkURL) - } - try fm.createSymbolicLink(at: linkURL, withDestinationURL: CMTooling.bundledFuseLib) - } - - private static func isSymlink(_ path: String) -> Bool { - (try? FileManager.default.attributesOfItem(atPath: path)[.type] as? FileAttributeType) - == FileAttributeType.typeSymbolicLink - } - - private static func copy(_ source: URL, to destination: URL) throws { - try? FileManager.default.removeItem(at: destination) - try FileManager.default.copyItem(at: source, to: destination) - try FileManager.default.setAttributes( - [.posixPermissions: 0o755], ofItemAtPath: destination.path) - } - - private static func findFile(under root: URL, matching: (String) -> Bool) -> URL? { - guard let enumerator = FileManager.default.enumerator(atPath: root.path) else { return nil } - for case let relative as String in enumerator { - let name = (relative as NSString).lastPathComponent - if matching(name) { return root.appendingPathComponent(relative) } - } - return nil - } - - /// Version and URL of the package from the latest release on GitHub. - static func latestPackage() async -> (version: String, url: String)? { - guard - let result = try? await ProcessRunner.run( - "/usr/bin/curl", ["-fsSL", releasesAPI], timeout: 60), - result.succeeded, - let data = result.stdout.data(using: .utf8), - let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let tag = json["tag_name"] as? String, - let assets = json["assets"] as? [[String: Any]] - else { return nil } - - let pkg = assets.first { - ($0["name"] as? String)?.hasSuffix(".pkg") == true - } - guard let url = pkg?["browser_download_url"] as? String else { return nil } - return (tag, url) - } -} diff --git a/mac-app/Sources/CloudMachineCore/HealthAlert.swift b/mac-app/Sources/CloudMachineCore/HealthAlert.swift index 1b247c7..baded79 100644 --- a/mac-app/Sources/CloudMachineCore/HealthAlert.swift +++ b/mac-app/Sources/CloudMachineCore/HealthAlert.swift @@ -21,7 +21,7 @@ public enum HealthAlert { } struct AlertState: Codable { - /// The text last SHOWN - kept for a person (`drive-status`, diagnosis from + /// The text last SHOWN - kept for a person (`vm-status`, diagnosis from /// the file), not for comparison. It is in the UI language of the run that /// wrote it. var lastSummary: String @@ -63,7 +63,7 @@ public enum HealthAlert { /// adding made-up failures to the production log. /// /// `log` is injectable for exactly the same reason as - /// `BufferGuardService.Probes.log`. Until it was, every `swift test` run + /// `VMGuard.Probes.log`. Until it was, every `swift test` run /// appended its invented "BACKUP FAILURE" (then still in Polish) to the /// real `cloudmachine.log` - measured 25.09.2026: 117 lines containing a /// word that existed only in `HealthAlertTests.raport(_:)` (now @@ -190,7 +190,7 @@ public enum HealthAlert { "osascript did not show the notification (permissions or no graphical session)" /// The last report that could NOT be delivered - to be shown in - /// `drive-status`. `nil` when the last report was delivered or when there + /// `vm-status`. `nil` when the last report was delivered or when there /// was none. A silent alarm has to be visible somewhere a person looks on /// their own, because by definition it will not reach them via a /// notification. diff --git a/mac-app/Sources/CloudMachineCore/ImageProbe.swift b/mac-app/Sources/CloudMachineCore/ImageProbe.swift deleted file mode 100644 index 26d6375..0000000 --- a/mac-app/Sources/CloudMachineCore/ImageProbe.swift +++ /dev/null @@ -1,322 +0,0 @@ -import Foundation - -/// Whether the attached image REALLY returns data - and does not merely appear -/// in the mount table. -/// -/// WHY THIS EXISTS -/// -/// On 22 Sep 2026 at 00:17 rclone got HTTP 401 from Google while listing the -/// bands, passed an I/O error to FUSE-T, and the image driver considered the -/// device disconnected. From that moment every read of a file under -/// `/Volumes/CloudMachine` ended with `errno 6 ENXIO` ("Device not -/// configured") and Time Machine failed every run with -/// `BACKUP_FAILED_DISCONNECTED_DESTINATION`. -/// -/// Meanwhile `mount` still showed the volume, `hdiutil info` still showed the -/// image, `ls` of the directory worked (from the kernel cache), and `statfs` -/// returned free space. Everything `isAttached` was built on said "OK". The -/// result: `drive-status` said "Image attached: OK", the GUI "Everything -/// uploaded", and the `gdrive-attach` agent reported "Already attached" every -/// 15 minutes and did NOT reattach. For 15 hours the only thing that cried out -/// was `backup-health` - after the fact, from the age of the last backup. -/// -/// Measured on the dead device: `open()` of the directory - OK, `listdir` - -/// OK, `statvfs` - OK, `fsync` - OK, **`open`+`read` of 1 byte of a regular -/// file - ENXIO**. That is why the probe reads a byte. Nothing weaker tells a -/// live device from a dead one. -public enum ImageProbe { - - public enum Verdict: Equatable, Sendable { - /// The read succeeded. - case readable - /// The device does not return data. `errno` from the failed read. - case dead(errno: Int32) - /// There is no regular file in the root directory that could be read - - /// that is what a fresh volume looks like before the first backup. A - /// failure cannot be established, so we do NOT report one. - case nothingToProbe - /// The probe DID NOT ANSWER within the allotted time. - /// - /// This is NOT `.dead`, and merging the two cases would be dangerous: - /// `.dead` triggers a FORCED detach in `attach-image` of an image on which - /// unsent data is still waiting, while here we do not even know whether - /// the device is dead. A lack of knowledge must HOLD BACK an irreversible - /// operation, not trigger it - that is why this verdict maps to - /// `BackupImageService.Attachment.unknown`, which already blocks `attach` - /// and `create`. - case timedOut - } - - /// Errors that mean "the device is gone", not "the file is odd". - /// - /// EACCES or EISDIR are a property of the file, not of the volume - we skip - /// such a file and try the next one. ENXIO/EIO/ENODEV/ENOTCONN are the - /// volume. - static let deviceErrors: Set = [ENXIO, EIO, ENODEV, ENOTCONN] - - /// Pure version: listing and reading are injected, so that a test can - /// substitute ENXIO without breaking a real device. - /// - /// - `regularFiles`: regular files in the volume's root directory. - /// - `readFirstByte`: returns the `errno` when the read fails. - public static func probe( - regularFiles: () throws -> [URL], - readFirstByte: (URL) -> Int32? - ) -> Verdict { - let files: [URL] - do { - files = try regularFiles() - } catch { - // Listing can also fail on a dead device - and that is the case `try?` - // used to swallow. `--dir-cache-time` is 5 minutes: while the cache is - // fresh, `contentsOfDirectory` runs from kernel memory and works even - // after ENXIO (hence the statement above that listing "is not a test"). - // Once the cache expires, the same listing goes to rclone for data and - // fails with the same ENXIO as the read. Until 23 September 2026 the - // probe then said `.nothingToProbe`, `BackupImageService.attachment` - // mapped that to `.attached`, and the `gdrive-attach` agent reported - // "Already attached" every 15 minutes - i.e. exactly the failure this - // probe was created for came back through the back door after the - // fifth minute. - if let code = deviceErrno(of: error), deviceErrors.contains(code) { - return .dead(errno: code) - } - // An error without a recognized device errno proves nothing about the volume. - return .nothingToProbe - } - guard !files.isEmpty else { return .nothingToProbe } - for file in files { - guard let errno = readFirstByte(file) else { return .readable } - if deviceErrors.contains(errno) { return .dead(errno: errno) } - // An error specific to the file - try another one. - } - return .nothingToProbe - } - - /// Extracts the raw `errno` from an error thrown by the directory listing. - /// - /// Foundation does not hand it over directly: `contentsOfDirectory` wraps - /// the POSIX error in `NSCocoaErrorDomain` (e.g. 256 - /// `NSFileReadUnknownError`), and hides the original `errno` under - /// `NSUnderlyingErrorKey` as `NSPOSIXErrorDomain`. We check three forms, - /// because each comes from a different layer: `POSIXError` from code calling - /// libc directly, `NSPOSIXErrorDomain` from a thin wrapper, and only then the - /// nested one. - static func deviceErrno(of error: Error) -> Int32? { - if let posix = error as? POSIXError { return posix.code.rawValue } - let ns = error as NSError - if ns.domain == NSPOSIXErrorDomain { return Int32(ns.code) } - if let underlying = ns.userInfo[NSUnderlyingErrorKey] as? NSError, - underlying.domain == NSPOSIXErrorDomain - { - return Int32(underlying.code) - } - return nil - } - - // MARK: - Time limit - // - // WHY A THREAD SEPARATED BY A DEADLINE, AND NOT `ProcessRunner.run(timeout:)` - // - // The probe is a `readdir` plus `open`/`read` on a volume living on FUSE-T. - // When rclone stops responding, these calls enter an UNINTERRUPTIBLE wait in - // the kernel (state "U" in `ps`). Such a thread can be neither cancelled - // nor killed: `Task.cancel()` is cooperative and `read()` in the kernel does - // not cooperate with it, and SIGKILL does not work either - the same - // limitation `ProcessRunner` already describes for its "final limit". Since - // the probe cannot be INTERRUPTED, the only thing that can be guaranteed is - // that its hang does not hang the CALLER. So the probe gets its own thread, - // and the caller gets a deadline and the `.timedOut` verdict. - // - // That is also why there is not (and cannot be) a synchronous - // `probe(volume:)` here - there was one until 26.09.2026 and it was exactly - // what froze the panel on `@MainActor` and silenced the `backup-health` - // watchdog permanently. The only entry point onto a live volume is `async`, - // so that the caller waits without blocking a thread. - // - // Considered and REJECTED: - // - // - A probe in a SUBPROCESS via `ProcessRunner.run(..., timeout:)` - the - // pattern that saves `tmutil` in this repo. Here it buys exactly as much - // as a thread (a hang does not stop the caller), and costs much more: a - // new agent subcommand, finding the binary in three layouts (GUI bundle, - // `.build/` when working from the terminal, `/Applications` under - // launchd), a `fork`+`exec` every 10 s in the panel's refresh loop and - - // just like here - an orphaned process stuck in the kernel that nobody - // will kill. Three new places where the probe can silently stop working, - // for a gain limited to the stuck thread belonging to another process. - // - // - `open(..., O_NONBLOCK)`. On a REGULAR FILE O_NONBLOCK does not make - // `read()` non-blocking - it applies to FIFOs, sockets and character - // devices, not to waiting for file I/O; `readdir` does not even have such - // a variant. The probe would thus lose code clarity without gaining a - // guarantee, and would incidentally stop measuring what it exists for: the - // device HANDING OVER a byte. - // - // - A race between two `Task`s with `Task.sleep` and `cancel()` on the loser - // - see above, cancellation has no way to reach `read()` in the kernel. A - // thread from the `DispatchQueue.global()` pool is out for the same - // reason, only worse: a stuck thread stays occupied forever, and the pool - // has ~64 slots and is shared with the whole rest of the process. - - /// How long we wait for a verdict before declaring `.timedOut`. - /// - /// On a live volume the probe takes microseconds - one `readdir` and a read - /// of one byte. So these 15 s are not a budget for work but a limit of - /// patience. The lower bound is set by a LIVE but slow FUSE-T (a band being - /// downloaded from the Drive during the read), which must not be taken for - /// an unknown; the upper one - by what this limit exists for: on 25.09.2026 - /// `drive-status` hung for over 25 s and had to be killed by hand, and the - /// `backup-health` watchdog runs every 1800 s, so the full 15 s will not come - /// anywhere near its window anyway. - public static let probeTimeout: TimeInterval = 15 - - /// Verdict passed from the probing thread to the caller. - /// - /// Both sides must survive the absence of the other: the caller may give up - /// on the deadline and never collect the verdict, and the thread may never - /// reach `finish`, because it got stuck in the kernel. - private final class ProbeBox: @unchecked Sendable { - private let lock = NSLock() - private var verdict: Verdict? - private var handler: ((Verdict) -> Void)? - - func finish(_ value: Verdict) { - lock.lock() - guard verdict == nil else { - lock.unlock() - return - } - verdict = value - let waiting = handler - handler = nil - lock.unlock() - waiting?(value) - } - - /// Calls `handler` with the verdict - immediately if the probe managed to - /// answer before the caller subscribed to the notification. - func whenDone(_ handler: @escaping (Verdict) -> Void) { - lock.lock() - if let verdict { - lock.unlock() - handler(verdict) - return - } - self.handler = handler - lock.unlock() - } - } - - /// Which volumes currently have a probe in flight. - private final class ProbeSlots: @unchecked Sendable { - static let shared = ProbeSlots() - private let lock = NSLock() - private var busy: Set = [] - - /// `true` = the slot was free and from now on belongs to the caller. - func claim(_ slot: String) -> Bool { - lock.lock() - defer { lock.unlock() } - return busy.insert(slot).inserted - } - - func release(_ slot: String) { - lock.lock() - busy.remove(slot) - lock.unlock() - } - } - - /// Starts the probe on its OWN thread. `nil` = the probe for this slot is - /// still running. - /// - /// One probe per slot is not an optimization. Without it, the GUI panel, - /// which refreshes every 10 s, would leave one hanging thread per run on a - /// permanently stuck volume - several hundred per hour, each with its own - /// stack and none recoverable. A second thread would not learn anything new - /// anyway: since the first one is stuck in the kernel, there is no answer, - /// so the next caller gets `.timedOut` right away. - private static func startProbe( - slot: String, - regularFiles: @escaping @Sendable () throws -> [URL], - readFirstByte: @escaping @Sendable (URL) -> Int32? - ) -> ProbeBox? { - guard ProbeSlots.shared.claim(slot) else { return nil } - let box = ProbeBox() - let thread = Thread { - let verdict = probe(regularFiles: regularFiles, readFirstByte: readFirstByte) - // Release the slot BEFORE handing over the verdict: otherwise a caller - // woken by `finish` would see the slot as still occupied. - ProbeSlots.shared.release(slot) - box.finish(verdict) - } - thread.name = "com.renacode.cloudmachine.image-probe" - thread.stackSize = 512 * 1024 - thread.start() - return box - } - - /// Probe on a live volume. After `timeout` it returns `.timedOut` and the - /// caller moves on - the read itself may stay in the kernel forever and that - /// is accepted, as long as it does not take the watchdog or the interface - /// down with it. - public static func probe(volume: URL, timeout: TimeInterval = probeTimeout) async -> Verdict { - await probe( - slot: volume.path, timeout: timeout, - regularFiles: { try regularFiles(in: volume) }, - readFirstByte: { readFirstByteErrno(of: $0) }) - } - - /// As above, but with injected listing and reading - so that a test can - /// substitute a probe that NEVER ANSWERS, without a dead volume at hand. - /// `slot` is a separate parameter for the same reason: two tests must not - /// occupy each other's slot. - static func probe( - slot: String, - timeout: TimeInterval, - regularFiles: @escaping @Sendable () throws -> [URL], - readFirstByte: @escaping @Sendable (URL) -> Int32? - ) async -> Verdict { - guard - let box = startProbe(slot: slot, regularFiles: regularFiles, readFirstByte: readFirstByte) - else { return .timedOut } - return await withCheckedContinuation { (continuation: CheckedContinuation) in - let once = ContinuationGuard() - box.whenDone { verdict in - if once.claim() { continuation.resume(returning: verdict) } - } - DispatchQueue.global().asyncAfter(deadline: .now() + timeout) { - if once.claim() { continuation.resume(returning: .timedOut) } - } - } - } - - /// Regular files in the root directory. While the directory cache is fresh - /// (`--dir-cache-time 5m`), listing runs from memory and works on a dead - /// device too - that is why a successful listing alone is NOT proof of - /// life, only a list of candidates for the test. Once the cache expires, - /// the same listing fails with ENXIO and then it is proof of death - that - /// is handled by `probe`, not by this function. - static func regularFiles(in volume: URL) throws -> [URL] { - try FileManager.default.contentsOfDirectory( - at: volume, includingPropertiesForKeys: [.isRegularFileKey], - options: [.skipsSubdirectoryDescendants] - ) - .filter { (try? $0.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile) == true } - .sorted { $0.lastPathComponent < $1.lastPathComponent } - } - - /// `nil` = the read succeeded (an empty file too), otherwise `errno`. - /// - /// Via libc `open`/`read`, not via `Data(contentsOf:)`: Foundation reads the - /// whole file, and we are interested in one byte and the raw errno. - static func readFirstByteErrno(of file: URL) -> Int32? { - let fd = open(file.path, O_RDONLY) - guard fd >= 0 else { return errno } - defer { close(fd) } - var byte: UInt8 = 0 - let n = read(fd, &byte, 1) - return n < 0 ? errno : nil - } -} diff --git a/mac-app/Sources/CloudMachineCore/InstallationTraces.swift b/mac-app/Sources/CloudMachineCore/InstallationTraces.swift deleted file mode 100644 index 6894712..0000000 --- a/mac-app/Sources/CloudMachineCore/InstallationTraces.swift +++ /dev/null @@ -1,103 +0,0 @@ -import Foundation - -/// What this Mac remembers of a backup it has already made - read from the -/// local disk, without rclone and without Google. -/// -/// The decisions that can orphan a backup (which Drive folder to use, whether -/// to create an image) used to rest on one witness: what rclone answered with -/// the current configuration. That witness can be wrong without failing. A -/// missing `rclone.conf` makes `rclone listremotes` succeed with an empty list -/// (checked: exit 0), and a token with the `drive.file` scope makes a folder -/// created under another `client_id` simply not there (`directory not found`). -/// Both read as "nothing here yet", and both lead to a new, empty backup next -/// to the real one. These traces are the second witness: they exist only -/// where a mount of this remote really ran. -public enum InstallationTraces { - - /// Folders of `remoteRoot` that a mount of `remoteName` has cached on this - /// Mac - `cache/vfs//CloudMachine/`. - /// - /// rclone names the cache directory after the remote, with a suffix when - /// the mount overrides backend options: on this project's own Mac it is - /// `gdrive{hekjD}`, not `gdrive`. Both forms are accepted, and nothing - /// else - `gdrive2` is another remote. - /// - /// rclone creates the directory only once the remote has been opened, so - /// a mount that failed for want of a remote leaves none. - public static func cachedFolders( - cacheDir: URL = DriveBufferService.cacheDir, - remoteName: String = DriveBufferService.remoteName, - remoteRoot: String = DriveBufferService.remoteRoot - ) -> [String] { - let fm = FileManager.default - let vfs = cacheDir.appendingPathComponent("vfs") - guard let remotes = try? fm.contentsOfDirectory(atPath: vfs.path) else { return [] } - var folders: [String] = [] - for remote in remotes where remote == remoteName || remote.hasPrefix(remoteName + "{") { - let root = vfs.appendingPathComponent(remote).appendingPathComponent(remoteRoot) - let names = (try? fm.contentsOfDirectory(atPath: root.path)) ?? [] - folders += names.filter { DriveFolder.isValid($0) } - } - return Array(Set(folders)).sorted() - } - - /// The cached copy of `folder`'s image, if a mount has ever held it. - public static func cachedImage( - folder: String, imageName: String, - cacheDir: URL = DriveBufferService.cacheDir, - remoteName: String = DriveBufferService.remoteName, - remoteRoot: String = DriveBufferService.remoteRoot - ) -> URL? { - let fm = FileManager.default - let vfs = cacheDir.appendingPathComponent("vfs") - guard let remotes = try? fm.contentsOfDirectory(atPath: vfs.path) else { return nil } - for remote in remotes.sorted() where remote == remoteName || remote.hasPrefix(remoteName + "{") - { - let image = vfs.appendingPathComponent(remote).appendingPathComponent(remoteRoot) - .appendingPathComponent(folder).appendingPathComponent("\(imageName).sparsebundle") - if fm.fileExists(atPath: image.path) { return image } - } - return nil - } - - // MARK: - The image this Mac has used - - /// Written when an image is created or attached: the Drive path of the - /// image this Mac backs up to. Unlike the cache it survives clearing the - /// buffer. - public static var knownImageFile: URL { - CMPaths.appSupportDir.appendingPathComponent("known-image") - } - - /// Remembers that the image at `remotePath` exists. Best effort: a failed - /// write leaves the cache trace as the only witness, as before. - public static func rememberImage( - remotePath: String = DriveBufferService.remotePath, file: URL = knownImageFile - ) { - let line = "\(DriveBufferService.remoteName):\(remotePath)" - if (try? String(contentsOf: file, encoding: .utf8))? - .trimmingCharacters(in: .whitespacesAndNewlines) == line - { - return - } - try? line.write(to: file, atomically: true, encoding: .utf8) - } - - /// Where this Mac keeps proof that the image of `folder` has existed - - /// `nil` when it keeps none. A path, so a refusal can say what to remove - /// when the image really was deleted on purpose. - public static func knownImageTrace( - folder: String = DriveFolder.name, - imageName: String = BackupImageService.imageName, - remotePath: String = DriveBufferService.remotePath, - knownImageFile: URL = knownImageFile, - cacheDir: URL = DriveBufferService.cacheDir - ) -> String? { - let remembered = (try? String(contentsOf: knownImageFile, encoding: .utf8))? - .trimmingCharacters(in: .whitespacesAndNewlines) - if remembered == "\(DriveBufferService.remoteName):\(remotePath)" { - return knownImageFile.path - } - return cachedImage(folder: folder, imageName: imageName, cacheDir: cacheDir)?.path - } -} diff --git a/mac-app/Sources/CloudMachineCore/L10n.swift b/mac-app/Sources/CloudMachineCore/L10n.swift index a6cf616..442126a 100644 --- a/mac-app/Sources/CloudMachineCore/L10n.swift +++ b/mac-app/Sources/CloudMachineCore/L10n.swift @@ -62,6 +62,7 @@ public enum L10n { L10nPolish.agent, L10nPolish.safety, L10nPolish.vm, + L10nPolish.machine, ] } diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift index 770b9d7..2a0fd7a 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift @@ -15,14 +15,10 @@ extension L10nPolish { "Teraz ustaw limit Time Machine (wymaga hasła administratora):", "CloudMachine - verification, Google Drive setup and installation. Called by launchd on a schedule, or by hand from Terminal.": "CloudMachine - weryfikacja, konfiguracja Google Drive i instalacja. Wołane przez launchd według harmonogramu albo ręcznie z Terminala.", - "Generates and installs the launchd agents (Drive buffer, image attach, buffer guard).": - "Generuje i instaluje agentów launchd (bufor Drive, podpięcie obrazu, dozorca bufora).", "Connects to Google Drive through rclone (OAuth in the browser) and creates this machine's folder.": "Łączy z Google Drive przez rclone (OAuth w przeglądarce) i tworzy folder tej maszyny.", "Overwrite the existing remote. RISKY: replaces the token and permissions.": "Nadpisz istniejący remote. RYZYKOWNE: podmienia token i uprawnienia.", - "Installs rclone through Homebrew - WARNING: this build CANNOT mount, see install-rclone.": - "Instaluje rclone przez Homebrew - UWAGA: ta wersja NIE umie montować, patrz install-rclone.", "Packs build/CloudMachine.app into build/CloudMachine-.dmg.": "Pakuje build/CloudMachine.app do pliku build/CloudMachine-.dmg.", "ERROR: %@ is missing - run 'cloudmachine-agent build-app' first": @@ -43,8 +39,8 @@ extension L10nPolish { "2. W Finderze kliknij CloudMachine.app PRAWYM przyciskiem -> Otwórz -> Otwórz\n (samo dwukliknięcie pokaże blokadę Gatekeepera „niezidentyfikowany deweloper”).", "3. Later launches work normally, with a double-click.": "3. Kolejne uruchomienia działają już normalnie, dwuklikiem.", - "Builds CloudMachine.app (Release) - GUI + cloudmachine-agent in Contents/MacOS/, plus launchd/config as Resources.": - "Buduje CloudMachine.app (Release) - GUI + cloudmachine-agent w Contents/MacOS/, plus launchd/config jako Resources.", + "Builds CloudMachine.app (Release) - GUI + cloudmachine-agent in Contents/MacOS/, plus launchd/, vm/ and config/ as Resources.": + "Buduje CloudMachine.app (Release) - GUI + cloudmachine-agent w Contents/MacOS/, plus launchd/, vm/ i config/ jako Resources.", "Binaries for Apple Silicon and Intel at once (how CI builds a release; unnecessary locally).": "Binarki dla Apple Silicon i Intela naraz (tak buduje wydanie CI; lokalnie zbędne).", "==> Building CloudMachineApp + cloudmachine-agent (release) - version %@ (%@)": @@ -93,44 +89,6 @@ extension L10nPolish { "Następne 'cloudmachine-agent build-app' użyje go automatycznie zamiast podpisu ad-hoc.", "After THAT ONE rebuild, grant Full Disk Access one last time - later\nrebuilds will no longer reset it, as long as you sign with the same certificate.": "Po TYM JEDNYM rebuildzie przyznaj Pełny dostęp do dysku ostatni raz - kolejne\nprzebudowy już go nie zresetują, dopóki podpisujesz tym samym certyfikatem.", - "Mounts Google Drive with a write buffer. Stays in the foreground (for launchd).": - "Montuje Google Drive z buforem zapisu. Zostaje na pierwszym planie (dla launchd).", - "Already mounted: %@": - "Już zamontowane: %@", - "Missing: %@\n %@\n": - "Brakuje: %@\n %@\n", - "Could not start %@\n": - "Nie udało się uruchomić %@\n", - "Creates the backup image on Google Drive. One-off.": - "Tworzy obraz backupu na Google Drive. Jednorazowo.", - "Declared size in GB (the image is sparse).": - "Rozmiar deklarowany w GB (obraz jest rzadki).", - "Next step: %@, then": - "Następny krok: %@, potem", - "Attaches the backup image as the Time Machine destination.": - "Podpina obraz backupu jako cel Time Machine.", - "The buffer did not come up within %@ min - not attaching the image.": - "Bufor nie stanął w %@ min - nie podpinam obrazu.", - "Time Machine now has NO DESTINATION. Check: %@": - "Time Machine jest teraz BEZ CELU. Sprawdź: %@", - "This is not an error - the agent's next run will try again.": - "Nie jest to błąd - następny przebieg agenta spróbuje ponownie.", - "Detaches the image and waits until everything reaches Google Drive.": - "Odpina obraz i czeka, aż wszystko doleci na Google Drive.", - "Do not wait for the upload - RISKY, see BackupImageService.detach.": - "Nie czekaj na wysyłkę - RYZYKOWNE, patrz BackupImageService.detach.", - "Checks the image's consistency with fsck_apfs (hdiutil verify does not work on a sparsebundle).": - "Sprawdza spójność obrazu przez fsck_apfs (hdiutil verify na sparsebundle nie działa).", - "Pauses Time Machine when the unsent backlog grows faster than the upload goes.": - "Wstrzymuje Time Machine, gdy zaległość niewysłana rośnie szybciej, niż idzie wysyłka.", - "Above this many GB of unsent backlog we pause Time Machine.": - "Powyżej tylu GB zaległości niewysłanej wstrzymujemy Time Machine.", - "Below this many GB of unsent backlog we resume.": - "Poniżej tylu GB zaległości niewysłanej wznawiamy.", - "Below this many GB free on disk we pause regardless of the buffer.": - "Poniżej tylu GB wolnych na dysku wstrzymujemy niezależnie od bufora.", - "How often to check, in seconds.": - "Co ile sekund sprawdzać.", "Checks whether the hourly cycle STILL works (date of the last SUCCESSFUL backup) and reports failures.": "Sprawdza, czy cykl godzinowy NADAL działa (data ostatniej UDANEJ kopii), i zgłasza awarie.", "After this many hours without a successful backup we consider the cycle broken.": @@ -153,40 +111,14 @@ extension L10nPolish { "AWARIA: %@", " %@": " %@", - "Prepares for a restart: pauses the backup, detaches the image and waits for the upload.": - "Przygotowuje do restartu: wstrzymuje backup, odpina obraz i czeka na wysyłkę.", - "Pausing the backup...": - "Wstrzymuję backup...", - "Detaching the image and waiting for the upload...": - "Odpinam obraz i czekam na wysyłkę...", - "Do NOT restart yet - the buffer holds data that has not reached Drive.": - "NIE RESTARTUJ jeszcze - w buforze są dane, które nie doleciały na Dysk.", "Check the state: %@": "Sprawdź stan: %@", - "Safe to restart. The image stays detached for %@ min; after startup the agents will bring up the buffer and attach the image themselves.": - "Można restartować. Obraz zostaje odpięty przez %@ min; po starcie agenty podniosą bufor i podepną obraz same.", - "Safe to restart now - but the image could not be held detached, so the agent may attach it again within 15 min. Restart soon. After startup the agents will bring up the buffer and attach the image themselves.": - "Można restartować teraz - ale nie udało się zatrzymać obrazu odpiętego, więc agent może go podpiąć w ciągu 15 min. Zrestartuj wkrótce. Po starcie agenty podniosą bufor i podepną obraz same.", - "The Mac is prepared for a restart - not attaching the image. This is not an error: after the restart, or %@ min after the preparation, the image is attached again.": - "Mac jest przygotowany do restartu - obraz nie jest podpinany. To nie błąd: po restarcie albo %@ min po przygotowaniu obraz zostanie podpięty znowu.", "Preparing for restart": "Przygotowanie do restartu", - "Ready for restart - the image is detached on purpose": - "Gotowe do restartu - obraz odpięty celowo", "Prepare for restart": "Przygotuj do restartu", "Prepare": "Przygotuj", - "The backup is paused, the image is detached and everything waits until it has reached Google Drive. It can take several minutes.": - "Kopia zostanie wstrzymana, obraz odpięty, a wszystko poczeka, aż trafi na Google Drive. To może potrwać kilka minut.", - "State of the buffer, the upload queue and Time Machine.": - "Stan bufora, kolejki wysyłki i Time Machine.", - "Tools: %@": - "Narzędzia: %@", - "missing: %@": - "brakuje: %@", - "Drive mount: %@": - "Montowanie Drive: %@", "==> WARNING: git rev-list failed (%@) - using the date as the build number.": "==> UWAGA: git rev-list nie powiódł się (%@) - jako numer budowy idzie data.", "Agents: %@": @@ -199,50 +131,8 @@ extension L10nPolish { "Folder na Drive: %@", "Name of this Mac's folder on Google Drive (default: derived from the computer name). Set once; it cannot be changed later.": "Nazwa folderu tego Maca na Google Drive (domyślnie z nazwy komputera). Ustawiana raz; później nie da się jej zmienić.", - "Image attached: %@": - "Obraz podpięty: %@", - "Remote control: %@": - "Sterowanie rc: %@", - "private socket": - "prywatne gniazdo", - "OPEN on %@ - the mount was started by an older version, and any web page can send it commands. It switches to the private socket on its next start: run restart-mount (or prepare-shutdown, then restart the Mac).": - "OTWARTE na %@ - montowanie uruchomiła starsza wersja i każda strona WWW może wysyłać mu polecenia. Przejdzie na prywatne gniazdo przy następnym starcie: uruchom restart-mount (albo prepare-shutdown, potem zrestartuj Maca).", - "no socket - UNKNOWN whether the mount is running": - "brak gniazda - NIE WIADOMO, czy montowanie działa", - "no socket - the mount is not running": - "brak gniazda - montowanie nie działa", - "Cache on disk: %@": - "Cache na dysku: %@", - "To upload: %@": - "Do wysłania: %@", - "Free on disk: %@": - "Wolne na dysku: %@", - "Upload queue: %@ in progress, %@ queued, %@ errors": - "Kolejka wysyłki: %@ w toku, %@ w kolejce, %@ błędów", - "Upload queue: (rc interface unreachable)": - "Kolejka wysyłki: (interfejs rc nieosiągalny)", - "Restart without asking: %@": - "Restart bez pytania: %@", - "YES - queue empty": - "TAK - kolejka pusta", - "NO - run prepare-shutdown first": - "NIE - najpierw prepare-shutdown", - "Upload: %@": - "Wysyłka: %@", - "TM destination: %@": - "Cel Time Machine: %@", - "TM destination: none": - "Cel Time Machine: brak", - "Backup: running, %@%% (%@)": - "Backup: trwa, %@%% (%@)", - "Backup: not running": - "Backup: nie trwa", "Backup watchdog: %@": "Czujka backupu: %@", - "Pulls FUSE-T into CloudMachine, so there is no separate app in the system.": - "Wciąga FUSE-T do CloudMachine, żeby nie było osobnej aplikacji w systemie.", - "Downloads the official rclone binary (the Homebrew one cannot mount).": - "Pobiera oficjalną binarkę rclone (ta z Homebrew nie umie montować).", "Prints the version, the build number and the commit this binary was built from.": "Wypisuje wersję, numer budowy i commit, z którego zbudowano tę binarkę.", "Only one line, without a description.": diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+App.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+App.swift index 57a1ef4..4c3403a 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+App.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+App.swift @@ -26,20 +26,10 @@ extension L10nPolish { "Set the Time Machine quota: run this in Terminal (needs sudo)": "Ustaw limit Time Machine: uruchom to w Terminalu (wymaga sudo)", // SetupPlan / setup card - "Install rclone (the official build, which can mount)": - "Zainstaluj rclone (oficjalną wersję, która umie montować)", - "Install FUSE-T": "Zainstaluj FUSE-T", "Connect Google Drive: run this in Terminal and approve in the browser": "Połącz Google Drive: uruchom to w Terminalu i zatwierdź w przeglądarce", "Grant Full Disk Access to CloudMachine, so it can tell whether backups complete": "Nadaj CloudMachine Pełny dostęp do dysku, żeby mógł sprawdzać, czy kopie się kończą", - "Install the background agents (mount, image attach, watchdogs)": - "Zainstaluj agentów w tle (montowanie, podpinanie obrazu, czujki)", - "Create the backup image on Google Drive": "Utwórz obraz kopii na Google Drive", - "Attach the backup image": "Podepnij obraz kopii", - "Point Time Machine at CloudMachine: run this in Terminal (needs sudo)": - "Ustaw CloudMachine jako dysk Time Machine: uruchom to w Terminalu (wymaga sudo)", - "Size (GB)": "Rozmiar (GB)", "Folder on Google Drive": "Folder na Google Drive", "One folder per Mac. Set once - it cannot be changed after connecting.": "Jeden folder na Maca. Ustawiany raz - po połączeniu nie da się go zmienić.", @@ -47,17 +37,12 @@ extension L10nPolish { "Install rclone": "Zainstaluj rclone", "Open System Settings": "Otwórz Ustawienia systemowe", "Install agents": "Zainstaluj agentów", - "Create image": "Utwórz obraz", - "Attach image": "Podepnij obraz", - "Installing FUSE-T": "Instaluję FUSE-T", // AppStatus "not checked": "nie sprawdzono", "none at all": "ani jednej", "%@ ago": "%@ temu", "Missing: %@": "Brakuje: %@", "Google Drive not connected": "Google Drive niepołączony", - "Buffer is not working": "Bufor nie działa", - "Backup image not attached": "Obraz backupu niepodpięty", "Time Machine does not point to CloudMachine": "Time Machine nie wskazuje na CloudMachine", "UNKNOWN whether Time Machine points to CloudMachine - tmutil did not answer": "NIE WIADOMO, czy Time Machine wskazuje na CloudMachine - tmutil nie odpowiedział", @@ -70,9 +55,6 @@ extension L10nPolish { // CloudMachineController "Not saved: %@": "Nie zapisano: %@", "Installing rclone": "Instaluję rclone", - "Creating the backup image": "Tworzę obraz backupu", - "Attaching the image": "Podpinam obraz", - "Checking image consistency": "Sprawdzam spójność obrazu", "Installing launchd agents": "Instaluję agentów launchd", "Starting backup": "Uruchamiam backup", "Backup started.": "Backup uruchomiony.", @@ -89,7 +71,6 @@ extension L10nPolish { "Waiting to upload": "Czeka na wysłanie", "%@ files": "%@ plików", "nothing": "nic", - "SSD buffer": "Bufor SSD", "Back up now": "Zrób backup teraz", "Stop backup": "Wstrzymaj backup", "Open CloudMachine": "Otwórz CloudMachine", @@ -101,47 +82,25 @@ extension L10nPolish { "Refreshed %@": "Odświeżono %@", "System Status": "Stan Systemu", "Action needed": "Wymaga akcji", - "Free on disk: %@": "Wolne na dysku: %@", - "%@ GB": "%@ GB", "not measured": "nie zmierzono", "rclone did not answer": "rclone nie odpowiedział", - "Attached": "Podpięty", - "Not attached": "Niepodpięty", - "Drive disconnected": "Drive rozłączony", "Backup in Progress": "Kopia Zapasowa w Toku", "Files processed": "Przetworzone pliki", "Write speed": "Prędkość zapisu", "Operation phase": "Faza operacji", "Required Setup Steps": "Wymagane Kroki Konfiguracji", "Copy": "Kopiuj", - "Local Buffer & Upload Status": "Bufor Lokalny & Stan Wysyłki", - "Google Drive mount (FUSE-T)": "Montowanie Google Drive (FUSE-T)", - "Mounted": "Zamontowany", - "Inactive": "Nieaktywny", - "Backup disk image (.sparsebundle)": "Obraz dysku backupu (.sparsebundle)", - "Attached to the system": "Podpięty do systemu", - "Detached": "Odłączony", - "Buffer allocated on the SSD": "Zalokowany bufor na dysku SSD", - "Free space on the local volume": "Wolne miejsce na lokalnym wolumenie", "Last completed backup": "Ostatnia ukończona kopia", "Last backup watchdog run": "Ostatni przebieg czujki backupu", - "Cloud sync queue": "Kolejka synchronizacji z chmurą", "not read": "nie odczytano", - "%@ in progress, %@ queued": "%@ w toku, %@ w kolejce", "Everything uploaded": "Wszystko wysłane", - "%@ fragments abandoned - only on this Mac": "%@ fragmentów porzuconych - tylko na tym Macu", "File upload errors": "Błędy wysyłki plików", - "Space on Google Drive": "Miejsce na Google Drive", - "Out of space": "Brak miejsca", - "Google Drive limit": "Limit Google Drive", - "Daily 750 GB exhausted": "Dobowe 750 GB wyczerpane", "Google Drive Credentials (OAuth 2.0)": "Poświadczenia Google Drive (OAuth 2.0)", "Keychain OK": "Keychain OK", "No custom keys": "Brak własnych kluczy", "Paste client_id...": "Wklej client_id...", "Paste client_secret...": "Wklej client_secret...", "Save securely in the Keychain": "Zapisz bezpiecznie w Keychainie", - "Check image consistency": "Sprawdź spójność obrazu", "Refresh": "Odśwież", // Sidebar and overview cards "Overview": "Przegląd", @@ -149,9 +108,6 @@ extension L10nPolish { "Storage": "Miejsce", "Settings": "Ustawienia", "Current status: %@": "Bieżący stan: %@", - "Disks": "Dyski", - "Backup image": "Obraz kopii", - "Points to CloudMachine": "Wskazuje na CloudMachine", "Does not point to CloudMachine": "Nie wskazuje na CloudMachine", "tmutil did not answer": "tmutil nie odpowiedział", "Backing up: %@ of %@": "Kopiowanie: %@ z %@", diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift new file mode 100644 index 0000000..3662183 --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift @@ -0,0 +1,176 @@ +// Polish translations, keyed by the English text passed to `L10n.tr`. +// The machine layout as the app, the watchdog and prepare-shutdown show it. +extension L10nPolish { + static let machine: [String: String] = [ + "%@ GiB": + "%@ GiB", + "%@ GiB in %@ bands, %@ uploading": + "%@ GiB w %@ pasmach, wysyłanych %@", + "%@ bands": + "%@ pasm", + "%@ fragments abandoned - only in the machine": + "%@ porzuconych fragmentów - tylko w maszynie", + "127.0.0.1:5572 answers on the Mac: Lima forwards the machine's rclone remote control, which has no password. Any web page open in a browser can send it commands - including deleting the backup on Google Drive. Add the portForwards rule from vm/cmsmb.yaml to the machine and restart it (see vm/README.md).": + "127.0.0.1:5572 odpowiada na Macu: Lima przekazuje zdalne sterowanie rclone z maszyny, które nie ma hasła. Każda strona otwarta w przeglądarce może wysłać mu polecenie - także usunięcie kopii na Google Drive. Dodaj do maszyny regułę portForwards z vm/cmsmb.yaml i uruchom ją ponownie (patrz vm/README.md).", + "A service in the CloudMachine machine is not running": + "Usługa w maszynie CloudMachine nie działa", + "Address": + "Adres", + "Agent installation INCOMPLETE - %@ of %@ did not go in:\n%@\n%@\nEvery missing agent is a function that has silently stopped working (vm-guard watches the machine's buffer, backup-health reports failures). Fix the cause and repeat the installation.": + "Instalacja agentów NIEPEŁNA - %@ z %@ nie weszło:\n%@\n%@\nKażdy brakujący agent to funkcja, która po cichu przestała działać (vm-guard pilnuje bufora maszyny, backup-health zgłasza awarie). Usuń przyczynę i powtórz instalację.", + "All running": + "Wszystkie działają", + "Also install the launchd agents (same as install-launchd).": + "Zainstaluj też agentów launchd (to samo co install-launchd).", + "At most this many minutes of waiting for the upload.": + "Najwyżej tyle minut czekania na wysyłkę.", + "At the limit every new band fails with EMFILE and the backup stops. Check: cloudmachine-agent vm-status": + "Przy limicie każde nowe pasmo kończy się błędem EMFILE i kopia staje. Sprawdź: cloudmachine-agent vm-status", + "Cache on the machine's disk": + "Pamięć podręczna na dysku maszyny", + "Connected to Google Drive, but could not create the folder '%@' - without it the machine has nowhere to keep the backup. Check the account permissions and try again.": + "Połączono z Google Drive, ale nie udało się utworzyć folderu '%@' - bez niego maszyna nie ma gdzie trzymać kopii. Sprawdź uprawnienia konta i spróbuj ponownie.", + "Convert the image": + "Przerób obraz", + "Converting the image to 32 MB bands": + "Przerabianie obrazu na pasma 32 MB", + "Could not check with the current connection whether the backup is on Google Drive (%@), so remote '%@' was NOT replaced: without that answer the new sign-in cannot be checked against it.": + "Nie udało się sprawdzić obecnym połączeniem, czy kopia jest na Google Drive (%@), więc zdalny '%@' NIE został zastąpiony: bez tej odpowiedzi nowego logowania nie da się z nią porównać.", + "Downloads the official rclone binary into ~/.cloudmachine/bin (Google sign-in and the machine's rclone.conf).": + "Pobiera oficjalną binarkę rclone do ~/.cloudmachine/bin (logowanie do Google i rclone.conf dla maszyny).", + "Free in the machine": + "Wolne w maszynie", + "Free in the machine: %@": + "Wolne w maszynie: %@", + "Generates and installs the launchd agents (app, backup watchdog, machine start, machine guard).": + "Generuje i instaluje agentów launchd (aplikacja, czujka kopii, start maszyny, strażnik maszyny).", + "Install Lima, which runs the CloudMachine machine: run this in Terminal": + "Zainstaluj Limę, która uruchamia maszynę CloudMachine: wykonaj to w Terminalu", + "Install rclone (signs in to Google Drive for the machine)": + "Zainstaluj rclone (loguje maszynę do Google Drive)", + "Install the background agents (machine start, buffer guard, backup watchdog)": + "Zainstaluj agentów w tle (start maszyny, strażnik bufora, czujka kopii)", + "Installs rclone through Homebrew - CloudMachine itself uses the one from install-rclone.": + "Instaluje rclone przez Homebrew - sam CloudMachine używa tego z install-rclone.", + "Lima says the machine is running, but a command in it did not return. Without it nothing is known about Samba, rclone or the upload. Check: cloudmachine-agent vm-status": + "Lima mówi, że maszyna działa, ale polecenie w niej nie wróciło. Bez tego nic nie wiadomo o Sambie, rclone ani wysyłce. Sprawdź: cloudmachine-agent vm-status", + "Machine": + "Maszyna", + "Machine %@": + "Maszyna %@", + "Machine & Upload Status": + "Stan maszyny i wysyłki", + "Machine buffer": + "Bufor maszyny", + "No Time Machine destination is the machine's share (smb://tm@
/CloudMachine) - backups are not being made there. The command is printed by: cloudmachine-agent vm-setup": + "Żaden cel Time Machine nie jest udziałem maszyny (smb://tm@/CloudMachine) - kopie nie trafiają tam. Polecenie wypisuje: cloudmachine-agent vm-setup", + "Not running: %@": + "Nie działa: %@", + "Not set up": + "Nieskonfigurowana", + "Once it runs out, rclone stops uploading, the machine's buffer fills up and backups stop being made. At a growth of ~600 MB per hourly cycle that is about %@ days.": + "Gdy się skończy, rclone przestaje wysyłać, bufor maszyny się zapełnia i kopie przestają powstawać. Przy przyroście ~600 MB na cykl godzinowy to około %@ dni.", + "Or rclone.conf is missing: this Mac has already set up the CloudMachine machine, so it is not connected anew blindly. Restore the file from a backup (the machine keeps its own copy in %@). To connect anyway, create an empty file at the path `%@ config file` prints and run again - the existing folder is kept.": + "Albo brakuje rclone.conf: ten Mac ma już skonfigurowaną maszynę CloudMachine, więc nie jest łączony od nowa na ślepo. Przywróć plik z kopii (maszyna trzyma własną kopię w %@). Żeby mimo to połączyć, utwórz pusty plik pod ścieżką, którą wypisuje `%@ config file`, i uruchom ponownie - istniejący folder zostanie zachowany.", + "Point Time Machine at the machine: run this in Terminal (needs sudo; Copy includes the password)": + "Skieruj Time Machine na maszynę: wykonaj to w Terminalu (wymaga sudo; Kopiuj zawiera hasło)", + "Points to the machine": + "Wskazuje na maszynę", + "Prepares for a restart: stops the backup, waits a while for the upload, then stops the machine cleanly.": + "Przygotowuje do restartu: zatrzymuje kopię, czeka chwilę na wysyłkę, potem czysto zatrzymuje maszynę.", + "Reading the Time Machine destination did not return within %@ s. That is how tmutil behaves when its destination hangs.": + "Odczyt celu Time Machine nie wrócił w %@ s. Tak zachowuje się tmutil, gdy jego cel wisi.", + "Remote '%@' was NOT replaced. The new sign-in gets the 'drive.file' scope, which sees only files created under its own client_id. The existing backup was most likely created by the current connection and would become invisible to it. Before anything is written, the new token is checked against the existing backup, and the replacement is abandoned if it cannot see it. The machine keeps its own copy of the configuration until vm-setup --replace-rclone-config. To go ahead, run again with --replace-existing --accept-scope-change.": + "Zdalny '%@' NIE został zastąpiony. Nowe logowanie dostaje zakres 'drive.file', który widzi tylko pliki utworzone pod własnym client_id. Istniejąca kopia najpewniej powstała przez obecne połączenie i stałaby się dla niego niewidoczna. Zanim cokolwiek zostanie zapisane, nowy token jest sprawdzany na istniejącej kopii, a zamiana jest porzucana, jeśli jej nie widzi. Maszyna trzyma własną kopię konfiguracji aż do vm-setup --replace-rclone-config. Żeby kontynuować, uruchom ponownie z --replace-existing --accept-scope-change.", + "Remote control: OPEN on the Mac at 127.0.0.1:5572 - any web page can send rclone commands; see vm/README.md (portForwards)": + "Zdalne sterowanie: OTWARTE na Macu pod 127.0.0.1:5572 - każda strona może wysyłać polecenia rclone; patrz vm/README.md (portForwards)", + "Running": + "Działa", + "Samba, rclone or punchfs in the machine is not active, so Time Machine has nowhere to write or the bands do not reach Google Drive. Check: cloudmachine-agent vm-status": + "Samba, rclone albo punchfs w maszynie nie jest aktywna, więc Time Machine nie ma gdzie pisać albo pasma nie trafiają na Google Drive. Sprawdź: cloudmachine-agent vm-status", + "Service %@": + "Usługa %@", + "Services": + "Usługi", + "Set up the CloudMachine machine (the first time it downloads Ubuntu - several minutes)": + "Skonfiguruj maszynę CloudMachine (za pierwszym razem pobiera Ubuntu - kilka minut)", + "Set up the CloudMachine machine again, so it prints the Time Machine command": + "Skonfiguruj maszynę CloudMachine ponownie, żeby wypisała polecenie Time Machine", + "Set up the machine": + "Skonfiguruj maszynę", + "Setting up the CloudMachine machine": + "Konfigurowanie maszyny CloudMachine", + "Start the CloudMachine machine": + "Uruchom maszynę CloudMachine", + "Start the machine": + "Uruchom maszynę", + "Starting the CloudMachine machine": + "Uruchamianie maszyny CloudMachine", + "Still to upload: %@ GiB": + "Zostało do wysłania: %@ GiB", + "Stopping the Time Machine backup...": + "Zatrzymywanie kopii Time Machine...", + "Stopping the machine %@...": + "Zatrzymywanie maszyny %@...", + "The CloudMachine machine does not answer": + "Maszyna CloudMachine nie odpowiada", + "The CloudMachine machine is not running": + "Maszyna CloudMachine nie działa", + "The CloudMachine machine is not set up yet": + "Maszyna CloudMachine nie jest jeszcze skonfigurowana", + "The CloudMachine machine is stopped": + "Maszyna CloudMachine jest zatrzymana", + "The backup is stopped, the upload gets up to 30 minutes, then the machine shuts down cleanly. It starts again at the next login.": + "Kopia zostaje zatrzymana, wysyłka dostaje do 30 minut, potem maszyna czysto się wyłącza. Uruchomi się znowu przy następnym logowaniu.", + "The machine %@ is not running - nothing to prepare.": + "Maszyna %@ nie działa - nie ma czego przygotowywać.", + "The machine is stopped and everything reached Google Drive. The Mac can be restarted; the machine starts again at login.": + "Maszyna jest zatrzymana i wszystko dotarło na Google Drive. Można zrestartować Maca; maszyna uruchomi się znowu przy logowaniu.", + "The machine is stopped. %@ GiB had not reached Google Drive yet - it is kept in the machine's buffer and uploaded after the next start. The Mac can be restarted.": + "Maszyna jest zatrzymana. %@ GiB nie dotarło jeszcze na Google Drive - zostaje w buforze maszyny i zostanie wysłane po następnym starcie. Można zrestartować Maca.", + "The machine is stopped. rclone did not say how much was still to upload; what was not uploaded is kept in the machine's buffer and uploaded after the next start. The Mac can be restarted.": + "Maszyna jest zatrzymana. rclone nie powiedział, ile zostało do wysłania; to, co nie wyszło, zostaje w buforze maszyny i zostanie wysłane po następnym starcie. Można zrestartować Maca.", + "The machine's disk is a file on this disk and grows as the upload buffer fills. With no space left the machine cannot write the buffer, and Time Machine's local snapshots have nowhere to go either.": + "Dysk maszyny to plik na tym dysku i rośnie razem z buforem wysyłki. Bez wolnego miejsca maszyna nie zapisze bufora, a lokalne migawki Time Machine też nie mają gdzie trafić.", + "The machine's disk is running out of space (%@ GB)": + "Na dysku maszyny kończy się miejsce (%@ GB)", + "The new sign-in does NOT see the existing backup (%@), so remote '%@' was NOT replaced and rclone.conf is unchanged. Keep the current connection, or sign in with the credential (client_id) that created the backup.": + "Nowe logowanie NIE widzi istniejącej kopii (%@), więc zdalny '%@' NIE został zastąpiony, a rclone.conf jest bez zmian. Zostaw obecne połączenie albo zaloguj się poświadczeniem (client_id), które utworzyło kopię.", + "The upload buffer lives on this disk. The guard pauses Time Machine before it fills; if the upload does not catch up, backups stay paused.": + "Na tym dysku jest bufor wysyłki. Strażnik wstrzymuje Time Machine, zanim się zapełni; jeśli wysyłka nie nadgoni, kopie zostają wstrzymane.", + "These bands exist only in the machine's buffer. The backup on Google Drive is INCOMPLETE and may not open.": + "Te pasma istnieją tylko w buforze maszyny. Kopia na Google Drive jest NIEPEŁNA i może się nie otworzyć.", + "Time Machine backs up to the machine, so without it no backup is made. Start it: limactl start %@ (or the Start button in the app).": + "Time Machine robi kopię do maszyny, więc bez niej kopia nie powstaje. Uruchom ją: limactl start %@ (albo przyciskiem w aplikacji).", + "Time Machine did not stop its backup within 2 minutes (or tmutil did not answer) - the machine was NOT stopped. Try again in a moment.": + "Time Machine nie zatrzymał kopii w ciągu 2 minut (albo tmutil nie odpowiedział) - maszyna NIE została zatrzymana. Spróbuj za chwilę.", + "Time Machine has created its image: stop the backup, then convert the image to 32 MB bands": + "Time Machine utworzył obraz: zatrzymaj kopię, potem przerób obraz na pasma 32 MB", + "To upload": + "Do wysłania", + "Unknown whether the CloudMachine machine is running": + "Nie wiadomo, czy maszyna CloudMachine działa", + "Waiting for the upload to Google Drive (at most %@ min)...": + "Czekanie na wysyłkę na Google Drive (najwyżej %@ min)...", + "With --replace-existing: confirms that the new sign-in (scope 'drive.file') may not see the existing backup. The new token is still checked against the backup first.": + "Z --replace-existing: potwierdza, że nowe logowanie (zakres 'drive.file') może nie widzieć istniejącej kopii. Nowy token i tak jest najpierw sprawdzany na kopii.", + "Without it there is no way to check whether anything reached the Drive - the machine's guard is blind then.": + "Bez niego nie da się sprawdzić, czy cokolwiek dotarło na Drive - strażnik maszyny jest wtedy ślepy.", + "limactl did not answer": + "limactl nie odpowiedział", + "limactl did not answer (or is not installed). That does not mean the machine is down - it means nobody has checked.": + "limactl nie odpowiedział (albo nie jest zainstalowany). To nie znaczy, że maszyna nie działa - znaczy, że nikt tego nie sprawdził.", + "limactl start failed: %@": + "limactl start nie powiódł się: %@", + "limactl stop failed: %@": + "limactl stop nie powiódł się: %@", + "punchfs is close to its open-file limit": + "punchfs jest blisko limitu otwartych plików", + "punchfs open files": + "Otwarte pliki punchfs", + "rclone's remote control is open on this Mac": + "Zdalne sterowanie rclone jest otwarte na tym Macu", + "statfs('/System/Volumes/Data') returned an error, so nobody knows whether the machine's disk can still grow on this Mac.": + "statfs('/System/Volumes/Data') zwrócił błąd, więc nikt nie wie, czy dysk maszyny może jeszcze rosnąć na tym Macu.", + ] +} diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+Safety.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+Safety.swift index b840486..7dc1871 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+Safety.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+Safety.swift @@ -6,84 +6,18 @@ extension L10nPolish { // MachineBudget "Usage on Google Drive is not being measured": "Zajętość na Google Drive nie jest mierzona", - "The last measurement (%@ GB) is from %@, %@ ago - rclone size has not answered since. Until it does, nothing watches the %@ GB limit: the 90%% and 100%% alarms stay silent. Check drive-status, or press Measure now in the window.": - "Ostatni pomiar (%@ GB) jest z %@, sprzed %@ - od tamtej pory rclone size nie odpowiada. Dopóki nie odpowie, nikt nie pilnuje limitu %@ GB: alarmy 90%% i 100%% milczą. Sprawdź drive-status albo naciśnij „Zmierz teraz” w oknie.", - "rclone size has not answered even once. Until it does, nothing watches the %@ GB limit: the 90%% and 100%% alarms stay silent. Check drive-status, or press Measure now in the window.": - "rclone size nie odpowiedział ani razu. Dopóki nie odpowie, nikt nie pilnuje limitu %@ GB: alarmy 90%% i 100%% milczą. Sprawdź drive-status albo naciśnij „Zmierz teraz” w oknie.", - // RemoteConfigurer: a missing rclone.conf - "Or rclone.conf is missing: this Mac has already backed up to Google Drive (folders %@), so it is not connected anew blindly. Restore the file from a backup. To connect anyway, create an empty file at the path `%@ config file` prints and run again - the existing folder is kept.": - "Albo brakuje rclone.conf: ten Mac robił już kopie na Google Drive (foldery %@), więc nie jest łączony od nowa na ślepo. Odtwórz plik z kopii. Żeby mimo to połączyć, utwórz pusty plik pod ścieżką, którą wypisuje `%@ config file`, i uruchom ponownie - istniejący folder zostanie zachowany.", + "The last measurement (%@ GB) is from %@, %@ ago - rclone size has not answered since. Until it does, nothing watches the %@ GB limit: the 90%% and 100%% alarms stay silent. Check vm-status, or press Measure now in the window.": + "Ostatni pomiar (%@ GB) jest z %@, sprzed %@ - od tamtej pory rclone size nie odpowiada. Dopóki nie odpowie, nikt nie pilnuje limitu %@ GB: alarmy 90%% i 100%% milczą. Sprawdź vm-status albo naciśnij „Zmierz teraz” w oknie.", + "rclone size has not answered even once. Until it does, nothing watches the %@ GB limit: the 90%% and 100%% alarms stay silent. Check vm-status, or press Measure now in the window.": + "rclone size nie odpowiedział ani razu. Dopóki nie odpowie, nikt nie pilnuje limitu %@ GB: alarmy 90%% i 100%% milczą. Sprawdź vm-status albo naciśnij „Zmierz teraz” w oknie.", // Replacing the Google credential "Saved in the Keychain. Connecting Google Drive (configure-remote) will use them.": "Zapisane w Keychainie. Łączenie z Google Drive (configure-remote) ich użyje.", "Saved in the Keychain. The existing Google Drive connection keeps its old client_id, and on this Mac it should stay that way: a connection made with the new keys (the 'drive.file' scope) cannot see the backup created by the old one. Do not replace the connection here; the new keys are for a Mac that is not connected yet.": "Zapisane w Keychainie. Istniejące połączenie z Google Drive zostaje przy starym client_id i na tym Macu tak ma zostać: połączenie z nowymi kluczami (zakres 'drive.file') nie widzi kopii założonej przez stare. Nie wymieniaj tu połączenia; nowe klucze są dla Maca, który jeszcze nie jest połączony.", - "Remote '%@' was NOT replaced. The new sign-in gets the 'drive.file' scope, which sees only files created under its own client_id. The existing backup was most likely created by the current connection and would become invisible: the mount would show an empty folder, and create-image could start a second, empty backup next to it. Before anything is written, the new token is checked against the existing image, and the replacement is abandoned if it cannot see it. To go ahead, run again with --replace-existing --accept-scope-change.": - "Remote '%@' NIE został wymieniony. Nowe logowanie dostaje zakres 'drive.file', który widzi tylko pliki założone przez jego własny client_id. Istniejącą kopię najpewniej założyło obecne połączenie i stałaby się niewidoczna: mount pokazałby pusty folder, a create-image mógłby założyć obok drugą, pustą kopię. Zanim cokolwiek zostanie zapisane, nowy token jest sprawdzany na istniejącym obrazie, a wymiana jest porzucana, jeśli go nie widzi. Żeby kontynuować, uruchom ponownie z --replace-existing --accept-scope-change.", - "Could not check with the current connection whether the backup image is on Google Drive (%@), so remote '%@' was NOT replaced: without that answer the new sign-in cannot be checked against it.": - "Nie udało się sprawdzić obecnym połączeniem, czy obraz kopii jest na Google Drive (%@), więc remote '%@' NIE został wymieniony: bez tej odpowiedzi nie ma z czym porównać nowego logowania.", "it is not there for the new token": - "dla nowego tokenu go nie ma", - "The new sign-in does NOT see the existing backup image (%@), so remote '%@' was NOT replaced and rclone.conf is unchanged. With this token the mount would show an empty folder. Keep the current connection, or sign in with the credential (client_id) that created the image.": - "Nowe logowanie NIE widzi istniejącego obrazu kopii (%@), więc remote '%@' NIE został wymieniony, a rclone.conf jest bez zmian. Z tym tokenem mount pokazałby pusty folder. Zostaw obecne połączenie albo zaloguj się poświadczeniem (client_id), które założyło obraz.", - "With --replace-existing: confirms that the new sign-in (scope 'drive.file') may not see the existing backup. The new token is still checked against the image first.": - "Z --replace-existing: potwierdza, że nowe logowanie (zakres 'drive.file') może nie widzieć istniejącej kopii. Nowy token i tak jest najpierw sprawdzany na obrazie.", - // BackupImageService.create - "Google Drive does not show the image %@, but this Mac has already used it (trace: %@). NOT creating a new one - that would start an empty backup and orphan the existing one. Most likely the token cannot see it: a connection with the 'drive.file' scope sees only files created under its own client_id. Check the token's permissions (scope and client_id of the remote in rclone.conf). Only if the image really was deleted on purpose: remove %@ and run again.": - "Google Drive nie pokazuje obrazu %@, a ten Mac już go używał (ślad: %@). NIE tworzę nowego - to zaczęłoby pustą kopię i osierociło istniejącą. Najpewniej token go nie widzi: połączenie z zakresem 'drive.file' widzi tylko pliki założone przez własny client_id. Sprawdź uprawnienia tokenu (scope i client_id remote'a w rclone.conf). Tylko jeśli obraz naprawdę został celowo usunięty: usuń %@ i uruchom ponownie.", - - // The open remote-control port and restarting the mount - "WARNING: remote control is OPEN on %@ - run restart-mount (or prepare-shutdown, then restart the Mac).": - "UWAGA: sterowanie rc jest OTWARTE na %@ - uruchom restart-mount (albo prepare-shutdown, potem zrestartuj Maca).", - "Restarts the Google Drive mount on the private socket: detaches the image, waits for the upload, restarts the mount and attaches the image again.": - "Restartuje montowanie Google Drive na prywatnym gnieździe: odłącza obraz, czeka na wysyłkę, restartuje montowanie i podłącza obraz z powrotem.", - "Detaching the image and waiting for the upload, then restarting the mount. This can take several minutes; Time Machine has no destination meanwhile.": - "Odłączam obraz i czekam na wysyłkę, potem restartuję montowanie. To może potrwać kilka minut; Time Machine nie ma w tym czasie celu.", - "Restarting the Google Drive mount": - "Restartowanie montowania Google Drive", - "Remote control of the Drive mount is open": - "Sterowanie montowaniem Dysku jest otwarte", - "The mount was started by an older version of CloudMachine and still takes commands on %@ without a password. Any web page open in a browser can send it one - including deleting the backup on Google Drive, with no trash to restore it from. It moves to a private socket the next time it starts.": - "Montowanie uruchomiła starsza wersja CloudMachine i nadal przyjmuje polecenia na %@ bez hasła. Każda strona otwarta w przeglądarce może je wysłać - łącznie z usunięciem kopii na Google Drive, bez kosza, z którego dałoby się ją odzyskać. Przejdzie na prywatne gniazdo przy następnym starcie.", - "Restart the mount: the image is detached, everything waits until it has reached Google Drive, the mount starts again and the image is attached. It takes a few minutes, during which Time Machine has no destination. Or prepare the shutdown and restart the Mac:": - "Zrestartuj montowanie: obraz zostanie odłączony, wszystko poczeka, aż dotrze na Google Drive, montowanie wystartuje od nowa, a obraz zostanie podłączony. To trwa kilka minut, w tym czasie Time Machine nie ma celu. Albo przygotuj zamknięcie i zrestartuj Maca:", - "Wait until the backup has finished.": - "Poczekaj, aż kopia się skończy.", - "Restart the Google Drive mount now?": - "Zrestartować teraz montowanie Google Drive?", - "Restart the mount": - "Zrestartuj montowanie", + "dla nowego tokenu jej nie ma", "Cancel": "Anuluj", - "Time Machine loses its destination for a few minutes. Nothing is stopped until everything waiting has reached Google Drive; if that does not happen, the mount is left as it is.": - "Time Machine straci cel na kilka minut. Nic nie zostanie zatrzymane, dopóki wszystko, co czeka, nie dotrze na Google Drive; jeśli to się nie uda, montowanie zostaje bez zmian.", - "Drive remote control is open - restart the mount": - "Sterowanie Dyskiem jest otwarte - zrestartuj montowanie", - "The mount was restarted and now listens only on the private socket. The image is attached again.": - "Montowanie zostało zrestartowane i słucha już tylko na prywatnym gnieździe. Obraz jest znowu podłączony.", - "Another operation on the image is in progress - the mount was NOT restarted. Try again in a moment.": - "Trwa inna operacja na obrazie - montowanie NIE zostało zrestartowane. Spróbuj ponownie za chwilę.", - "Could not read the mount table - it is UNKNOWN whether the image is attached.": - "Nie udało się odczytać tablicy montowań - NIE WIADOMO, czy obraz jest podłączony.", - "Time Machine is backing up right now - the mount was NOT restarted. Try again when the backup has finished.": - "Time Machine właśnie robi kopię - montowanie NIE zostało zrestartowane. Spróbuj ponownie, gdy kopia się skończy.", - "Could not tell whether Time Machine is backing up (tmutil did not answer) - the mount was NOT restarted.": - "Nie wiadomo, czy Time Machine robi kopię (tmutil nie odpowiedział) - montowanie NIE zostało zrestartowane.", - "The Google Drive mount is not running (or its state is unknown) - there is nothing to restart. The agents start it by themselves.": - "Montowanie Google Drive nie działa (albo jego stan jest nieznany) - nie ma czego restartować. Agenci uruchamiają je sami.", - "The mount was NOT restarted: %@\nImage: %@": - "Montowanie NIE zostało zrestartowane: %@\nObraz: %@", - "Could not stop the mount (launchctl refused) - nothing was restarted. Image: %@": - "Nie udało się zatrzymać montowania (launchctl odmówił) - nic nie zostało zrestartowane. Obraz: %@", - "The mount was stopped, but did not come back on the private socket in time. The agents keep trying (backup-health reloads one launchd cannot start). Check: cloudmachine-agent drive-status; if it stays down, restart the Mac.": - "Montowanie zostało zatrzymane, ale nie wróciło na czas na prywatnym gnieździe. Agenci dalej próbują (backup-health przeładowuje agenta, którego launchd nie potrafi uruchomić). Sprawdź: cloudmachine-agent drive-status; jeśli dalej nie działa, zrestartuj Maca.", - "The mount runs on the private socket now, but the image did not attach: %@ The attach agent retries every 15 minutes.": - "Montowanie działa już na prywatnym gnieździe, ale obraz się nie podłączył: %@ Agent podłączania próbuje co 15 minut.", - "The image kept being refused - the Google Drive mount was restarted: %@": - "Obraz był wciąż odrzucany - montowanie Dysku Google zostało zrestartowane: %@", - "CloudMachine: Drive remote control is open": - "CloudMachine: sterowanie Dyskiem jest otwarte", - "Any web page can send it commands, including deleting the backup. Open CloudMachine and restart the mount, or run prepare-shutdown and restart the Mac.": - "Każda strona WWW może wysyłać mu polecenia, łącznie z usunięciem kopii. Otwórz CloudMachine i zrestartuj montowanie albo uruchom prepare-shutdown i zrestartuj Maca.", ] } diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+Storage.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+Storage.swift index bc70040..a8e555c 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+Storage.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+Storage.swift @@ -30,75 +30,9 @@ extension L10nPolish { "Nic nie czeka w kolejce — kopia na Google Drive jest kompletna.", "rclone did not answer the question about the queue, so it is unknown how many backups are still waiting to be uploaded. This does not mean something broke — under load the answer can be late. It only means that right now nobody knows. If it persists, the backup cycle check will report it.": "rclone nie odpowiedział na pytanie o kolejkę, więc nie wiadomo, ile kopii czeka jeszcze na wysłanie. To nie znaczy, że coś się zepsuło — pod obciążeniem odpowiedź potrafi się spóźnić. Znaczy tylko tyle, że w tej chwili nikt tego nie wie. Jeśli utrzymuje się dłużej, zgłosi to kontrola cyklu backupu.", - // BackupImageService - "%@: another image operation is in progress (creating, attaching, detaching or verifying) - NOTHING was done. Try again shortly.": - "%@: inna operacja na obrazie jest w toku (tworzenie, podpinanie, odpinanie albo sprawdzanie) - NIE zrobiono nic. Spróbuj za chwilę.", - "NOT ATTACHED": "BRAK", - "DEAD - in the mount table, but reads fail (errno %@); attach-image attaches it again": - "MARTWY - w tablicy montowań, ale odczyt pada (errno %@); attach-image podpina na nowo", - "UNKNOWN - in the mount table, but the readability probe did not answer within %@ s": - "NIE WIADOMO - w tablicy montowań, ale sonda czytelności nie odpowiedziała w %@ s", - "UNKNOWN - could not read the mount table": - "NIE WIADOMO - nie udało się odczytać tablicy montowań", - "Image creation": "Utworzenie obrazu", - "Drive is not mounted - start the buffer first.": - "Drive nie jest zamontowany - najpierw uruchom bufor.", - "Could not read the mount table - it is UNKNOWN whether the buffer is mounted. NOT creating the image.": - "Nie udało się odczytać tablicy montowań - NIE WIADOMO, czy bufor jest zamontowany. NIE tworzę obrazu.", - "The buffer did not come up within %@ min - NOT creating the image.": - "Bufor nie stanął w %@ min - NIE tworzę obrazu.", - "The image already exists. Deleting it erases the whole backup - do it deliberately.": - "Obraz już istnieje. Usunięcie go kasuje cały backup - zrób to świadomie.", - "The image already exists on Google Drive (the mount cache did not show it, but the remote has it). Deleting it erases the whole backup - do it deliberately.": - "Obraz już istnieje na Google Drive (cache montowania go nie pokazywał, ale zdalny go ma). Usunięcie go kasuje cały backup - zrób to świadomie.", - "Could not confirm on Google Drive that the image is not there yet (%@) - ABORTING. Creating the image over an existing backup is irreversible, so I do not start without that answer.": - "Nie udało się potwierdzić na Google Drive, że obrazu tam jeszcze nie ma (%@) - PRZERYWAM. Tworzenie obrazu na istniejącym backupie jest nieodwracalne, więc bez tej odpowiedzi nie zaczynam.", - "Could not create the image: %@": "Nie udało się utworzyć obrazu: %@", + // RemoteConfigurer "unknown error": "nieznany błąd", - "Created a %@ GB image, band size %@ MB.": "Utworzono obraz %@ GB, pasmo %@ MB.", "rclone did not answer": "rclone nie odpowiedział", "rclone lsf ended with an error": "rclone lsf zakończyło się błędem", - "Image attach": "Podpięcie obrazu", - "Drive is not mounted.": "Drive nie jest zamontowany.", - "Could not read the mount table - it is UNKNOWN whether the buffer is mounted. NOT attaching the image.": - "Nie udało się odczytać tablicy montowań - NIE WIADOMO, czy bufor jest zamontowany. NIE podpinam obrazu.", - "No image - create it first.": "Brak obrazu - najpierw go utwórz.", - "Already attached: %@": "Już podpięte: %@", - "The image is in the mount table, but the readability probe did not answer within %@ s - it is UNKNOWN whether the device is alive. NOT force-detaching and NOT attaching.": - "Obraz jest w tablicy montowań, ale sonda czytelności nie odpowiedziała w %@ s - NIE WIADOMO, czy urządzenie żyje. NIE odpinam na siłę i NIE podpinam.", - "Could not read the mount table - it is UNKNOWN whether the image is attached. NOT attaching.": - "Nie udało się odczytać tablicy montowań - NIE WIADOMO, czy obraz jest podpięty. NIE podpinam.", - "Image dead (errno %@) and could not be detached: %@": - "Obraz martwy (errno %@) i nie dał się odpiąć: %@", - "An orphaned mount point blocks the attach: %@\nRemove it and try again: sudo rmdir '%@'": - "Osierocony punkt montowania blokuje podpięcie: %@\nUsuń go i spróbuj ponownie: sudo rmdir '%@'", - "Could not attach the image: %@": "Nie udało się podpiąć obrazu: %@", - "Attached: %@": "Podpięte: %@", - "Image detach": "Odpięcie obrazu", - "Could not detach - the image is held by browsed backup snapshots that could not be unmounted:\n%@\nClose the Time Machine / Finder window on the backup and try again.": - "Nie udało się odpiąć - obraz trzymają przeglądane migawki backupu, których nie dało się odmontować:\n%@\nZamknij okno Time Machine / Findera na backupie i spróbuj ponownie.", - "Could not detach.": "Nie udało się odpiąć.", - "Detached (without waiting for the upload - the data may be local only).": - "Odpięte (bez czekania na wysyłkę - dane mogą być tylko lokalnie).", - "Detached, but the upload did NOT finish in time - do not delete the buffer.": - "Odpięte, ale wysyłka NIE zakończyła się w czasie - nie kasuj bufora.", - "Detached, but rclone ABANDONED %@ backup fragments - they exist only on this Mac and are not on Google Drive. Do not delete the buffer.": - "Odpięte, ale rclone PORZUCIŁ %@ fragmentów kopii - istnieją wyłącznie na tym Macu i na Google Drive ich nie ma. Nie kasuj bufora.", - "Detached, everything uploaded to Google Drive.": "Odpięte, wszystko wysłane na Google Drive.", - "Image verification": "Sprawdzenie obrazu", - "No image.": "Brak obrazu.", - "The image is attached - detach it before verifying.": - "Obraz jest podpięty - odepnij go przed sprawdzeniem.", - "Could not find an APFS device in the image.": "Nie znalazłem urządzenia APFS w obrazie.", - "Could not run %@ - the image's consistency REMAINS UNCHECKED.": - "Nie udało się uruchomić %@ - spójność obrazu POZOSTAJE NIESPRAWDZONA.", - "Check INTERRUPTED - device %@ disappeared midway (someone force-detached the image). This is not a result about the backup's state: the image's consistency REMAINS UNCHECKED. Repeat the check.": - "Sprawdzenie PRZERWANE - urządzenie %@ zniknęło w trakcie (ktoś odpiął obraz na siłę). To nie jest wynik o stanie backupu: spójność obrazu POZOSTAJE NIESPRAWDZONA. Powtórz sprawdzenie.", - "Image consistent.": "Obraz spójny.", - "Image INCONSISTENT: %@": "Obraz NIESPÓJNY: %@", - // BufferGuardService - "CloudMachine: upload to Drive is stalled": "CloudMachine: wysyłka na Dysk stoi", - "Upload to Google Drive is stalled - daily limit exhausted.": - "Wysyłka na Google Drive stoi - wyczerpany limit dobowy.", ] } diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+System.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+System.swift index ce620c4..bd2167b 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+System.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+System.swift @@ -3,12 +3,12 @@ extension L10nPolish { static let system: [String: String] = [ "This Mac is close to its space limit on Google Drive": "Ten Mac zbliża się do swojego limitu miejsca na Google Drive", - "%@ GB of %@ GB used. Time Machine deletes the oldest backups to stay within its quota; check that the quota is set (drive-status).": - "Zajęte %@ GB z %@ GB. Time Machine kasuje najstarsze kopie, żeby zmieścić się w limicie; sprawdź, czy limit jest ustawiony (drive-status).", + "%@ GB of %@ GB used. Time Machine deletes the oldest backups to stay within its quota; check that the quota is set (vm-status).": + "Zajęte %@ GB z %@ GB. Time Machine kasuje najstarsze kopie, żeby zmieścić się w limicie; sprawdź, czy limit jest ustawiony (vm-status).", "This Mac is over its space limit on Google Drive": "Ten Mac przekroczył swój limit miejsca na Google Drive", - "%@ GB of %@ GB used. Set the Time Machine quota (command in the app window or drive-status), or raise the limit.": - "Zajęte %@ GB z %@ GB. Ustaw limit Time Machine (polecenie w oknie aplikacji albo w drive-status) albo podnieś limit.", + "%@ GB of %@ GB used. Set the Time Machine quota (command in the app window or vm-status), or raise the limit.": + "Zajęte %@ GB z %@ GB. Ustaw limit Time Machine (polecenie w oknie aplikacji albo w vm-status) albo podnieś limit.", "not set": "nie ustawiony", "%@ GB - usage not measured yet": @@ -48,20 +48,6 @@ extension L10nPolish { "Keychain odmówił: %@", "DIRTY-TREE": "BRUDNE-DRZEWO", - "rclone with mount support": - "rclone z obsługą montowania", - "Could not determine the FUSE-T version.": - "Nie udało się ustalić wersji FUSE-T.", - "Could not download the FUSE-T package.": - "Nie udało się pobrać pakietu FUSE-T.", - "Could not unpack the FUSE-T package.": - "Nie udało się rozpakować pakietu FUSE-T.", - "The FUSE-T package does not contain the expected files.": - "W pakiecie FUSE-T nie ma spodziewanych plików.", - "Installing FUSE-T failed: %@": - "Instalacja FUSE-T nie powiodła się: %@", - "Installed FUSE-T %@ inside CloudMachine (%@).\nThe separate fuse-t application is no longer needed - you can remove it:\n sudo \"/Library/Application Support/fuse-t/uninstall.sh\"": - "Zainstalowano FUSE-T %@ wewnątrz CloudMachine (%@).\nOsobna aplikacja fuse-t nie jest już potrzebna - możesz ją usunąć:\n sudo \"/Library/Application Support/fuse-t/uninstall.sh\"", "Cannot create the working directory: %@": "Nie mogę utworzyć katalogu roboczego: %@", "Could not read the rclone version number.": @@ -90,50 +76,18 @@ extension L10nPolish { "Nie udało się sprawdzić, czy remote '%@' już istnieje (rclone listremotes nie odpowiedział albo zgłosił błąd), więc nic nie zostało zmienione. Uruchom `%@ listremotes`, żeby zobaczyć błąd, i spróbuj ponownie.", "Could not check whether this Mac already has a CloudMachine installation (rclone did not answer), so no folder was chosen. Try again in a moment.": "Nie udało się sprawdzić, czy ten Mac ma już instalację CloudMachine (rclone nie odpowiedział), więc folder nie został wybrany. Spróbuj ponownie za chwilę.", - "The remote control socket path is too long for macOS (%@ characters, at most %@): %@": - "Ścieżka gniazda zdalnego sterowania jest za długa dla macOS (%@ znaków, najwyżej %@): %@", "rclone authorize failed (%@). If that binary is missing, start with: cloudmachine-agent install-rclone.": "rclone authorize nie powiodło się (%@). Jeśli tej binarki nie ma, zacznij od: cloudmachine-agent install-rclone.", "Could not read the token from the output of rclone authorize.": "Nie udało się odczytać tokenu z wyniku rclone authorize.", "rclone config create failed.": "rclone config create nie powiodło się.", - "Connected to Google Drive, but could not create the folder '%@' - without it the mount will not start. Check the account permissions and try again.": - "Połączono z Google Drive, ale nie udało się utworzyć folderu '%@' - bez niego montowanie nie ruszy. Sprawdź uprawnienia konta i spróbuj ponownie.", "Connected to Google Drive as remote '%@', folder %@.": "Połączono z Google Drive jako remote '%@', folder %@.", - "Google Drive mount is not working": - "Montowanie Google Drive nie działa", - "Without it the backup image is unreachable and Time Machine has nowhere to write.": - "Bez niego obraz backupu jest nieosiągalny i Time Machine nie ma gdzie pisać.", - "Unknown whether the Google Drive mount is working": - "Nie wiadomo, czy montowanie Google Drive działa", - "Could not read the mount table. That does not mean the Drive is unmounted - it means nobody has checked. Without this answer there is no way to tell whether backups have anywhere to go.": - "Nie udało się odczytać tablicy montowań. To nie znaczy, że Dysk jest odmontowany - znaczy, że nikt tego nie sprawdził. Bez tej odpowiedzi nie da się stwierdzić, czy kopie mają gdzie powstawać.", - "The backup image is attached, but DEAD (errno %@)": - "Obraz backupu jest podpięty, ale MARTWY (errno %@)", - "The image device stopped returning data - Time Machine sees it as a disconnected disk. Fix: cloudmachine-agent attach-image (force-detaches and attaches again).": - "Urządzenie obrazu przestało oddawać dane - Time Machine widzi to jako odłączony dysk. Naprawa: cloudmachine-agent attach-image (odpina na siłę i podpina na nowo).", - "The backup image is not attached": - "Obraz backupu nie jest podpięty", - "Time Machine cannot see the destination %@.": - "Time Machine nie widzi celu %@.", - "Unknown whether the backup image returns data": - "Nie wiadomo, czy obraz backupu oddaje dane", - "The image %@ is listed in the mount table, but the readability probe did not answer within %@ s - that is how a read blocked on a dead FUSE-T mount behaves. This is NOT proof that the image is dead, so do NOT force-detach it: `attach-image` deliberately does nothing in that case, because detaching a live device abandons data waiting to be uploaded. First check whether rclone responds (cloudmachine-agent drive-status) and whether the gdrive-buffer agent is alive.": - "Obraz %@ figuruje w tablicy montowań, ale sonda czytelności nie odpowiedziała w %@ s - tak zachowuje się odczyt zablokowany na martwym montowaniu FUSE-T. To NIE jest dowód, że obraz jest martwy, więc NIE odpinaj go na siłę: `attach-image` świadomie nic wtedy nie robi, bo odpięcie żywego urządzenia porzuca dane czekające na wysyłkę. Sprawdź najpierw, czy rclone odpowiada (cloudmachine-agent drive-status) i czy agent gdrive-buffer żyje.", - "Unknown whether the backup image is attached": - "Nie wiadomo, czy obraz backupu jest podpięty", - "Could not read the mount table, so the state of the image %@ is UNKNOWN. Do not attach it blindly - first check whether `mount` responds at all (with a dead FUSE-T mount it can hang).": - "Nie udało się odczytać tablicy montowań, więc stan obrazu %@ jest NIEZNANY. Nie podpinaj go na oślep - najpierw sprawdź, czy `mount` w ogóle odpowiada (przy martwym montowaniu FUSE-T potrafi wisieć).", "Time Machine does not point to CloudMachine": "Time Machine nie wskazuje na CloudMachine", - "The backup destination was changed or unregistered - backups are not being made.": - "Cel backupu został przestawiony albo wyrejestrowany - kopie nie powstają.", "tmutil is not responding - unknown where the backup goes": "tmutil nie odpowiada - nie wiadomo, gdzie idzie backup", - "Reading the Time Machine destination did not return within %@ s. That is how tmutil behaves when blocked on a dead Google Drive mount. Fix: cloudmachine-agent attach-image, and if that does not help - restart the gdrive-buffer agent.": - "Odczyt celu Time Machine nie wrócił w %@ s. Tak zachowuje się tmutil zablokowany na martwym montowaniu Google Drive. Naprawa: cloudmachine-agent attach-image, a gdy to nie pomoże - restart agenta gdrive-buffer.", "No successful backup for %@": "Brak udanej kopii od %@", "Last COMPLETED backup: %@. The cycle is hourly, so that is %@ missed runs.": @@ -152,32 +106,22 @@ extension L10nPolish { "Niezerowy RESULT w preferencjach Time Machine znaczy, że przebieg się nie udał.", "rclone failed to upload %@ files": "rclone nie wysłał %@ plików", - "These image bands exist only locally. The backup on Google Drive is INCOMPLETE and may not open.": - "Te pasma obrazu istnieją tylko lokalnie. Kopia na Google Drive jest NIEPEŁNA i może się nie otworzyć.", "Buffer full of nothing but unsent data": "Bufor pełny samymi niewysłanymi danymi", "rclone has nothing left to evict from the buffer - the upload cannot keep up or has stalled.": "rclone nie ma już czego usunąć z bufora - wysyłka nie nadąża albo stoi.", "The rclone control interface is not responding": "Interfejs sterujący rclone nie odpowiada", - "Without it there is no way to check whether anything reached the Drive - the buffer guard is blind then.": - "Bez niego nie da się sprawdzić, czy cokolwiek doleciało na Dysk - dozorca bufora jest wtedy ślepy.", "Google Drive is running out of space (%@ GB)": "Kończy się miejsce na Google Drive (%@ GB)", - "Once it runs out, rclone exits with a storageQuotaExceeded error, the mount disappears and backups stop being made. At a growth of ~600 MB per hourly cycle that is about %@ days.": - "Po wyczerpaniu rclone kończy pracę z błędem storageQuotaExceeded, montowanie znika i backupy przestają powstawać. Przy przyroście ~600 MB na cykl godzinowy to około %@ dni.", "The Mac's disk is running out of space (%@ GB)": "Kończy się miejsce na dysku Maca (%@ GB)", - "The upload buffer lives on this disk. When it fills up, the guard pauses Time Machine, and with no space left at all rclone has nowhere to put data waiting to be uploaded.": - "Bufor wysyłki leży na tym dysku. Gdy się zapełni, dozorca wstrzyma Time Machine, a przy całkowitym braku miejsca rclone nie ma gdzie odłożyć danych czekających na wysłanie.", "Cannot read the Time Machine preferences": "Nie da się odczytać preferencji Time Machine", "%@ is unreadable - most often Full Disk Access is missing. Without this file it is UNKNOWN when the last backup was made, so we treat it as a failure, not as the absence of a problem.": "%@ jest nieczytelny - najczęściej brak Pełnego dostępu do dysku. Bez tego pliku NIE WIADOMO, kiedy ostatnio powstała kopia, więc traktujemy to jak awarię, a nie jak brak problemu.", "Cannot measure free space on the Mac's disk": "Nie da się zmierzyć wolnego miejsca na dysku Maca", - "statfs('/System/Volumes/Data') returned an error. The buffer guard will then not pause Time Machine before the disk fills up, because it does not know the number it bases that decision on.": - "statfs('/System/Volumes/Data') zwrócił błąd. Dozorca bufora nie wstrzyma wtedy Time Machine przed zapełnieniem dysku, bo nie zna liczby, na której opiera tę decyzję.", "%@ days": "%@ dni", "CloudMachine: backup is not working": @@ -186,20 +130,6 @@ extension L10nPolish { "nieznany powód", "osascript did not show the notification (permissions or no graphical session)": "osascript nie pokazał powiadomienia (uprawnienia albo brak sesji graficznej)", - "MISSING": - "BRAK", - "UNKNOWN - could not read the mount table": - "NIE WIADOMO - nie udało się odczytać tablicy montowań", - "NOT MEASURED - the buffer guard will not pause Time Machine before the disk fills up": - "NIE ZMIERZONO - dozorca bufora nie wstrzyma Time Machine przed zapełnieniem dysku", - "NOT MEASURED - rclone did not respond, and walking the buffer directory failed": - "NIE ZMIERZONO - rclone nie odpowiedział, a obchód katalogu bufora się nie udał", - "%@ GB of %@G": - "%@ GB z %@G", - "UNKNOWN - the rclone control interface did not respond (the buffer guard will neither pause nor resume Time Machine on this basis)": - "NIE WIADOMO - interfejs sterujący rclone nie odpowiedział (dozorca bufora nie wstrzyma ani nie wznowi Time Machine na tej podstawie)", - "~%@ GB (%@ items)": - "~%@ GB (%@ pozycji)", "UNDELIVERED ALARM: %@": "NIEDORĘCZONY ALARM: %@", "from %@, reason: %@": @@ -214,8 +144,6 @@ extension L10nPolish { "%@ (%@ temu) - CZUJKA MOŻE NIE CHODZIĆ", "NEVER - the watchdog has not recorded a single run": "NIGDY - czujka nie zapisała żadnego przebiegu", - "The image was not detached before reloading the agents - aborting, so as not to lose data waiting in the buffer.\n%@": - "Nie odpięto obrazu przed przeładowaniem agentów - przerywam, żeby nie stracić danych czekających w buforze.\n%@", "Could not find the launchd/ directory with templates.": "Nie znaleziono katalogu launchd/ z szablonami.", "Could not find the compiled cloudmachine-agent binary.": @@ -236,8 +164,6 @@ extension L10nPolish { "Nie załadowano ANI JEDNEGO agenta.", "Only these went in: %@.": "Weszły tylko: %@.", - "Agent installation INCOMPLETE - %@ of %@ did not go in:\n%@\n%@\nEvery missing agent is a function that has silently stopped working (buffer-guard watches the disk, backup-health reports failures). Fix the cause and repeat the installation.": - "Instalacja agentów NIEPEŁNA - nie weszło %@ z %@:\n%@\n%@\nKażdy brakujący agent to funkcja, która przestała działać po cichu (buffer-guard pilnuje dysku, backup-health zgłasza awarie). Napraw powód i powtórz instalację.", "Could not load any launchd agent.": "Nie udało się załadować żadnego agenta launchd.", "Installed agents: %@": diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift index c372e6c..a220604 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift @@ -10,8 +10,6 @@ extension L10nPolish { "Adres: %@", "Already prepared (%@ exists) - nothing to do.": "Już przygotowany (istnieje %@) - nie ma nic do zrobienia.", - "Also install the vm-start and vm-guard launchd agents.": - "Zainstaluj też agentów launchd vm-start i vm-guard.", "Converts Time Machine's new, empty image in the machine to 32 MB bands and moves the bands to Google Drive.": "Przerabia nowy, pusty obraz Time Machine w maszynie na pasma 32 MB i przenosi pasma na Google Drive.", "Copy rclone.conf into the machine even if it already has one.": diff --git a/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift b/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift index 61a780e..6f4f342 100644 --- a/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift +++ b/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift @@ -3,9 +3,9 @@ import Foundation /// Generates `.plist` files with the path to the compiled `cloudmachine-agent` /// binary substituted in and installs them as LaunchAgents (the logged-in /// user's session) - generically installs EVERY `*.plist.template` template -/// found in `launchd/` (currently `verify-watchdog` and `archive-watchdog`). -/// The removed earlier network NFS mount architecture additionally had -/// templates for mount/backup/quota here, which went away with it. +/// found in `launchd/`: `app` (the menu bar), `backup-health` (the watchdog), +/// `vm-start` (boots the machine at login) and `vm-guard` (the machine's +/// upload-buffer guard). public enum LaunchdInstaller { public static var launchAgentsDir: URL { FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/LaunchAgents") @@ -50,39 +50,8 @@ public enum LaunchdInstaller { // app bundle. CMTooling.linkCommandIntoPath() - // Installation reloads the agents, including the one holding the mount. - // Doing that with the image attached pulls the floor out from under it - // mid-way - and detaching is a write that still has to reach the Drive. I - // made this mistake three times in a row, so we no longer rely on - // remembering it. - if BackupImageService.isAttached { - // A dead image (see `ImageProbe`) cannot be detached politely - - // `hdiutil detach` without `-force` refuses, and the installation would - // get stuck on exactly the state it is meant to fix. - // ONLY `.dead`, not `!isUsable`. `.unknown` is not "usable" either, but - // it means "I do not know" - and `detach -force` on a device that may be - // alive abandons writes waiting to be uploaded to the Drive. Since - // 26.09.2026 `.unknown` is REACHABLE here (the readability probe has a - // time limit and, once it is exceeded, returns exactly this state), so - // the difference is no longer theoretical. Without `-force`, - // `hdiutil detach` simply refuses, the installation aborts with a - // message and nobody loses data. - var force = false - if case .dead = await BackupImageService.attachment() { force = true } - CMLogger.log( - "Agent installation: detaching the image first\(force ? " (dead - forcefully)" : "") and waiting for the upload" - ) - let detached = await BackupImageService.detach(force: force) - CMLogger.log("Agent installation: \(detached.message)") - if !detached.succeeded { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The image was not detached before reloading the agents - aborting, so as not to lose data waiting in the buffer.\n%@", - detached.message)) - } - } - + // Reloading touches nothing that holds data: `vm-start` on a running + // machine only says so, the others are one-shot checks. guard let templatesDir = CMPaths.launchdTemplatesDir else { return CMActionResult( succeeded: false, message: L10n.tr("Could not find the launchd/ directory with templates.")) @@ -115,18 +84,8 @@ public enum LaunchdInstaller { try? FileManager.default.createDirectory(at: launchAgentsDir, withIntermediateDirectories: true) - // Migration: an older version installed a separate agent - // "com.renacode.cloudmachine.mount", long since replaced by - // mount-watchdog - remove it if it is still loaded on someone's Mac. - let oldMountPlist = launchAgentsDir.appendingPathComponent( - "com.renacode.cloudmachine.mount.plist") - if FileManager.default.fileExists(atPath: oldMountPlist.path) { - CMLogger.log( - "Removing the obsolete agent com.renacode.cloudmachine.mount (replaced by mount-watchdog)." - ) - _ = try? await ProcessRunner.run("/bin/launchctl", ["unload", oldMountPlist.path]) - try? FileManager.default.removeItem(at: oldMountPlist) - } + // The disk-image layout's agents go first - see `AgentRepair.retired`. + await AgentRepair.removeRetiredAgents() // The interface has to be KILLED before launchd starts it again. // @@ -156,44 +115,6 @@ public enum LaunchdInstaller { templates: templates, into: launchAgentsDir, agentBin: agentBin, logDir: CMPaths.logDir)) } - /// The machine's agents (`launchd/vm/`): `vm-start` and `vm-guard`. - /// - /// A subdirectory on purpose: `install()` installs every template at the - /// top of `launchd/`, and a Mac on the attached-image layout must not get - /// an agent that starts a machine it does not have. These go in only when - /// asked for (`vm-setup --install-agents`). No image detach here: reloading - /// them does not touch the Drive mount. - public static func installVMAgents() async -> CMActionResult { - guard let templatesDir = CMPaths.launchdTemplatesDir?.appendingPathComponent("vm") else { - return CMActionResult( - succeeded: false, message: L10n.tr("Could not find the launchd/ directory with templates.")) - } - guard let resolvedAgentBin = CMPaths.agentBinaryPath else { - return CMActionResult( - succeeded: false, - message: L10n.tr("Could not find the compiled cloudmachine-agent binary.")) - } - let agentBin: URL - do { - agentBin = try stableAgentBinaryPath(resolvedFrom: resolvedAgentBin) - } catch { - return CMActionResult( - succeeded: false, message: L10n.tr("ABORTED: %@", error.localizedDescription)) - } - guard - let templates = try? FileManager.default.contentsOfDirectory( - at: templatesDir, includingPropertiesForKeys: nil) - else { - return CMActionResult( - succeeded: false, - message: L10n.tr("Could not list the templates in %@.", templatesDir.path)) - } - try? FileManager.default.createDirectory(at: launchAgentsDir, withIntermediateDirectories: true) - return installVerdict( - await installAgents( - templates: templates, into: launchAgentsDir, agentBin: agentBin, logDir: CMPaths.logDir)) - } - /// What went in and what did NOT - with a reason, one per agent. /// /// Until 25.09.2026 we collected only `installedLabels`, and failures left no @@ -244,8 +165,8 @@ public enum LaunchdInstaller { } catch { // Previously: `guard ... else { continue }`. A template that could not // be read dropped out of the installation WITHOUT A TRACE - neither in - // the log nor in the result - and `buffer-guard` is the only protection - // of the disk on this machine. + // the log nor in the result - and `vm-guard` is the only protection of + // the machine's disk. let reason = L10n.tr("could not read the template %@", template.lastPathComponent) outcome.failed.append(.init(label: label, reason: reason)) log("NOT installed \(label): \(reason)") @@ -288,7 +209,7 @@ public enum LaunchdInstaller { /// /// ONE successful agent was enough for `succeeded: true` and for the message /// "Installed agents: ...", which listed only the successful ones. The - /// observed result: `buffer-guard` did not load, the installer reported + /// observed result: the buffer guard did not load, the installer reported /// success, the only protection of the disk was not working and nobody knew /// - and nobody who does not know the list by heart sees a name missing from /// it. @@ -305,7 +226,7 @@ public enum LaunchdInstaller { return CMActionResult( succeeded: false, message: L10n.tr( - "Agent installation INCOMPLETE - %@ of %@ did not go in:\n%@\n%@\nEvery missing agent is a function that has silently stopped working (buffer-guard watches the disk, backup-health reports failures). Fix the cause and repeat the installation.", + "Agent installation INCOMPLETE - %@ of %@ did not go in:\n%@\n%@\nEvery missing agent is a function that has silently stopped working (vm-guard watches the machine's buffer, backup-health reports failures). Fix the cause and repeat the installation.", "\(outcome.failed.count)", "\(outcome.failed.count + outcome.installed.count)", list, wentIn)) } diff --git a/mac-app/Sources/CloudMachineCore/MachineBudget.swift b/mac-app/Sources/CloudMachineCore/MachineBudget.swift index df53c5d..56a77e1 100644 --- a/mac-app/Sources/CloudMachineCore/MachineBudget.swift +++ b/mac-app/Sources/CloudMachineCore/MachineBudget.swift @@ -7,9 +7,9 @@ import Foundation /// /// - **Time Machine quota.** Time Machine keeps a destination under its quota /// by deleting the oldest backups. The quota is set below the limit, because -/// the image on Drive is larger than the backup inside it: freed blocks are -/// never returned to the band files (29 Sep 2026: 417 GiB of data in 589 GiB -/// of bands, 71%). Setting it needs root, so the app offers the command. +/// the bands on Drive take more than the backup inside them (29 Sep 2026, +/// with the disk-image layout: 417 GiB of data in 589 GiB of bands, 71%). +/// Setting it needs root, so the app offers the command. /// - **An alarm on the real usage**: the size of this Mac's folder on Drive, /// measured with `rclone size`, compared with the limit. The watchdog warns /// from 90% and reports a failure above 100%. Nothing is deleted by it. @@ -142,7 +142,7 @@ public enum MachineBudget { /// rclone did not answer - never zero, which would read as "empty". @discardableResult public static func measureUsage(now: Date = Date()) async -> Usage? { - let path = "\(DriveBufferService.remoteName):\(DriveBufferService.remotePath)" + let path = VMLayout.remote guard let result = try? await CMTooling.runRclone(["size", "--json", path], timeout: 300), result.succeeded, let bytes = bytes(fromSizeJSON: result.stdout) else { @@ -204,7 +204,7 @@ public enum MachineBudget { BackupHealth.Problem( summary: L10n.tr("This Mac is close to its space limit on Google Drive"), detail: L10n.tr( - "%@ GB of %@ GB used. Time Machine deletes the oldest backups to stay within its quota; check that the quota is set (drive-status).", + "%@ GB of %@ GB used. Time Machine deletes the oldest backups to stay within its quota; check that the quota is set (vm-status).", used, "\(limitGB)"), code: "drive-budget-near") ] @@ -213,7 +213,7 @@ public enum MachineBudget { BackupHealth.Problem( summary: L10n.tr("This Mac is over its space limit on Google Drive"), detail: L10n.tr( - "%@ GB of %@ GB used. Set the Time Machine quota (command in the app window or drive-status), or raise the limit.", + "%@ GB of %@ GB used. Set the Time Machine quota (command in the app window or vm-status), or raise the limit.", used, "\(limitGB)"), code: "drive-budget-exceeded") ] @@ -224,12 +224,12 @@ public enum MachineBudget { let detail: String if let usage { detail = L10n.tr( - "The last measurement (%@ GB) is from %@, %@ ago - rclone size has not answered since. Until it does, nothing watches the %@ GB limit: the 90%% and 100%% alarms stay silent. Check drive-status, or press Measure now in the window.", + "The last measurement (%@ GB) is from %@, %@ ago - rclone size has not answered since. Until it does, nothing watches the %@ GB limit: the 90%% and 100%% alarms stay silent. Check vm-status, or press Measure now in the window.", String(format: "%.0f", usage.gb), BackupHealth.stamp(usage.measuredAt), BackupHealth.formatAge(now.timeIntervalSince(usage.measuredAt)), "\(limitGB)") } else { detail = L10n.tr( - "rclone size has not answered even once. Until it does, nothing watches the %@ GB limit: the 90%% and 100%% alarms stay silent. Check drive-status, or press Measure now in the window.", + "rclone size has not answered even once. Until it does, nothing watches the %@ GB limit: the 90%% and 100%% alarms stay silent. Check vm-status, or press Measure now in the window.", "\(limitGB)") } return BackupHealth.Problem( diff --git a/mac-app/Sources/CloudMachineCore/MountRestart.swift b/mac-app/Sources/CloudMachineCore/MountRestart.swift deleted file mode 100644 index 9d7e645..0000000 --- a/mac-app/Sources/CloudMachineCore/MountRestart.swift +++ /dev/null @@ -1,207 +0,0 @@ -import Foundation - -/// Restarts the Google Drive mount without restarting the Mac - the way out of -/// a mount that predates the private socket (`DriveBufferService -/// .legacyPortOpen`). An upgrade deliberately leaves the mount running, and -/// "prepare-shutdown, then restart the Mac" is a price few pay soon. -/// -/// It is the shutdown path, in the same order, with a start at the end: -/// -/// 1. Refuse while Time Machine is backing up, or when that is not known, or -/// when the mount is not up. -/// 2. Detach the image and wait until everything has reached Drive - the same -/// `detach` as `prepare-shutdown`, whose verdict counts abandoned bands -/// too. If the image was not attached, drain the queue all the same. Any -/// doubt stops here, with the mount untouched. -/// 3. `launchctl kill SIGTERM` the mount. rclone unmounts and exits, and -/// launchd's `KeepAlive` starts `mount-drive` again - the new one, on the -/// socket. Not `bootout`: that unloads the job, and a failed `bootstrap` -/// after it would leave no mount and no agent to bring one back. Not -/// `kickstart -k`, which does not promise SIGTERM. If launchd cannot spawn -/// the new binary (`AgentRepair`), the job is reloaded once it has exited. -/// 4. Wait for the new mount on the private socket, then attach the image. -/// -/// The whole run holds the image lock, so the attach agent cannot attach the -/// image between steps 2 and 3, under a mount about to be stopped. -public enum MountRestart { - - /// Why the restart must not start now; `nil` = it may. - public static func refusal(backupRunning: Bool?, mounted: Bool?) -> String? { - switch backupRunning { - case .some(true): - return L10n.tr( - "Time Machine is backing up right now - the mount was NOT restarted. Try again when the backup has finished." - ) - case .none: - return L10n.tr( - "Could not tell whether Time Machine is backing up (tmutil did not answer) - the mount was NOT restarted." - ) - case .some(false): - break - } - guard mounted == true else { - return L10n.tr( - "The Google Drive mount is not running (or its state is unknown) - there is nothing to restart. The agents start it by themselves." - ) - } - return nil - } - - /// The steps, replaceable so the order and every early exit can be tested - /// without launchd, hdiutil or rclone. - struct Steps { - var backupRunning: () async -> Bool? - var mounted: () -> Bool? - /// Detaches the image (or drains the queue when it is not attached) and - /// says whether everything reached Drive. - var detachAndDrain: () async -> CMActionResult - /// Stops the running mount. `false` = it could not be stopped. - var stopMount: () async -> Bool - /// Waits for the new mount on the private socket. - var waitForNewMount: () async -> Bool - var attach: () async -> CMActionResult - } - - static func run(_ steps: Steps) async -> CMActionResult { - if let reason = refusal(backupRunning: await steps.backupRunning(), mounted: steps.mounted()) { - // `didNotRun`: nothing was touched, so `AttachStall` does not count a - // refusal as the restart its cooldown waits after. - return CMActionResult(succeeded: false, message: reason, didNotRun: true) - } - - let drained = await steps.detachAndDrain() - guard drained.succeeded else { - // The mount was not touched. Put the image back where it was, so the - // refusal does not leave Time Machine without a destination. - let reattached = await steps.attach() - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The mount was NOT restarted: %@\nImage: %@", drained.message, reattached.message)) - } - - guard await steps.stopMount() else { - let reattached = await steps.attach() - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not stop the mount (launchctl refused) - nothing was restarted. Image: %@", - reattached.message)) - } - - guard await steps.waitForNewMount() else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The mount was stopped, but did not come back on the private socket in time. The agents keep trying (backup-health reloads one launchd cannot start). Check: cloudmachine-agent drive-status; if it stays down, restart the Mac." - )) - } - - let attached = await steps.attach() - guard attached.succeeded else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The mount runs on the private socket now, but the image did not attach: %@ The attach agent retries every 15 minutes.", - attached.message)) - } - return CMActionResult( - succeeded: true, - message: L10n.tr( - "The mount was restarted and now listens only on the private socket. The image is attached again." - )) - } - - // MARK: - Live - - static let mountAgent = "gdrive-buffer" - /// How long the new mount may take to come up. After a restart with an - /// empty queue it is seconds; `KeepAlive` waits `ThrottleInterval` (30 s) - /// before a respawn, and a respawn launchd refuses costs one more cycle. - static let newMountTimeout: TimeInterval = 300 - - public static func perform() async -> CMActionResult { - // A restart of the mount ends with the image attached - a preparation for - // a Mac restart (`ShutdownPrep`) no longer applies. - ShutdownPrep.clear() - return await withCMLock(BackupImageService.lockName) { await run(live) } - ?? CMActionResult( - succeeded: false, - message: L10n.tr( - "Another operation on the image is in progress - the mount was NOT restarted. Try again in a moment." - ), - didNotRun: true) - } - - static var live: Steps { - Steps( - backupRunning: { await TimeMachineStatus.runningState() }, - mounted: { DriveBufferService.mountedState() }, - detachAndDrain: { - switch BackupImageService.attachedState() { - case .some(true): - return await BackupImageService.detachLocked() - case .some(false): - CMLogger.log( - BackupImageService.expiryLogLine(await DriveBufferService.expireQueuedUploads())) - return BackupImageService.detachVerdict( - settled: await DriveBufferService.statsWhenIdle(timeout: 600)) - case .none: - return CMActionResult( - succeeded: false, - message: L10n.tr( - "Could not read the mount table - it is UNKNOWN whether the image is attached.")) - } - }, - stopMount: { - CMLogger.log("Restarting the Drive mount: SIGTERM to \(mountAgent)") - let result = try? await ProcessRunner.run( - "/bin/launchctl", - ["kill", "SIGTERM", "gui/\(getuid())/\(AgentRepair.prefix)\(mountAgent)"], timeout: 30) - return result?.succeeded == true - }, - waitForNewMount: { await waitForNewMount() }, - attach: { - let ready = await BufferReadiness.wait( - sleep: { try? await Task.sleep(nanoseconds: UInt64($0 * 1_000_000_000)) }, - probe: { - BufferReadiness.isReady( - mounted: DriveBufferService.isMounted, imageVisible: BackupImageService.exists) - }) - guard ready else { - return CMActionResult( - succeeded: false, - message: L10n.tr( - "The buffer did not come up within %@ min - not attaching the image.", - "\(Int(BufferReadiness.defaultTimeout / 60))")) - } - return await BackupImageService.attachLocked() - }) - } - - /// Polls for the new mount: mounted, and answering on the private socket. - /// Once the old rclone has gone, a job launchd cannot spawn (a replaced - /// app bundle - see `AgentRepair`) is reloaded; the mount is down by then, - /// so the reload costs nothing. - static func waitForNewMount() async -> Bool { - let deadline = Date().addingTimeInterval(newMountTimeout) - var repaired = false - while Date() < deadline { - try? await Task.sleep(nanoseconds: 2_000_000_000) - if DriveBufferService.mountedState() == true, - DriveBufferService.socketAnswers(DriveBufferService.rcSocket.path) - { - CMLogger.log("Drive mount is back, on the private socket") - return true - } - if !repaired, let output = await AgentRepair.printOutput(label: mountAgent), - AgentRepair.cannotStart(printOutput: output) - { - repaired = true - CMLogger.log("\(mountAgent) cannot start after the restart - reloading it") - await AgentRepair.reload(mountAgent) - } - } - return false - } -} diff --git a/mac-app/Sources/CloudMachineCore/OpenPortAlert.swift b/mac-app/Sources/CloudMachineCore/OpenPortAlert.swift deleted file mode 100644 index 7aea864..0000000 --- a/mac-app/Sources/CloudMachineCore/OpenPortAlert.swift +++ /dev/null @@ -1,89 +0,0 @@ -import Foundation - -/// Tells the user, by a system notification, that the Drive mount still -/// takes commands on the open TCP port (see -/// `DriveBufferService.legacyPortOpen`). -/// -/// Not a `HealthAlert` problem, on purpose. Backups work in this state, so -/// "CloudMachine: backup is not working" would be false, `backup-health` -/// would exit 1 for a working cycle, and the 12-hour reminder would turn a -/// state that lasts until the next restart of the Mac into noise. -/// -/// Instead: once at once, then with a growing gap - 1, 2, 4 and then every 7 -/// days. A notice that never comes back is forgotten after one dismissal; one -/// that comes back twice a day is muted. A notification that was not -/// delivered is retried at the next watchdog run, as in `HealthAlert`. -public enum OpenPortAlert { - - public static var stateFile: URL { - CMPaths.appSupportDir.appendingPathComponent("open-port-alert.json") - } - - struct State: Codable, Equatable { - /// Notifications delivered so far for this episode. - var delivered: Int - var lastAttemptAt: Date - var lastDelivered: Bool - } - - /// The gap after the `count`-th delivered notification. - static func gap(afterDelivered count: Int) -> TimeInterval { - let days = min(7.0, pow(2.0, Double(max(0, count - 1)))) - return days * 86_400 - } - - static func shouldNotify(_ state: State?, now: Date) -> Bool { - guard let state else { return true } - if !state.lastDelivered { return true } - return now.timeIntervalSince(state.lastAttemptAt) >= gap(afterDelivered: state.delivered) - } - - /// Reports the open port if it is time to. `open == false` ends the - /// episode: the state is removed, so a later one is reported at once. - /// Returns `true` when a notification was delivered. - /// - /// `stateFile`, `deliver` and `log` are replaceable for the reason given at - /// `HealthAlert.report`: tests must not write to the user's directory, show - /// notifications or add made-up lines to the real log. - @discardableResult - public static func report( - open: Bool, - now: Date = Date(), - stateFile: URL = OpenPortAlert.stateFile, - deliver: @Sendable (String, String) async -> Bool = { - await HealthAlert.notify(title: $0, message: $1) - }, - log: @Sendable (String) -> Void = { CMLogger.log($0) } - ) async -> Bool { - guard open else { - try? FileManager.default.removeItem(at: stateFile) - return false - } - let state = load(stateFile) - guard shouldNotify(state, now: now) else { return false } - - log( - "SECURITY: rclone remote control is OPEN on \(DriveBufferService.legacyRCAddress) without a password - the mount predates the private socket. Restart the mount (window or restart-mount), or prepare-shutdown and restart the Mac." - ) - let delivered = await deliver( - L10n.tr("CloudMachine: Drive remote control is open"), - L10n.tr( - "Any web page can send it commands, including deleting the backup. Open CloudMachine and restart the mount, or run prepare-shutdown and restart the Mac." - )) - if !delivered { - log("FAILED to show the open-port notification; will retry at the next check.") - } - let next = State( - delivered: (state?.delivered ?? 0) + (delivered ? 1 : 0), lastAttemptAt: now, - lastDelivered: delivered) - if let data = try? JSONEncoder().encode(next) { - try? data.write(to: stateFile, options: .atomic) - } - return delivered - } - - static func load(_ file: URL) -> State? { - guard let data = try? Data(contentsOf: file) else { return nil } - return try? JSONDecoder().decode(State.self, from: data) - } -} diff --git a/mac-app/Sources/CloudMachineCore/OrphanMount.swift b/mac-app/Sources/CloudMachineCore/OrphanMount.swift deleted file mode 100644 index 77040e5..0000000 --- a/mac-app/Sources/CloudMachineCore/OrphanMount.swift +++ /dev/null @@ -1,133 +0,0 @@ -import Foundation - -/// The way out of a Drive mount whose rclone is gone. -/// -/// 25 Oct 2026 00:02 and 27 Oct 2026 00:19, both during a backup: rclone -/// logged "Mount failed" / "CRITICAL: Fatal error: failed to umount FUSE fs" -/// and quit, but the FUSE-T NFS mount stayed in the mount table. From then on -/// every `stat` of the mount (Time Machine, `tmutil`, Xcode, our own agents) -/// hung, and nothing brought rclone back: `mount-drive` read the mount table, -/// saw the mount and said "Already mounted". On 27.10 that lasted 8 h, until -/// the owner restarted the Mac. On 25.10 `umount -f` of the mount point by -/// hand was enough - the agent brought rclone up 30 s later. -/// -/// So the remedy is that `umount -f`, done by the agents, when the mount is -/// listed and no live rclone holds it. Everything here avoids the file -/// system: the mount table is read with `MNT_NOWAIT`, the process table with -/// `pgrep`/`ps`, so a dead mount cannot hang the check that looks for it. -/// -/// Two callers: -/// - `buffer-guard` runs `watch()` in its own task, beside its main loop, so -/// a probe of that loop hanging on the dead mount cannot hold it up. It -/// waits `grace` before acting - the normal restart of rclone by launchd -/// also has a moment with the mount listed and no rclone yet. -/// - `mount-drive`, at start-up. launchd starts it only once the previous -/// rclone has exited, so a listed mount with no rclone is an orphan. -public enum OrphanMount { - - /// Whether an rclone holding our mount is alive. - public enum RcloneState: Equatable { - case running - /// No such process, or only ones on their way out (`ps` state `E`/`Z`). - case gone - /// The process table could not be read - never a reason to unmount. - case unknown - } - - public static let grace: TimeInterval = 60 - public static let checkInterval: TimeInterval = 30 - static let unmountTimeout: TimeInterval = 60 - - /// Since when the mount has been seen orphaned; `nil` = it is not. - public static func orphanedSince( - _ previous: TimeInterval?, mounted: Bool?, rclone: RcloneState, now: TimeInterval - ) -> TimeInterval? { - // Only a sure "mounted" and a sure "gone" make an orphan; any doubt - // resets the clock rather than counting towards an unmount. - guard mounted == true, rclone == .gone else { return nil } - return previous ?? now - } - - /// Whether the orphan has lasted long enough to unmount it. - public static func shouldUnmount(orphanedSince: TimeInterval?, now: TimeInterval) -> Bool { - guard let orphanedSince else { return false } - return now - orphanedSince >= grace - } - - /// `ps -o stat=` lines -> whether any of them is a process still running. - /// `E` = trying to exit, `Z` = zombie: neither serves the mount any more. - static func anyAlive(psStates: String) -> Bool { - psStates.split(whereSeparator: \.isNewline).contains { line in - let state = line.trimmingCharacters(in: .whitespaces) - return !state.isEmpty && !state.contains("E") && !state.contains("Z") - } - } - - /// Our rclone, by the path it runs from and the `mount` subcommand. - static var rclonePattern: String { - NSRegularExpression.escapedPattern(for: CMTooling.managedRclonePath.path) + " mount " - } - - public static func rcloneState() async -> RcloneState { - guard - let found = try? await ProcessRunner.run( - "/usr/bin/pgrep", ["-f", rclonePattern], timeout: 15) - else { return .unknown } - switch found.exitCode { - case 1: return .gone - case 0: break - default: return .unknown - } - let pids = found.stdout.split(whereSeparator: \.isWhitespace).map(String.init) - guard !pids.isEmpty, - let states = try? await ProcessRunner.run( - "/bin/ps", ["-o", "stat=", "-p", pids.joined(separator: ",")], timeout: 15) - else { return .unknown } - // `ps` exits 1 when the processes have gone in the meantime. - if !states.succeeded, states.stdout.isEmpty { return .gone } - return anyAlive(psStates: states.stdout) ? .running : .gone - } - - /// `umount -f` of the mount point; `true` once it has left the mount table. - @discardableResult - public static func forceUnmount() async -> Bool { - let path = DriveBufferService.mountPoint.path - CMLogger.log("Orphaned Drive mount (no rclone holds it) - umount -f \(path)") - let result = try? await ProcessRunner.run( - "/sbin/umount", ["-f", path], timeout: unmountTimeout) - let gone = DriveBufferService.mountedState() == false - CMLogger.log( - gone - ? "Orphaned Drive mount removed" - : "Orphaned Drive mount NOT removed (umount -f: \(result.map { "exit code \($0.exitCode) \($0.stderr)" } ?? "no answer"))" - ) - return gone - } - - /// `mount-drive`'s check: unmount at once, no grace (see the type comment). - public static func clearBeforeMount() async { - guard DriveBufferService.mountedState() == true, await rcloneState() == .gone else { return } - await forceUnmount() - } - - /// `buffer-guard`'s watch, forever. - public static func watch() async { - var since: TimeInterval? - while true { - try? await Task.sleep(nanoseconds: UInt64(checkInterval * 1_000_000_000)) - let mounted = DriveBufferService.mountedState() - let rclone = mounted == true ? await rcloneState() : .unknown - let now = Date().timeIntervalSince1970 - since = orphanedSince(since, mounted: mounted, rclone: rclone, now: now) - guard shouldUnmount(orphanedSince: since, now: now) else { continue } - since = nil - guard await forceUnmount() else { continue } - // launchd starts `mount-drive` again by itself once rclone has exited; - // a kickstart without `-k` only covers a job that is not running. - _ = try? await ProcessRunner.run( - "/bin/launchctl", - ["kickstart", "gui/\(getuid())/\(AgentRepair.prefix)\(MountRestart.mountAgent)"], - timeout: 30) - } - } -} diff --git a/mac-app/Sources/CloudMachineCore/ProcessRunner.swift b/mac-app/Sources/CloudMachineCore/ProcessRunner.swift index dec93bf..269ac28 100644 --- a/mac-app/Sources/CloudMachineCore/ProcessRunner.swift +++ b/mac-app/Sources/CloudMachineCore/ProcessRunner.swift @@ -40,10 +40,8 @@ public enum ProcessRunnerError: LocalizedError { /// still fires later, it MUST do nothing instead of causing a fatal error with /// a second `continuation.resume`. /// -/// Internal rather than private: since 26.09.2026 `ImageProbe` gives up on a -/// deadline in exactly the same way (a probe stuck in the kernel may answer -/// later or never), so both paths must share the same, proven semantics of -/// "whoever was first resumes". +/// Internal rather than private, so its "whoever was first resumes" semantics +/// can be tested on their own. final class ContinuationGuard: @unchecked Sendable { private let lock = NSLock() private var done = false diff --git a/mac-app/Sources/CloudMachineCore/RcloneInstaller.swift b/mac-app/Sources/CloudMachineCore/RcloneInstaller.swift index 2d2fa51..ea80cf5 100644 --- a/mac-app/Sources/CloudMachineCore/RcloneInstaller.swift +++ b/mac-app/Sources/CloudMachineCore/RcloneInstaller.swift @@ -2,14 +2,10 @@ import Foundation /// Downloads the official rclone binary - port of `gdrive/install-rclone.sh`. /// -/// It exists because rclone from Homebrew is built without FUSE support and, -/// when asked to mount, refuses outright: -/// -/// rclone mount is not supported on MacOS when rclone is installed via Homebrew -/// -/// We install it alongside, in our own directory -/// (`CMTooling.managedRclonePath`), without touching the Homebrew installation -/// - the remaining, non-mounting code paths can keep using it. +/// Installed in our own directory (`CMTooling.managedRclonePath`), without +/// touching Homebrew: the Mac's rclone signs in to Google Drive, writes the +/// `rclone.conf` that `vm-setup` copies into the machine, and measures usage. +/// The machine has its own rclone (`vm/cmsmb.yaml`). public enum RcloneInstaller { private static let versionURL = "https://downloads.rclone.org/version.txt" diff --git a/mac-app/Sources/CloudMachineCore/RemoteConfigurer.swift b/mac-app/Sources/CloudMachineCore/RemoteConfigurer.swift index 3c09a7d..3c843c8 100644 --- a/mac-app/Sources/CloudMachineCore/RemoteConfigurer.swift +++ b/mac-app/Sources/CloudMachineCore/RemoteConfigurer.swift @@ -22,34 +22,41 @@ public enum RemoteConfigurer { /// folder. /// /// "Not listed" is an answer only when rclone READ a configuration. Without - /// `rclone.conf` it lists nothing and exits 0 - and on a Mac whose mount has - /// already run (`InstallationTraces`), a lost file is far likelier than a - /// remote that never was. That case is `nil` too: answered "no remote", it - /// gave such a Mac a new, empty Drive folder for good (second audit, S4). + /// `rclone.conf` it lists nothing and exits 0 - and on a Mac that has + /// already set up the machine (its SMB password file exists), a lost file is + /// far likelier than a remote that never was. That case is `nil` too: + /// answered "no remote", it would give such a Mac a new, empty Drive folder + /// for good (second audit, S4). public static func isConfigured(remoteName: String) async -> Bool? { let listed = remoteListed( remoteName, in: try? await CMTooling.runRclone(["listremotes"], timeout: 30)) guard listed == false else { return listed } let configFile = configFileExists( in: try? await CMTooling.runRclone(["config", "file"], timeout: 30)) - let folders = InstallationTraces.cachedFolders() - let answer = absentAnswer(configFileExists: configFile, cachedFolders: folders) + let machineSetUp = machineWasSetUp() + let answer = absentAnswer(configFileExists: configFile, machineSetUp: machineSetUp) if answer == nil { CMLogger.log( "rclone lists no '\(remoteName)' remote, but its configuration file " + (configFile == false ? "does not exist" : "could not be checked") - + " while this Mac has cached Drive folders (\(folders.joined(separator: ", "))) -" - + " treating it as UNKNOWN, not as 'no remote'") + + " while this Mac has already set up the machine - treating it as UNKNOWN, not as 'no remote'" + ) } return answer } + /// The trace a backup leaves on this Mac: `vm-setup` writes the machine's + /// SMB password file only after rclone.conf has been copied into it. + static func machineWasSetUp() -> Bool { + FileManager.default.fileExists(atPath: VMLayout.passwordFile.path) + } + /// Pure: what "the remote is not listed" means, given whether rclone's - /// configuration file exists (`nil` = could not tell) and which Drive - /// folders a mount has already cached here. - static func absentAnswer(configFileExists: Bool?, cachedFolders: [String]) -> Bool? { + /// configuration file exists (`nil` = could not tell) and whether this Mac + /// has already set up the machine. + static func absentAnswer(configFileExists: Bool?, machineSetUp: Bool) -> Bool? { if configFileExists == true { return false } - return cachedFolders.isEmpty ? false : nil + return machineSetUp ? nil : false } /// Reads `rclone config file`. rclone says outright when the file is @@ -107,39 +114,29 @@ public enum RemoteConfigurer { } /// Connects to Google Drive and prepares EXACTLY the remote and folder that - /// the mount uses. - /// - /// Previously the names were taken from `machines.json` (`remote_name`, - /// `gdrive-cloudmachine` by default, plus a folder with the machine key), - /// while the mount runs on the constants `DriveBufferService.remoteName` / - /// `.remotePath` (`gdrive:CloudMachine/mac-studio`). The documented - /// installation path - `configure-remote`, then `create-image` - therefore - /// ended with a remote nobody ever uses, and a "Drive is not mounted" error - /// at the next step. The working installation on this machine has a - /// `[gdrive]` put together by hand; it could not be reproduced from the repo - /// alone. + /// the machine uses (`DriveFolder.remoteName`, `VMLayout.remote`). + /// `vm-setup` copies the resulting `rclone.conf` into the machine. /// /// `config` and `machineKey` stay in the signature because the GUI and the - /// CLI have them, but from now on the remote's name is decided by the same - /// constant that builds the mount command. One source of truth or none. + /// CLI have them; the remote's name is the constant every command uses. /// /// `acceptScopeChange` is the second, explicit "yes" that replacing an /// existing remote needs - see `replacementGate`. Even with it, the new - /// token is checked against the existing image before anything is written + /// token is checked against the existing backup before anything is written /// (`replacementVerdict`). @discardableResult public static func connect( config: MachinesConfig, machineKey: String, replaceExisting: Bool = false, acceptScopeChange: Bool = false, folder: String? = nil ) async -> CMActionResult { - let remoteName = DriveBufferService.remoteName + let remoteName = DriveFolder.remoteName // We ABORT rather than warn. `rclone config create` overwrites an entry // with the same name without asking, and with it the token, `client_id` // and `scope` of the working installation. A new credential with the // `drive.file` scope sees only files created by ITSELF - the existing // backup image, created by the previous credential, then becomes - // invisible and the mount stops finding it. The backup is intact, but + // invisible to every rclone using that token. The backup is intact, but // inaccessible, which in practice means the same thing. let configured = await isConfigured(remoteName: remoteName) // "Could not check" stops here as firmly as "it exists": both mean we do @@ -149,13 +146,12 @@ public enum RemoteConfigurer { var message = L10n.tr( "Could not check whether remote '%@' already exists (rclone listremotes did not answer or failed), so nothing was changed. Run `%@ listremotes` to see the error, then try again.", remoteName, CMTooling.managedRclonePath.path) - let folders = InstallationTraces.cachedFolders() - if !folders.isEmpty { + if machineWasSetUp() { message += "\n" + L10n.tr( - "Or rclone.conf is missing: this Mac has already backed up to Google Drive (folders %@), so it is not connected anew blindly. Restore the file from a backup. To connect anyway, create an empty file at the path `%@ config file` prints and run again - the existing folder is kept.", - folders.joined(separator: ", "), CMTooling.managedRclonePath.path) + "Or rclone.conf is missing: this Mac has already set up the CloudMachine machine, so it is not connected anew blindly. Restore the file from a backup (the machine keeps its own copy in %@). To connect anyway, create an empty file at the path `%@ config file` prints and run again - the existing folder is kept.", + VMLayout.Guest.rcloneConfig, CMTooling.managedRclonePath.path) } return CMActionResult(succeeded: false, message: message) } @@ -176,22 +172,22 @@ public enum RemoteConfigurer { case .success: break } - let remotePath = "\(remoteName):\(DriveBufferService.remotePath)" + let remotePath = VMLayout.remote // What the CURRENT connection sees, asked before it is replaced: the new // token will be held against it. No answer stops here - the check below // would have nothing to compare with. - var imageBefore: BackupImageService.RemotePresence? + var backupBefore: RemotePresence? if configured { - let presence = await BackupImageService.remoteImagePresence() + let presence = await backupPresence() if case .unknown(let why) = presence { return CMActionResult( succeeded: false, message: L10n.tr( - "Could not check with the current connection whether the backup image is on Google Drive (%@), so remote '%@' was NOT replaced: without that answer the new sign-in cannot be checked against it.", + "Could not check with the current connection whether the backup is on Google Drive (%@), so remote '%@' was NOT replaced: without that answer the new sign-in cannot be checked against it.", why, remoteName)) } - imageBefore = presence + backupBefore = presence } // Own OAuth credentials from the Keychain. On rclone's shared `client_id` @@ -223,19 +219,13 @@ public enum RemoteConfigurer { // what the working installation has - grants reading, changing and // DELETING the entire contents of the Google account. That is an // incomparably broader permission than a directory with the bands of one - // image needs, all the more so because the mount runs with + // backup needs, all the more so because the machine's rclone runs with // `--drive-use-trash=false`, so deletion has no trash from which anything // could be undone. // - // Binary: THE SAME one the rest of the system uses - // (`~/.cloudmachine/bin/rclone`), not `/usr/bin/env rclone`. Until - // 23.09.2026 `connect` went through `env`, i.e. to rclone from Homebrew - - // while `isConfigured` in this same file and the whole mount go through - // `CMTooling.runRclone`. There were two consequences, both silent: without - // Homebrew the documented installation path (`install-rclone`, then - // `configure-remote`) ended with exit code 127 and a message without a - // cause, and WITH Homebrew the configuration was written by a DIFFERENT - // binary than the one the system then uses. + // Binary: THE SAME one the rest of the Mac side uses + // (`~/.cloudmachine/bin/rclone`), not `/usr/bin/env rclone`, so the + // configuration is written by the binary that reads it afterwards. // // `authorize` is called via `ProcessRunner.run` directly on the managed // binary's path, because `CMTooling.runRclone` does not accept `env`, @@ -259,11 +249,11 @@ public enum RemoteConfigurer { succeeded: false, message: L10n.tr("Could not read the token from the output of rclone authorize.")) } - if let imageBefore { - let imageAfter = await imagePresence( + if let backupBefore { + let backupAfter = await backupPresence( token: token, clientID: clientID, clientSecret: clientSecret) if let refusal = replacementVerdict( - before: imageBefore, after: imageAfter, remoteName: remoteName) + before: backupBefore, after: backupAfter, remoteName: remoteName) { return refusal } @@ -278,10 +268,6 @@ public enum RemoteConfigurer { createArgs += ["client_id=\(clientID)", "client_secret=\(clientSecret)"] } createArgs.append("token=\(token)") - // Again the same binary as the mount: if `config create` went through - // Homebrew, it would write the entry into a configuration that the binary - // used by the system may not read - and then `isConfigured` says "there is - // no remote" right after a successful "connected". let createResult = try? await CMTooling.runRclone(createArgs, timeout: 60) guard createResult?.succeeded == true else { return CMActionResult(succeeded: false, message: L10n.tr("rclone config create failed.")) @@ -290,12 +276,11 @@ public enum RemoteConfigurer { guard mkdirResult?.succeeded == true else { // We RETURN AN ERROR, not "success": the remote exists, but without // this folder we have no confirmation that writing to this account - // actually works - and the next installation step (`create-image`) - // assumes it does. + // actually works - and the next step (`vm-setup`) assumes it does. return CMActionResult( succeeded: false, message: L10n.tr( - "Connected to Google Drive, but could not create the folder '%@' - without it the mount will not start. Check the account permissions and try again.", + "Connected to Google Drive, but could not create the folder '%@' - without it the machine has nowhere to keep the backup. Check the account permissions and try again.", remotePath)) } return CMActionResult( @@ -313,8 +298,7 @@ public enum RemoteConfigurer { /// `--replace-existing`, is easy to reach for - until 11.10.2026 the panel /// itself suggested it after every change of the Google credentials. On a /// Mac like this project's own (`scope = drive`, its own `client_id`), the - /// new sign-in gets `drive.file` and does not see the existing image: the - /// mount shows an empty folder and `create-image` sees nothing to protect. + /// new sign-in gets `drive.file` and does not see the existing backup. /// The second flag is there so that the user has read that sentence. static func replacementGate( configured: Bool, replaceExisting: Bool, acceptScopeChange: Bool, remoteName: String @@ -331,23 +315,31 @@ public enum RemoteConfigurer { return CMActionResult( succeeded: false, message: L10n.tr( - "Remote '%@' was NOT replaced. The new sign-in gets the 'drive.file' scope, which sees only files created under its own client_id. The existing backup was most likely created by the current connection and would become invisible: the mount would show an empty folder, and create-image could start a second, empty backup next to it. Before anything is written, the new token is checked against the existing image, and the replacement is abandoned if it cannot see it. To go ahead, run again with --replace-existing --accept-scope-change.", + "Remote '%@' was NOT replaced. The new sign-in gets the 'drive.file' scope, which sees only files created under its own client_id. The existing backup was most likely created by the current connection and would become invisible to it. Before anything is written, the new token is checked against the existing backup, and the replacement is abandoned if it cannot see it. The machine keeps its own copy of the configuration until vm-setup --replace-rclone-config. To go ahead, run again with --replace-existing --accept-scope-change.", remoteName)) } return nil } + /// What a token sees of this Mac's backup folder. + public enum RemotePresence: Equatable { + case present + case absent + /// Unknown. `why` goes into the message, so that the user knows what + /// exactly was missing. + case unknown(String) + } + /// Whether the new token may replace the remote, given what the old one and /// the new one see. `nil` = it may. /// - /// Only an image that WAS visible can be lost: with none before, there is + /// Only a backup that WAS visible can be lost: with none before, there is /// nothing for the new token to miss. When there was one, the new token /// has to see it too - "absent" and "no answer" both stop the replacement, /// because writing the token is the step that cannot be taken back (the /// old token is gone from `rclone.conf` with it). static func replacementVerdict( - before: BackupImageService.RemotePresence, after: BackupImageService.RemotePresence, - remoteName: String + before: RemotePresence, after: RemotePresence, remoteName: String ) -> CMActionResult? { guard before == .present else { return nil } let why: String @@ -359,34 +351,61 @@ public enum RemoteConfigurer { return CMActionResult( succeeded: false, message: L10n.tr( - "The new sign-in does NOT see the existing backup image (%@), so remote '%@' was NOT replaced and rclone.conf is unchanged. With this token the mount would show an empty folder. Keep the current connection, or sign in with the credential (client_id) that created the image.", + "The new sign-in does NOT see the existing backup (%@), so remote '%@' was NOT replaced and rclone.conf is unchanged. Keep the current connection, or sign in with the credential (client_id) that created the backup.", why, remoteName)) } - /// Whether `token` sees this Mac's image - the same `lsf` as - /// `BackupImageService.remoteImagePresence`, on a remote defined only for - /// this call (`:drive:`), so nothing is written to `rclone.conf`. The token - /// and the keys go through the environment, not the command line, for the - /// reason given at `authorize`. - static func imagePresence(token: String, clientID: String?, clientSecret: String?) async - -> BackupImageService.RemotePresence + /// `bands/` in the machine's Drive folder, as the current connection sees it. + static func backupPresence() async -> RemotePresence { + guard + let result = try? await CMTooling.runRclone( + ["lsf", "--dirs-only", VMLayout.remote], timeout: 120) + else { return .unknown(L10n.tr("rclone did not answer")) } + return classifyRemoteListing( + succeeded: result.succeeded, stdout: result.stdout, stderr: result.stderr) + } + + /// The same `lsf` with `token`, on a remote defined only for this call + /// (`:drive:`), so nothing is written to `rclone.conf`. The token and the + /// keys go through the environment, not the command line, for the reason + /// given at `authorize`. + static func backupPresence(token: String, clientID: String?, clientSecret: String?) async + -> RemotePresence { var env = ["RCLONE_DRIVE_TOKEN": token, "RCLONE_DRIVE_SCOPE": "drive.file"] if let clientID, let clientSecret { env["RCLONE_DRIVE_CLIENT_ID"] = clientID env["RCLONE_DRIVE_CLIENT_SECRET"] = clientSecret } + let path = VMLayout.remote.split(separator: ":", maxSplits: 1).last.map(String.init) ?? "" guard let result = try? await ProcessRunner.run( - CMTooling.managedRclonePath.path, - ["lsf", "--dirs-only", ":drive:\(DriveBufferService.remotePath)"], env: env, timeout: 120) + CMTooling.managedRclonePath.path, ["lsf", "--dirs-only", ":drive:\(path)"], env: env, + timeout: 120) else { return .unknown(L10n.tr("rclone did not answer")) } - return BackupImageService.classifyRemoteListing( + return classifyRemoteListing( succeeded: result.succeeded, stdout: result.stdout, stderr: result.stderr) } + /// `lsf --dirs-only` of the folder -> whether the backup's `bands/` is there. + /// A missing folder is an answer ("absent"), any other error is not. + static func classifyRemoteListing(succeeded: Bool, stdout: String, stderr: String) + -> RemotePresence + { + if succeeded { + let present = stdout.split(whereSeparator: \.isNewline).contains { + $0.trimmingCharacters(in: .whitespaces) == "bands/" + } + return present ? .present : .absent + } + if stderr.lowercased().contains("directory not found") { return .absent } + let reason = stderr.trimmingCharacters(in: .whitespacesAndNewlines) + return .unknown( + reason.isEmpty ? L10n.tr("rclone lsf ended with an error") : String(reason.suffix(200))) + } + // MARK: - Saving credentials /// What the panel says after the Google credentials are saved. diff --git a/mac-app/Sources/CloudMachineCore/ShutdownPrep.swift b/mac-app/Sources/CloudMachineCore/ShutdownPrep.swift deleted file mode 100644 index 6490666..0000000 --- a/mac-app/Sources/CloudMachineCore/ShutdownPrep.swift +++ /dev/null @@ -1,123 +0,0 @@ -import Foundation - -/// Preparing the Mac for a restart: pause the backup, detach the image, wait -/// until everything has reached Drive - the `prepare-shutdown` command and the -/// "Prepare for restart" button in the menu bar. -/// -/// The preparation also leaves a marker, because the `gdrive-attach` agent -/// runs every 900 s and attaches whatever image it finds detached. Without the -/// marker it undid the preparation by itself: Time Machine got its destination -/// back, wrote into the buffer again, and the restart a few minutes later left -/// those writes local only - the very backlog that the preparation is there -/// to prevent (`UploadDrain`, 01.10.2026). -/// -/// The marker holds the boot time and the time of the preparation. It blocks -/// the agent only in the same boot session (after the restart the agents bring -/// the image up as usual) and only for `window`: a preparation without a -/// restart must not leave Time Machine without a destination for good. An -/// explicit attach from the window clears it. -public enum ShutdownPrep { - - /// 30 min: enough to close programs and restart, short enough that a - /// forgotten preparation costs at most two missed hourly backups. - public static let window: TimeInterval = 1800 - - public static var markerURL: URL { - CMPaths.appSupportDir.appendingPathComponent("shutdown-prepared") - } - - public struct Marker: Equatable { - public var bootTime: TimeInterval - public var preparedAt: TimeInterval - - public var text: String { "\(Int(bootTime)) \(Int(preparedAt))" } - - public init(bootTime: TimeInterval, preparedAt: TimeInterval) { - self.bootTime = bootTime - self.preparedAt = preparedAt - } - - public init?(text: String) { - let parts = text.split(whereSeparator: \.isWhitespace).compactMap { TimeInterval($0) } - guard parts.count == 2 else { return nil } - self.init(bootTime: parts[0], preparedAt: parts[1]) - } - } - - /// Whether the marker still holds the agent back. The boot time is compared - /// with a second of slack: `kern.boottime` is read with microseconds and - /// stored in whole seconds. - public static func isActive(_ marker: Marker?, bootTime: TimeInterval?, now: TimeInterval) -> Bool - { - guard let marker, let bootTime else { return false } - guard abs(marker.bootTime - bootTime) <= 1 else { return false } - let age = now - marker.preparedAt - return age >= 0 && age < window - } - - /// The marker as it stands on disk, `nil` when there is none (or it is unreadable). - public static func readMarker() -> Marker? { - guard let text = try? String(contentsOf: markerURL, encoding: .utf8) else { return nil } - return Marker(text: text) - } - - public static func holdsAttach(now: Date = Date()) -> Bool { - isActive(readMarker(), bootTime: bootTime(), now: now.timeIntervalSince1970) - } - - public static func clear() { - try? FileManager.default.removeItem(at: markerURL) - } - - /// When the Mac was started (`kern.boottime`), seconds since 1970. - public static func bootTime() -> TimeInterval? { - var tv = timeval() - var size = MemoryLayout.stride - var mib: [Int32] = [CTL_KERN, KERN_BOOTTIME] - guard sysctl(&mib, 2, &tv, &size, nil, 0) == 0, tv.tv_sec > 0 else { return nil } - return TimeInterval(tv.tv_sec) + TimeInterval(tv.tv_usec) / 1_000_000 - } - - /// The whole preparation. `progress` gets the steps as they start (the CLI - /// prints them; the app shows its own busy label). - public static func perform(progress: (String) -> Void = { _ in }) async -> CMActionResult { - // The order is not arbitrary. First Time Machine stops adding new writes, - // only then do we detach the image - otherwise the detach would fight with - // the running backup. - if await TimeMachineStatus.isRunning() { - progress(L10n.tr("Pausing the backup...")) - _ = try? await ProcessRunner.run("/usr/bin/tmutil", ["stopbackup"], timeout: 120) - try? await Task.sleep(nanoseconds: 3_000_000_000) - } - - progress(L10n.tr("Detaching the image and waiting for the upload...")) - let detached = await BackupImageService.detach() - guard detached.succeeded else { - return CMActionResult( - succeeded: false, - message: detached.message + "\n\n" - + L10n.tr("Do NOT restart yet - the buffer holds data that has not reached Drive.")) - } - - // Written only after a successful detach: a marker after a failure would - // keep the agent from repairing a half-detached state. - var written = false - if let boot = bootTime() { - let marker = Marker(bootTime: boot, preparedAt: Date().timeIntervalSince1970) - try? FileManager.default.createDirectory( - at: markerURL.deletingLastPathComponent(), withIntermediateDirectories: true) - written = (try? marker.text.write(to: markerURL, atomically: true, encoding: .utf8)) != nil - } - let minutes = "\(Int(window / 60))" - return CMActionResult( - succeeded: true, - message: detached.message + "\n\n" - + (written - ? L10n.tr( - "Safe to restart. The image stays detached for %@ min; after startup the agents will bring up the buffer and attach the image themselves.", - minutes) - : L10n.tr( - "Safe to restart now - but the image could not be held detached, so the agent may attach it again within 15 min. Restart soon. After startup the agents will bring up the buffer and attach the image themselves." - ))) - } -} diff --git a/mac-app/Sources/CloudMachineCore/StatusLines.swift b/mac-app/Sources/CloudMachineCore/StatusLines.swift index 2a3e9c1..4ef267f 100644 --- a/mac-app/Sources/CloudMachineCore/StatusLines.swift +++ b/mac-app/Sources/CloudMachineCore/StatusLines.swift @@ -1,105 +1,11 @@ import Foundation -/// Building the `drive-status` lines. Pure functions, because the status line -/// is what a person READS when asking "is the backup working" - and until now -/// it could not be tested, because it was produced in a `print` inside a CLI -/// command. -/// -/// These lines have already broken twice in the same way: by turning "I do -/// not know" into some value. Once by substituting zeros for rclone not -/// answering (hence `UploadState.queueUnknown`), once via `Optional(427)` -/// after `BufferGuardService.freeGB()` rightly stopped pretending that a -/// missing measurement is zero. That is why each of the functions below has an -/// explicit branch for missing data. +/// Status lines shared by `vm-status` and the app. Pure functions, because +/// the status line is what a person READS when asking "is the backup working" +/// - and it has broken before by turning "I do not know" into some value. +/// Each function below has an explicit branch for missing data. public enum StatusLines { - /// The "Drive mount" line. - /// - /// The third state is separate for the same reason as with the queue: - /// `MISSING` means "I checked and it is not there", and that is a conclusion - /// nobody has the right to draw when reading the mount table failed. - public static func mounted(_ state: Bool?) -> String { - switch state { - case .some(true): return "OK" - case .some(false): return L10n.tr("MISSING") - case .none: return L10n.tr("UNKNOWN - could not read the mount table") - } - } - - /// The "Remote control" line. - /// - /// A mount started before 09.10.2026 keeps listening on TCP with no password - /// until it is restarted, and an upgrade deliberately does not restart it. - /// Until then any web page can delete the backup through it - which is why - /// that state is named, with the way out, instead of quietly working. - public static func remoteControl(_ transport: DriveBufferService.RCTransport, mounted: Bool?) - -> String - { - switch transport { - case .socket: - return L10n.tr("private socket") - case .legacyTCP where mounted == true: - return L10n.tr( - "OPEN on %@ - the mount was started by an older version, and any web page can send it commands. It switches to the private socket on its next start: run restart-mount (or prepare-shutdown, then restart the Mac).", - DriveBufferService.legacyRCAddress) - case .legacyTCP where mounted == false: - return L10n.tr("no socket - the mount is not running") - case .legacyTCP: - // Not "the mount is not running": the mount table could not be read, - // and the mount may be up - on the open port, for all we know. - return L10n.tr("no socket - UNKNOWN whether the mount is running") - } - } - - /// The "Free on disk" line. - /// - /// `nil` MUST be named. Not `Optional(427)` (because that looks like a - /// program defect, not like information) and not a substituted zero - /// (because zero is a CONCRETE number at which the buffer guard pauses Time - /// Machine - exactly the bug the other agent fixed by changing the type to - /// `Int?`). A missing measurement means the guard no longer protects the - /// disk from filling up, so the line has to say that plainly. - public static func freeDisk(_ gb: Int?) -> String { - guard let gb else { - return L10n.tr( - "NOT MEASURED - the buffer guard will not pause Time Machine before the disk fills up") - } - return "\(gb) GB" - } - - /// The "Cache on disk" line. - /// - /// Separate from the backlog line, and that is the most important thing - /// here: throughout September 2026 a single line "Buffer: 103 GB of 100G" was - /// supposed to answer two questions - how much space the cache takes and - /// how much is left to upload. It did not answer the second one, because - /// with `--vfs-cache-max-age 9999h` the cache sits at the limit constantly - /// (281 measurements, minimum 99 GB). The buffer guard made its decision to - /// pause Time Machine on this number - hence this change. - public static func cacheSize(_ gb: Int?, limitGB: Int) -> String { - guard let gb else { - return L10n.tr( - "NOT MEASURED - rclone did not respond, and walking the buffer directory failed") - } - return L10n.tr("%@ GB of %@G", "\(gb)", "\(limitGB)") - } - - /// The "To upload" line - the UNSENT BACKLOG, i.e. the quantity on which the - /// buffer guard decides to pause and resume. - /// - /// The item count is a MEASUREMENT, the gigabytes are an ESTIMATE from that - /// count (see `BufferGuardService.backlogGB`) - that is why they carry a "~" - /// and why we show both. A line that gives only the estimate as a number - /// hides how solid the basis for the decision to pause the backup is. - public static func backlog(_ gb: Int?, items: Int?) -> String { - guard let gb, let items else { - return L10n.tr( - "UNKNOWN - the rclone control interface did not respond (the buffer guard will neither pause nor resume Time Machine on this basis)" - ) - } - return L10n.tr("~%@ GB (%@ items)", "\(gb)", "\(items)") - } - /// Lines about a notification that could NOT be delivered. /// /// `HealthAlert.notify` has recently started returning `Bool`, and @@ -132,10 +38,7 @@ public enum StatusLines { /// than silence - and silence is the normal state here (README: "Empty logs /// after a fresh install are normal"). Without this line "no alarm" meant /// both "the backup works" and "nobody checked", i.e. it meant nothing. - /// - /// Named separately and added at the end of `StatusLines` instead of being - /// woven into the existing functions - `drive-status` is being rebuilt in - /// parallel on the `naprawy/dozorca-bufora` branch (l10n-polish-ok: a git branch name). + public static func watchdogRun(_ freshness: WatchdogHeartbeat.Freshness) -> String { switch freshness { case .fresh(let lastRun, let age): diff --git a/mac-app/Sources/CloudMachineCore/TimeMachineStatus.swift b/mac-app/Sources/CloudMachineCore/TimeMachineStatus.swift index 6e45470..326fd95 100644 --- a/mac-app/Sources/CloudMachineCore/TimeMachineStatus.swift +++ b/mac-app/Sources/CloudMachineCore/TimeMachineStatus.swift @@ -23,9 +23,9 @@ public enum TimeMachineStatus { /// /// Until 23.09.2026 there was no limit here at all, and that was a failure /// waiting for its day. `tmutil destinationinfo` reaches the backup - /// destination, i.e. the FUSE-T mount living on Google Drive. With a dead - /// mount (the ENXIO incident of 22.09) the read enters uninterruptible I/O - /// and NEVER returns. The `backup-health` watchdog then hangs on + /// destination - then a FUSE-T mount on Google Drive, now an SMB share in + /// the machine. With a dead destination (the ENXIO incident of 22.09) the + /// read enters uninterruptible I/O and NEVER returns. The `backup-health` watchdog then hangs on /// `currentReport()`, and launchd with `StartInterval` does not start a /// second instance while the first one is alive - i.e. the watchdog goes /// silent PERMANENTLY, exactly at the moment it is supposed to speak. @@ -96,12 +96,40 @@ public enum TimeMachineStatus { return result.stdout } + /// `sudo -n tmutil ` (the NOPASSWD rule, for agents without a GUI + /// session), then plain `tmutil` when there is no such rule. `startbackup` + /// and `stopbackup` need no root, so the second attempt usually works. + /// `true` only when one of them succeeded; failures are logged. + public static func control(_ command: String) async -> Bool { + if let viaSudo = try? await ProcessRunner.runTmutilUnattended([command], timeout: 120) { + if viaSudo.succeeded { return true } + if !viaSudo.isSudoAuthFailure { + CMLogger.log( + "sudo tmutil \(command): exit code \(viaSudo.exitCode) \(shortError(viaSudo))") + return false + } + } + guard let direct = try? await ProcessRunner.run("/usr/bin/tmutil", [command], timeout: 120) + else { + CMLogger.log("tmutil \(command): NO ANSWER within the time limit.") + return false + } + if !direct.succeeded { + CMLogger.log("tmutil \(command): exit code \(direct.exitCode) \(shortError(direct))") + } + return direct.succeeded + } + + private static func shortError(_ result: ProcessResult) -> String { + let text = (result.stderr + " " + result.stdout).trimmingCharacters(in: .whitespacesAndNewlines) + return text.isEmpty ? "(no message)" : text.replacingOccurrences(of: "\n", with: " ") + } + /// Whether a backup is in progress. `nil` = tmutil did not answer, i.e. /// UNKNOWN. /// - /// The distinction matters here for the buffer guard: "not in progress" - /// tells it to go into standby and forget it was supervising a backup, while - /// "unknown" must leave the state unchanged. + /// The distinction matters for the machine's guard: it pauses only a backup + /// that is RUNNING, and "unknown" is not that. public static func runningState() async -> Bool? { guard let out = await output(["status"]) else { return nil } return isRunning(statusOutput: out) @@ -172,149 +200,73 @@ public enum TimeMachineStatus { return running ? progress : nil } - /// Counterpart of `tmutil destinationinfo | awk ... -v mp="$SP_MOUNT"` - - /// looks for the block whose "Mount Point" contains `mountPoint`, and returns - /// its ID. - public static func destinationID(forMountPointContaining mountPoint: String) async -> String? { - guard let out = await output(["destinationinfo"]) else { return nil } - return destinationID(forMountPointContaining: mountPoint, destinationInfoOutput: out) - } + /// One block of `tmutil destinationinfo`. Fields macOS left out are `nil`: + /// a network destination has a `Mount Point` only while it is mounted. + public struct Destination: Equatable, Sendable { + public var name: String? + public var kind: String? + public var url: String? + public var mountPoint: String? + public var id: String? + /// `Quota : 300 GB` -> 300; `nil` = no quota set. + public var quotaGB: Double? - static func destinationID( - forMountPointContaining mountPoint: String, destinationInfoOutput: String - ) - -> String? - { - var found = false - for rawLine in destinationInfoOutput.split(separator: "\n") { - let line = String(rawLine) - if line.hasPrefix("Mount Point") { - found = line.contains(mountPoint) - continue - } - if found, line.hasPrefix("ID") { - let parts = line.split(separator: ":", maxSplits: 1) - guard parts.count == 2 else { continue } - return parts[1].trimmingCharacters(in: .whitespaces) - } + public init( + name: String? = nil, kind: String? = nil, url: String? = nil, mountPoint: String? = nil, + id: String? = nil, quotaGB: Double? = nil + ) { + self.name = name + self.kind = kind + self.url = url + self.mountPoint = mountPoint + self.id = id + self.quotaGB = quotaGB } - return nil - } - - /// Quota (GB) configured for the TM destination at the given mount point - - /// parses the "Quota" line (e.g. "300 GB") from the block found the same way - /// as in `destinationID`. `nil` if TM reports no quota for this destination. - public static func destinationQuotaGB(forMountPointContaining mountPoint: String) async - -> Double? - { - guard let out = await output(["destinationinfo"]) else { return nil } - return destinationQuotaGB(forMountPointContaining: mountPoint, destinationInfoOutput: out) } - static func destinationQuotaGB( - forMountPointContaining mountPoint: String, destinationInfoOutput: String - ) -> Double? { - var found = false + /// The destinations, in order. Blocks are separated by `====` lines; a + /// `Name` line also starts a new block, so output without separators parses + /// the same way. Pure, so it is tested on recorded output. + static func destinations(destinationInfoOutput: String) -> [Destination] { + var result: [Destination] = [] + var current: Destination? + func flush() { + if let current, current != Destination() { result.append(current) } + current = nil + } for rawLine in destinationInfoOutput.split(separator: "\n") { - let line = String(rawLine) - if line.hasPrefix("Mount Point") { - found = line.contains(mountPoint) + let line = rawLine.trimmingCharacters(in: .whitespaces) + if line.hasPrefix("====") { + flush() continue } - if found, line.hasPrefix("Quota") { - let parts = line.split(separator: ":", maxSplits: 1) - guard parts.count == 2 else { continue } - let valueText = parts[1].trimmingCharacters(in: .whitespaces) - let numberText = valueText.split(separator: " ").first.map(String.init) ?? valueText - return Double(numberText) - } - } - return nil - } - - /// Mount point of the CURRENTLY registered Time Machine destination (the - /// architecture guarantees exactly one active local destination - see - /// LocalBackupService). Returns the real, registered path instead of - /// guessing it from the default volume name - the user may name the local - /// volume anything (e.g. a manually created "TimeMachine" partition instead - /// of the default "CloudMachine-Local"), and guessing by name was a real - /// bug: after a manual volume rename the whole GUI/watchdog status showed - /// "no local volume" / "TimeMachine not registered", despite a correctly - /// working, registered destination. - /// - /// Returns `nil` both when there is no destination and when tmutil did not - /// answer - whoever has to tell these two apart (the `backup-health` - /// watchdog: one means "someone changed the destination", the other "we know - /// nothing") asks `destinationReading()`. - public static func currentDestinationMountPoint() async -> String? { - if case .mountPoint(let path) = await destinationReading() { return path } - return nil - } - - /// The answer of `tmutil destinationinfo` with an explicit third state: NO - /// ANSWER. - /// - /// The third state has to exist separately for the same reason as - /// `queueUnknown` in `UploadState`: without it a hung tmutil looked exactly - /// like an unregistered destination and the watchdog would report "Time - /// Machine does not point to CloudMachine" - a true-sounding and false - /// sentence that sends the person the wrong way. - public enum DestinationReading: Equatable, Sendable { - case mountPoint(String) - /// tmutil answered, but there is no destination. - case none - /// tmutil did not answer within the time limit. - case noAnswer - } - - public static func destinationReading() async -> DestinationReading { - guard let out = await output(["destinationinfo"]) else { return .noAnswer } - guard let path = currentDestinationMountPoint(destinationInfoOutput: out) else { return .none } - return .mountPoint(path) - } - - static func currentDestinationMountPoint(destinationInfoOutput: String) -> String? { - for rawLine in destinationInfoOutput.split(separator: "\n") { - let line = String(rawLine) - guard line.hasPrefix("Mount Point") else { continue } - let parts = line.split(separator: ":", maxSplits: 1) - guard parts.count == 2 else { continue } - return parts[1].trimmingCharacters(in: .whitespaces) - } - return nil - } - - /// All registered Time Machine destination IDs - used by - /// `LocalBackupService.setAsDestination` to remove previous destinations - /// before registering a new one (this architecture keeps exactly one active - /// local destination, unlike the legacy approach). - /// - /// An empty list on no answer is SAFE here, and that is the only reason it - /// stays: the only caller deletes the returned destinations one by one - /// before registering a new one, so "I do not know" ends with something not - /// being removed, not with the wrong thing being removed. - public static func allDestinationIDs() async -> [String] { - guard let out = await output(["destinationinfo"]) else { return [] } - return allDestinationIDs(destinationInfoOutput: out) - } - - static func allDestinationIDs(destinationInfoOutput: String) -> [String] { - var ids: [String] = [] - for rawLine in destinationInfoOutput.split(separator: "\n") { - let line = String(rawLine) - guard line.hasPrefix("ID") else { continue } let parts = line.split(separator: ":", maxSplits: 1) guard parts.count == 2 else { continue } - ids.append(parts[1].trimmingCharacters(in: .whitespaces)) + let key = parts[0].trimmingCharacters(in: .whitespaces) + let value = parts[1].trimmingCharacters(in: .whitespaces) + if key == "Name", current?.name != nil { flush() } + var block = current ?? Destination() + switch key { + case "Name": block.name = value + case "Kind": block.kind = value + case "URL": block.url = value + case "Mount Point": block.mountPoint = value + case "ID": block.id = value + case "Quota": + let number = value.split(separator: " ").first.map(String.init) ?? value + block.quotaGB = Double(number) + default: break + } + current = block } - return ids + flush() + return result } - /// `nil` = tmutil did not answer. Deliberately NOT `false`: "the - /// configuration does not contain this text" and "could not ask" are two - /// different answers. - public static func destinationInfoContains(_ needle: String) async -> Bool? { + /// The registered destinations; `nil` = tmutil DID NOT ANSWER, `[]` = it + /// answered that there are none. + public static func destinationsReading() async -> [Destination]? { guard let out = await output(["destinationinfo"]) else { return nil } - return out.contains(needle) + return destinations(destinationInfoOutput: out) } } diff --git a/mac-app/Sources/CloudMachineCore/UploadDrain.swift b/mac-app/Sources/CloudMachineCore/UploadDrain.swift deleted file mode 100644 index 63a6eec..0000000 --- a/mac-app/Sources/CloudMachineCore/UploadDrain.swift +++ /dev/null @@ -1,90 +0,0 @@ -import Foundation - -/// Waiting for rclone to upload the backlog BEFORE `hdiutil attach` starts. -/// -/// Until 01.10.2026 `attach()` waited a fixed 120 s for an empty queue. That is -/// enough for an ordinary attach, but not after a Mac restart without -/// `prepare-shutdown`: with `--vfs-write-back 600s` about 10 min of writes are -/// left in the buffer (01.10 - ~19 GB, ~600 bands). Measured that day: rclone -/// read the dirty cache 15:32-15:36, uploaded 15:37-15:42 (~120 bands/min), and -/// the 120 s ran out halfway through. The first `hdiutil attach` started at -/// 15:39 in the middle of the full upload, opened the lock file and HUNG for -/// 5 min, then failed with "image not recognized"; the second attempt failed -/// after 90 s, the third - by then in quiet - succeeded in 15 s. Time Machine -/// sat without a destination for 17 min, and the monitor was shouting FAILURE. -/// -/// A longer fixed limit is not the answer: with Google's daily limit exhausted -/// the queue will not drain at all, and every attach would pay the whole limit. -/// So we wait as long as the upload IS MAKING PROGRESS, and give up when for -/// `stallTimeout` the number of unsent items has not dropped below its minimum -/// so far. `maxTotal` is a hard ceiling - launchd waits for this process, and -/// an image that is not attached means Time Machine has no destination. -public enum UploadDrain { - - public static let defaultStallTimeout: TimeInterval = 120 - /// 20 min: the backlog of 01.10 (~19 GB) drained in ~6 min, so this is three - /// times that. More makes no sense anyway - a queue that grows faster than it - /// drains is no longer a start-up but a jam, and the watchdog will report it. - public static let defaultMaxTotal: TimeInterval = 1200 - public static let defaultPoll: TimeInterval = 5 - /// How often we repeat moving the upload deadlines forward. After start-up - /// rclone reads the dirty cache band by band (01.10: four minutes) and each - /// one gets a deadline `writeBackSeconds` ahead - a single move at the start - /// would not cover the ones read later. - public static let defaultExpiryInterval: TimeInterval = 60 - - public enum Outcome: Equatable { - /// Queue empty - safe to mount. - case idle - /// No progress for `stallTimeout`. - case stalled(unsent: Int) - /// There was progress, but it did not finish before `maxTotal`. - case timedOut(unsent: Int) - /// rclone did not answer for the whole `stallTimeout`. - case noAnswer - } - - /// `unsent` returns the number of unsent items, or `nil` when rclone did not - /// answer. No answer is not progress: it counts towards `stallTimeout` just - /// like a queue that is standing still. - public static func wait( - stallTimeout: TimeInterval = defaultStallTimeout, - maxTotal: TimeInterval = defaultMaxTotal, - poll: TimeInterval = defaultPoll, - expiryInterval: TimeInterval = defaultExpiryInterval, - now: () -> Date = Date.init, - sleep: (TimeInterval) async -> Void, - expire: () async -> Void, - unsent: () async -> Int? - ) async -> Outcome { - let start = now() - var lastProgress = start - var lastExpiry: Date? - var best: Int? - var last: Int? - - while true { - if lastExpiry.map({ now().timeIntervalSince($0) >= expiryInterval }) ?? true { - await expire() - lastExpiry = now() - } - let reading = await unsent() - if let reading { - if reading == 0 { return .idle } - last = reading - if best.map({ reading < $0 }) ?? true { - best = reading - lastProgress = now() - } - } - let t = now() - if t.timeIntervalSince(lastProgress) >= stallTimeout { - return last.map { .stalled(unsent: $0) } ?? .noAnswer - } - if t.timeIntervalSince(start) >= maxTotal { - return last.map { .timedOut(unsent: $0) } ?? .noAnswer - } - await sleep(poll) - } - } -} diff --git a/mac-app/Sources/CloudMachineCore/UploadState.swift b/mac-app/Sources/CloudMachineCore/UploadState.swift index da60252..d4d70f3 100644 --- a/mac-app/Sources/CloudMachineCore/UploadState.swift +++ b/mac-app/Sources/CloudMachineCore/UploadState.swift @@ -101,7 +101,7 @@ public enum UploadState: Equatable, Sendable { } /// What it means and what to do about it. Written for reading, not for - /// diagnosis - whoever needs numbers has `cloudmachine-agent drive-status`. + /// diagnosis - whoever needs numbers has `cloudmachine-agent vm-status`. public var explanation: String { switch self { case .mountDown: diff --git a/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift b/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift index 6757b3a..f695846 100644 --- a/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift +++ b/mac-app/Sources/CloudMachineCore/VMBundlePrep.swift @@ -46,7 +46,9 @@ public enum VMBundlePrep { case .some(true): // A backup to another destination does not write this image (a second // machine, or a throwaway one next to `cmsmb`); one that may is stopped. - guard await VMStatus.timeMachineTargetsMachine() == false else { + let address = await VMLayout.currentAddress() + guard let address, await VMStatus.timeMachineTargetsMachine(address: address) == false + else { return fail(L10n.tr("Time Machine is backing up - stop it first: tmutil stopbackup")) } step("Time Machine is backing up, but not to \(VMLayout.instanceName) - going on") diff --git a/mac-app/Sources/CloudMachineCore/VMGuard.swift b/mac-app/Sources/CloudMachineCore/VMGuard.swift index 342fd42..fcdc1f0 100644 --- a/mac-app/Sources/CloudMachineCore/VMGuard.swift +++ b/mac-app/Sources/CloudMachineCore/VMGuard.swift @@ -57,7 +57,7 @@ public struct VMGuard: Sendable { return .nothing } - static let gib: Int64 = 1 << 30 + public static let gib: Int64 = 1 << 30 /// What the guard reads and does - replaceable, so the tests walk the whole /// step (marker included) without a machine or tmutil. @@ -85,7 +85,7 @@ public struct VMGuard: Sendable { try? FileManager.default.removeItem(at: VMLayout.guardMarker) } }, - tmutil: { await BufferGuardService.tmutil($0) }, + tmutil: { await TimeMachineStatus.control($0) }, log: { CMLogger.log($0) }, minute: { Calendar.current.component(.minute, from: Date()) }, excludeMachine: { await VMLayout.excludeMachineFromTimeMachine() }) diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift index 4189535..a942fe1 100644 --- a/mac-app/Sources/CloudMachineCore/VMLayout.swift +++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift @@ -7,9 +7,6 @@ import Foundation /// destination (`tmutil setdestination` -> error 45, even for a local /// sparsebundle), and NetFS refuses SMB to the Mac's own address. A share /// served from a VM on the Mac is what Time Machine still accepts. -/// -/// This is the new path, next to the attached-image one; neither touches the -/// other. public enum VMLayout { /// Lima instance name: `cmsmb`, which the launchd agents look up. /// `CLOUDMACHINE_VM_INSTANCE` points the commands at another machine - @@ -46,19 +43,20 @@ public enum VMLayout { /// Host-side state: `~/.cloudmachine/vm`. public static var hostDir: URL { - DriveBufferService.root.appendingPathComponent("vm") + CMPaths.stateDir.appendingPathComponent("vm") } /// The SMB password of `tm`, mode 600. Never logged. public static var passwordFile: URL { hostDir.appendingPathComponent("tm-smb.pass") } /// Present while the guard holds Time Machine paused; it resumes only then. public static var guardMarker: URL { hostDir.appendingPathComponent("guard-paused") } - /// `gdrive:CloudMachine/-tm`: a folder of its own, next to the - /// attached image's folder, so the two layouts never share bands. + /// `gdrive:CloudMachine/-tm`. The `-tm` suffix keeps it apart from + /// `CloudMachine/`, where the retired disk-image layout kept its + /// image - the two never share bands. /// `CLOUDMACHINE_VM_REMOTE` replaces it whole (see `instanceName`). public static var remote: String { override("CLOUDMACHINE_VM_REMOTE") - ?? "\(DriveBufferService.remoteName):\(DriveBufferService.remoteRoot)/\(DriveFolder.name)-tm" + ?? "\(DriveFolder.remoteName):\(DriveFolder.remoteRoot)/\(DriveFolder.name)-tm" } /// Lima's directory with the machine's disk (`LIMA_HOME`, else `~/.lima`). @@ -71,8 +69,7 @@ public enum VMLayout { } /// Keeps `limaHome` out of Time Machine. The exclusion is an xattr on the - /// directory, so it is renewed rather than set once (as for the old buffer, - /// `DriveBufferService.excludeBufferFromTimeMachine`). + /// directory, so it is renewed rather than set once. @discardableResult public static func excludeMachineFromTimeMachine() async -> Bool { let result = try? await ProcessRunner.run( @@ -198,14 +195,22 @@ public enum VMLayout { public var uploadsInProgress: Int /// Bytes still to upload; `nil` = `vfs/queue` did not answer. public var unsentBytes: Int64? + /// Bands rclone gave up on: they exist only in the machine's buffer, so + /// the backup on Drive is incomplete until they are uploaded. + public var erroredFiles: Int + /// rclone has nothing left to evict - the buffer holds only unsent data. + public var outOfSpace: Bool public init( - bytesUsed: Int64, uploadsQueued: Int, uploadsInProgress: Int, unsentBytes: Int64? = nil + bytesUsed: Int64, uploadsQueued: Int, uploadsInProgress: Int, unsentBytes: Int64? = nil, + erroredFiles: Int = 0, outOfSpace: Bool = false ) { self.bytesUsed = bytesUsed self.uploadsQueued = uploadsQueued self.uploadsInProgress = uploadsInProgress self.unsentBytes = unsentBytes + self.erroredFiles = erroredFiles + self.outOfSpace = outOfSpace } } @@ -227,7 +232,11 @@ public enum VMLayout { return BufferStats( bytesUsed: used, uploadsQueued: (cache["uploadsQueued"] as? NSNumber)?.intValue ?? 0, - uploadsInProgress: (cache["uploadsInProgress"] as? NSNumber)?.intValue ?? 0) + uploadsInProgress: (cache["uploadsInProgress"] as? NSNumber)?.intValue ?? 0, + erroredFiles: (cache["erroredFiles"] as? NSNumber)?.intValue ?? 0, + // A flag, not a counter: older rclone does not report it, and its + // absence cannot be mistaken for "buffer full". + outOfSpace: (cache["outOfSpace"] as? Bool) ?? false) } /// One reading of the machine for the guard and the status: the rclone @@ -235,6 +244,11 @@ public enum VMLayout { public struct Reading: Equatable, Sendable { public var buffer: BufferStats public var freeBytes: Int64 + + public init(buffer: BufferStats, freeBytes: Int64) { + self.buffer = buffer + self.freeBytes = freeBytes + } } /// The script prints `vfs/stats` JSON on one line, `CMQUEUE ` @@ -284,6 +298,20 @@ public enum VMLayout { return vmAddress(ipAddrOutput: result.stdout) } + /// Starts a stopped machine (`limactl start`); on a running one Lima only + /// says so. The first start after a Mac restart can take minutes: rclone + /// re-reads its buffer before it serves anything. + public static func start() async -> CMActionResult { + let started = try? await limactl(["start", "--tty=false", instanceName], timeout: 1800) + guard started?.succeeded == true else { + return CMActionResult( + succeeded: false, + message: L10n.tr("limactl start failed: %@", errorTail(started))) + } + return CMActionResult( + succeeded: true, message: L10n.tr("Machine %@ is running.", instanceName)) + } + /// `Running`, `Stopped`, ... or `nil` when there is no instance. Throws /// when `limactl` itself does not answer. public static func instanceStatus() async throws -> String? { diff --git a/mac-app/Sources/CloudMachineCore/VMSetup.swift b/mac-app/Sources/CloudMachineCore/VMSetup.swift index b3e888f..0295399 100644 --- a/mac-app/Sources/CloudMachineCore/VMSetup.swift +++ b/mac-app/Sources/CloudMachineCore/VMSetup.swift @@ -150,8 +150,21 @@ public enum VMSetup { return Result(succeeded: true, messages: messages, destinationCommand: command) } + /// The SMB password `vm-setup` stored, if it did. Never creates one. + public static func storedPassword(file: URL = VMLayout.passwordFile) -> String? { + guard let text = try? String(contentsOf: file, encoding: .utf8) else { return nil } + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } + + /// The same command with the password masked - for showing on screen; the + /// real one goes only to the clipboard. + public static func maskedDestinationCommand(address: String) -> String { + "sudo tmutil setdestination 'smb://\(VMLayout.smbUser):••••••@\(address)/\(VMLayout.shareName)'" + } + /// `sudo tmutil setdestination smb://tm:@
/CloudMachine`. - static func destinationCommand(password: String, address: String) -> String { + public static func destinationCommand(password: String, address: String) -> String { let allowed = CharacterSet.alphanumerics let user = VMLayout.smbUser let encoded = password.addingPercentEncoding(withAllowedCharacters: allowed) ?? password diff --git a/mac-app/Sources/CloudMachineCore/VMShutdown.swift b/mac-app/Sources/CloudMachineCore/VMShutdown.swift new file mode 100644 index 0000000..83e5dcc --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/VMShutdown.swift @@ -0,0 +1,111 @@ +import Foundation + +/// `prepare-shutdown`: stops the machine cleanly before the Mac restarts. +/// +/// What it protects: a band Samba is halfway through writing, and rclone's +/// list of what is still to upload. Both survive a clean guest shutdown - +/// systemd stops smbd, punchfs and rclone in order, and rclone keeps unsent +/// bands in its cache on the machine's disk and uploads them after the next +/// start. A machine killed together with the Mac gets no such order. +/// +/// So: stop the backup, give the upload a while to drain (a shorter backlog +/// after the restart, nothing more - the cache is what keeps the data), then +/// `limactl stop`. A backlog left after the wait is reported, not a reason to +/// refuse: refusing would leave the person to restart anyway, uncleanly. +/// +/// The machine then stays stopped until the next login (`vm-start` runs at +/// load) or until someone starts it (the app's Start button, `vm-setup`). +public enum VMShutdown { + /// How long to wait for the upload before stopping the machine. + public static let drainTimeout: TimeInterval = 30 * 60 + static let pollInterval: TimeInterval = 15 + + public static func perform( + drainTimeout: TimeInterval = drainTimeout, + progress: @Sendable (String) -> Void = { _ in } + ) async -> CMActionResult { + let status: String? + do { + status = try await VMLayout.instanceStatus() + } catch { + return CMActionResult( + succeeded: false, + message: L10n.tr("Could not ask Lima about the machine: %@", error.localizedDescription)) + } + guard status == "Running" else { + return CMActionResult( + succeeded: true, + message: L10n.tr( + "The machine %@ is not running - nothing to prepare.", VMLayout.instanceName)) + } + + // 1. No new writes. A backup that does not stop is a reason to stop here: + // stopping the machine under it is what this command exists to avoid. + if await TimeMachineStatus.runningState() != false { + progress(L10n.tr("Stopping the Time Machine backup...")) + _ = await TimeMachineStatus.control("stopbackup") + var stopped = false + for _ in 0..<24 { + if await TimeMachineStatus.runningState() == false { + stopped = true + break + } + try? await Task.sleep(nanoseconds: 5_000_000_000) + } + guard stopped else { + return CMActionResult( + succeeded: false, + message: L10n.tr( + "Time Machine did not stop its backup within 2 minutes (or tmutil did not answer) - the machine was NOT stopped. Try again in a moment." + )) + } + } + + // 2. Let the upload drain for a while. + progress( + L10n.tr( + "Waiting for the upload to Google Drive (at most %@ min)...", "\(Int(drainTimeout / 60))")) + let deadline = Date().addingTimeInterval(drainTimeout) + var unsent: Int64? + while true { + unsent = await VMLayout.currentReading()?.buffer.unsentBytes + if unsent == 0 || Date() >= deadline { break } + if let unsent { + progress(L10n.tr("Still to upload: %@ GiB", "\(unsent / VMGuard.gib)")) + } + try? await Task.sleep(nanoseconds: UInt64(pollInterval * 1_000_000_000)) + } + + // 3. A clean guest shutdown. + progress(L10n.tr("Stopping the machine %@...", VMLayout.instanceName)) + let stopped = try? await VMLayout.limactl(["stop", VMLayout.instanceName], timeout: 300) + guard stopped?.succeeded == true else { + return CMActionResult( + succeeded: false, + message: L10n.tr("limactl stop failed: %@", VMLayout.errorTail(stopped))) + } + CMLogger.log( + "prepare-shutdown: machine \(VMLayout.instanceName) stopped, unsent \(unsent.map { "\($0 / VMGuard.gib) GiB" } ?? "unknown")" + ) + return CMActionResult(succeeded: true, message: outcomeMessage(unsentBytes: unsent)) + } + + /// What the person reads after the machine stopped. Pure, so every branch + /// is tested: a backlog that is unknown must not read as "all uploaded". + static func outcomeMessage(unsentBytes: Int64?) -> String { + switch unsentBytes { + case .some(0): + return L10n.tr( + "The machine is stopped and everything reached Google Drive. The Mac can be restarted; the machine starts again at login." + ) + case .some(let bytes): + return L10n.tr( + "The machine is stopped. %@ GiB had not reached Google Drive yet - it is kept in the machine's buffer and uploaded after the next start. The Mac can be restarted.", + "\(max(1, bytes / VMGuard.gib))") + case .none: + return L10n.tr( + "The machine is stopped. rclone did not say how much was still to upload; what was not uploaded is kept in the machine's buffer and uploaded after the next start. The Mac can be restarted." + ) + } + } +} diff --git a/mac-app/Sources/CloudMachineCore/VMStatus.swift b/mac-app/Sources/CloudMachineCore/VMStatus.swift index 7a7bed8..56c4509 100644 --- a/mac-app/Sources/CloudMachineCore/VMStatus.swift +++ b/mac-app/Sources/CloudMachineCore/VMStatus.swift @@ -12,6 +12,12 @@ public enum VMStatus { public var punchfsOpenFiles: Int? public var punchfsFileLimit: Int? public var reading: VMLayout.Reading? + /// `vm-prepare-bundle` has run (`bundle.env` exists); `nil` = not read. + public var bundlePrepared: Bool? + /// How many images Time Machine has created in the machine; `nil` = not read. + public var imageCount: Int? + + public init() {} } static let script = @@ -19,6 +25,8 @@ public enum VMStatus { + "; pid=$(systemctl show -p MainPID --value punchfs); " + "if [ \"${pid:-0}\" -gt 0 ]; then echo FD $(ls /proc/$pid/fd | wc -l) " + "$(awk '/Max open files/{print $4}' /proc/$pid/limits); fi; " + + "echo BUNDLEENV $([ -s \(VMLayout.Guest.bundleEnv) ] && echo 1 || echo 0); " + + "echo IMAGES $(ls -d \(VMLayout.Guest.localRoot)/*.sparsebundle 2>/dev/null | wc -l); " + VMLayout.readingScript static func report(scriptOutput: String) -> Report { @@ -30,41 +38,102 @@ public enum VMStatus { } else if fields.first == "FD", fields.count >= 2 { report.punchfsOpenFiles = Int(fields[1]) if fields.count >= 3 { report.punchfsFileLimit = Int(fields[2]) } + } else if fields.first == "BUNDLEENV", fields.count >= 2 { + report.bundlePrepared = fields[1] == "1" + } else if fields.first == "IMAGES", fields.count >= 2 { + report.imageCount = Int(fields[1]) } } report.reading = VMLayout.reading(scriptOutput: scriptOutput) return report } - /// Whether Time Machine's destinations include this machine; `nil` = unknown - /// (no address, or `tmutil` did not answer). - public static func timeMachineTargetsMachine() async -> Bool? { - guard let address = await VMLayout.currentAddress(), - let info = try? await ProcessRunner.run( - "/usr/bin/tmutil", ["destinationinfo"], timeout: TimeMachineStatus.commandTimeout), - info.succeeded || info.stdout.contains("No destinations") + /// The machine's services, as a verdict: every one `active`, except + /// `tm-bands-bind`, which is legitimately `inactive` until + /// `vm-prepare-bundle` (its condition skips it). `failed` is never fine: a + /// bind that failed after the image was prepared sends the bands to the + /// machine's local disk instead of Drive. `nil` = no answer for some unit. + public static func servicesHealthy(_ services: [String: String]) -> Bool? { + var unknown = false + for service in VMLayout.services { + guard let state = services[service] else { + unknown = true + continue + } + if state == "active" { continue } + if service == "tm-bands-bind" && state == "inactive" { continue } + return false + } + return unknown ? nil : true + } + + /// One run of the status script in the machine. `nil` = no answer (machine + /// down, limactl stuck); a report whose `reading` is `nil` means rclone in + /// the machine did not answer. + public static func currentReport() async -> Report? { + guard let result = try? await VMLayout.rootShell(script, timeout: 60), result.succeeded else { return nil } - return targets(urls: destinationURLs(destinationInfo: info.stdout), address: address) + return report(scriptOutput: result.stdout) + } + + /// Whether a destination is this machine's share: `smb://tm@
/CloudMachine`. + /// Without an address (the machine did not answer) any `tm@.../CloudMachine` + /// share counts - the question is then "does Time Machine point at a + /// CloudMachine at all". The `@` and `/` keep 192.168.64.1 from matching + /// 192.168.64.10. + public static func isMachineShare(_ url: String, address: String?) -> Bool { + guard url.hasPrefix("smb://"), url.hasSuffix("/\(VMLayout.shareName)"), + url.contains("\(VMLayout.smbUser)@") || url.contains("\(VMLayout.smbUser):") + else { return false } + guard let address else { return true } + return url.contains("@\(address)/") + } + + /// This machine's destination among Time Machine's; `nil` when none is. + public static func machineDestination( + in destinations: [TimeMachineStatus.Destination], address: String? + ) -> TimeMachineStatus.Destination? { + destinations.first { $0.url.map { isMachineShare($0, address: address) } ?? false } } - /// `smb://tm@
/...` among the destinations. The `@` and `/` keep - /// 192.168.64.1 from matching 192.168.64.10. - static func targets(urls: [String], address: String) -> Bool { - urls.contains { $0.contains("@\(address)/") } + /// Whether Time Machine points at the machine; `nil` = `tmutil` did not answer. + public static func timeMachineTargetsMachine(address: String?) async -> Bool? { + guard let destinations = await TimeMachineStatus.destinationsReading() else { return nil } + return machineDestination(in: destinations, address: address) != nil } - /// `URL : smb://...` lines of `tmutil destinationinfo`. - static func destinationURLs(destinationInfo: String) -> [String] { - destinationInfo.split(separator: "\n").compactMap { raw in - let line = raw.trimmingCharacters(in: .whitespaces) - guard line.hasPrefix("URL") else { return nil } - let parts = line.split(separator: ":", maxSplits: 1) - return parts.count == 2 ? parts[1].trimmingCharacters(in: .whitespaces) : nil + /// rclone's remote control in the machine listens on 127.0.0.1:5572 without + /// a password. Lima forwards the guest's localhost ports to the Mac's unless + /// the template says otherwise (`portForwards` in `vm/cmsmb.yaml`), and on + /// the Mac's localhost any web page can send it commands - including + /// deleting the backup, with no trash. `true` = something answers there. + public static func remoteControlOpenOnMac() -> Bool { + tcpPortAnswers(port: 5572) + } + + /// A plain TCP connect to 127.0.0.1, with a short timeout. + static func tcpPortAnswers(port: UInt16, timeoutSeconds: Int = 1) -> Bool { + let socketFD = socket(AF_INET, SOCK_STREAM, 0) + guard socketFD >= 0 else { return false } + defer { close(socketFD) } + var timeout = timeval(tv_sec: timeoutSeconds, tv_usec: 0) + setsockopt( + socketFD, SOL_SOCKET, SO_SNDTIMEO, &timeout, socklen_t(MemoryLayout.size)) + var address = sockaddr_in() + address.sin_len = UInt8(MemoryLayout.size) + address.sin_family = sa_family_t(AF_INET) + address.sin_port = port.bigEndian + address.sin_addr.s_addr = inet_addr("127.0.0.1") + let result = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + connect(socketFD, $0, socklen_t(MemoryLayout.size)) + } } + return result == 0 } /// Above this share of the limit the status warns about descriptors. - static let fileWarningRatio = 0.8 + public static let fileWarningRatio = 0.8 public static func lines() async -> (lines: [String], healthy: Bool) { var out: [String] = [] @@ -87,16 +156,14 @@ public enum VMStatus { out.append(L10n.tr("Address: %@", address ?? L10n.tr("unknown"))) if address == nil { healthy = false } - let result = try? await VMLayout.rootShell(script, timeout: 60) - let report = report(scriptOutput: result?.stdout ?? "") + let report = await currentReport() ?? Report() for service in VMLayout.services { let state = report.services[service] ?? L10n.tr("unknown") out.append( L10n.tr("Service %@: %@", service.padding(toLength: 14, withPad: " ", startingAt: 0), state) ) - // tm-bands-bind is legitimately inactive until vm-prepare-bundle. - if state != "active" && service != "tm-bands-bind" { healthy = false } } + if servicesHealthy(report.services) != true { healthy = false } if let reading = report.reading { out.append( L10n.tr( @@ -121,24 +188,42 @@ public enum VMStatus { } } - let destination = try? await ProcessRunner.run( - "/usr/bin/tmutil", ["destinationinfo"], timeout: TimeMachineStatus.commandTimeout) - if let destination, destination.succeeded || destination.stdout.contains("No destinations") { - let urls = destinationURLs(destinationInfo: destination.stdout) - let ours = urls.first { url in address.map { url.contains("\($0)/") } ?? false } - if let ours { - out.append(L10n.tr("Time Machine: %@", ours)) + if let destinations = await TimeMachineStatus.destinationsReading() { + let urls = destinations.compactMap(\.url) + if let ours = address.flatMap({ machineDestination(in: destinations, address: $0)?.url }) { + out.append(L10n.tr("Time Machine: %@", redactedPassword(ours))) } else { out.append( L10n.tr( "Time Machine: NOT pointing at the machine (%@)", - urls.isEmpty ? L10n.tr("no SMB destination") : urls.joined(separator: ", "))) + urls.isEmpty + ? L10n.tr("no SMB destination") + : urls.map(redactedPassword).joined(separator: ", "))) healthy = false } } else { out.append(L10n.tr("Time Machine: tmutil did not answer")) healthy = false } + if remoteControlOpenOnMac() { + out.append( + L10n.tr( + "Remote control: OPEN on the Mac at 127.0.0.1:5572 - any web page can send rclone commands; see vm/README.md (portForwards)" + )) + healthy = false + } return (out, healthy) } + + /// `smb://tm:secret@host/...` -> `smb://tm:***@host/...`. tmutil prints + /// the URL without the password, but a status line must never be the one + /// place it leaks from. + static func redactedPassword(_ url: String) -> String { + guard let scheme = url.range(of: "://"), let at = url.range(of: "@"), + scheme.upperBound < at.lowerBound + else { return url } + let userInfo = url[scheme.upperBound.. ProcessResult { - let result = try await ProcessRunner.run(executable, args, timeout: timeout) - guard result.succeeded else { - throw Failure( - message: """ - Failed: \(executable) \(args.joined(separator: " ")) - \(result.stderr.isEmpty ? result.stdout : result.stderr) - """) - } - return result - } - - /// The equivalent of `... || true` - called where failure is expected - /// (cleaning up after something that may not exist). - static func runIgnoringFailure( - _ executable: String, _ args: [String], timeout: TimeInterval? = 300 - ) async { - _ = try? await ProcessRunner.run(executable, args, timeout: timeout) - } - - static func detachQuietly(_ path: String, force: Bool = false) async { - var args = ["detach", path, "-quiet"] - if force { args.append("-force") } - await runIgnoringFailure("/usr/bin/hdiutil", args) - } - - /// After a forced detach a device can linger in the system as a zombie. - /// Attaching then returns a dead handle on which `fsck_apfs` reports - /// "failed to read container superblock" with an all-zero UUID - it looks - /// like a deleted backup, but it is only an unreadable device. - /// - /// The `hdiutil info` parsing logic lives in `CloudMachineCore` and is - /// covered by tests - the harness does not duplicate it. - static func purgeStaleDevices(forImage image: URL) async { - await BackupImageService.purgeStaleDevices(image) - } - - // MARK: - Images - - static func bandSectors(bandMB: Int) -> Int { - bandMB * 1024 * 1024 / 512 - } - - static func createSparseImage(at path: URL, sizeGB: Int, volumeName: String) async throws { - try await run( - "/usr/bin/hdiutil", - [ - "create", "-type", "SPARSE", "-size", "\(sizeGB)g", "-fs", "APFS", - "-volname", volumeName, path.path, - ]) - } - - static func createSparsebundle( - at path: URL, sizeGB: Int, volumeName: String, bandMB: Int - ) async throws { - try await run( - "/usr/bin/hdiutil", - [ - "create", "-type", "SPARSEBUNDLE", "-size", "\(sizeGB)g", - "-fs", "Case-sensitive APFS", "-volname", volumeName, - "-imagekey", "sparse-band-size=\(bandSectors(bandMB: bandMB))", - path.path, - ]) - } - - static func attach(_ image: URL, mountpoint: URL) async throws { - try await run( - "/usr/bin/hdiutil", - ["attach", image.path, "-nobrowse", "-mountpoint", mountpoint.path]) - } - - // MARK: - Files - - static func recreateDirectory(_ url: URL) throws { - try? FileManager.default.removeItem(at: url) - try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) - } - - static func randomData(kilobytes: Int) -> Data { - var data = Data(count: kilobytes * 1024) - data.withUnsafeMutableBytes { raw in - guard let base = raw.baseAddress else { return } - arc4random_buf(base, raw.count) - } - return data - } - - /// Overwrites a file IN PLACE - the equivalent of `dd conv=notrunc`. - /// - /// This is not a detail: writing through `Data.write(to:)` creates a new - /// file and swaps it in, so the data would land in other places of the image - /// and the dirtied-bands measurement would measure something other than a - /// real in-place change. - static func overwriteInPlace(_ url: URL, with data: Data) throws { - guard let handle = FileHandle(forWritingAtPath: url.path) else { - throw Failure(message: "Cannot open for writing: \(url.path)") - } - defer { try? handle.close() } - try handle.seek(toOffset: 0) - try handle.write(contentsOf: data) - try handle.synchronize() - } - - static func createFile(_ url: URL, kilobytes: Int) throws { - try randomData(kilobytes: kilobytes).write(to: url) - } - - static func fileCount(in directory: URL) -> Int { - (try? FileManager.default.contentsOfDirectory(atPath: directory.path).count) ?? 0 - } - - /// How many bands changed after the marker - the equivalent of `find -newer`. - static func filesModified(after mark: Date, in directory: URL) -> Int { - guard - let entries = try? FileManager.default.contentsOfDirectory( - at: directory, includingPropertiesForKeys: [.contentModificationDateKey]) - else { return 0 } - return entries.filter { url in - guard - let modified = try? url.resourceValues(forKeys: [.contentModificationDateKey]) - .contentModificationDate - else { return false } - return modified > mark - }.count - } - - /// Disk usage in MB - the equivalent of `du -sk`, i.e. the space ACTUALLY - /// allocated, not the sum of logical sizes. - static func allocatedMegabytes(of directory: URL) -> Int { - guard - let walker = FileManager.default.enumerator( - at: directory, includingPropertiesForKeys: [.totalFileAllocatedSizeKey, .isRegularFileKey]) - else { return 0 } - var total = 0 - for case let url as URL in walker { - let values = try? url.resourceValues(forKeys: [ - .totalFileAllocatedSizeKey, .isRegularFileKey, - ]) - guard values?.isRegularFile == true, let size = values?.totalFileAllocatedSize else { - continue - } - total += size - } - return total / 1024 / 1024 - } - - static func sync() async { - await runIgnoringFailure("/bin/sync", []) - } -} diff --git a/mac-app/Sources/CloudMachinePOC/PullPlugCommand.swift b/mac-app/Sources/CloudMachinePOC/PullPlugCommand.swift deleted file mode 100644 index 6a047f5..0000000 --- a/mac-app/Sources/CloudMachinePOC/PullPlugCommand.swift +++ /dev/null @@ -1,347 +0,0 @@ -import ArgumentParser -import CloudMachineCore -import Foundation - -/// Simulates the death of the cloud layer in the middle of a write. -/// -/// The most dangerous scenario of this architecture: Time Machine writes to -/// the attached image, and underneath it the rclone mount disappears - because -/// the process died, because FUSE-T crashed, because the system put the disk -/// to sleep. The image loses its bands in the middle of a write. -/// -/// The test detaches the outer image (the stand-in for the rclone mount) while -/// writing to the inner one, then attaches everything back and runs -/// `fsck_apfs`. -/// -/// What interests us is not whether the write survives - it will not - but -/// whether the image can be repaired afterwards or has to be thrown away. The -/// difference between "backup interrupted, it will resume" and "backup lost, -/// we start from zero". -struct PullPlugCommand: AsyncParsableCommand { - static let configuration = CommandConfiguration( - commandName: "pullplug", - abstract: "Pulls the cloud layer out mid-write and checks whether the image survived.") - - @Option(name: .long, help: "Band size in MB.") - var bandMB: Int = 64 - - @Option(name: .long, help: "How many times to repeat pulling the floor out.") - var rounds: Int = 3 - - @Option(name: .long, help: "Working directory.") - var root: String = "/tmp/cm-plug" - - @Flag(name: .long, help: "Only clean up after the previous run and exit.") - var clean = false - - private static let fsck = - "/System/Library/Filesystems/apfs.fs/Contents/Resources/fsck_apfs" - - private var runRoot: URL { URL(fileURLWithPath: root) } - private var outerImage: URL { runRoot.appendingPathComponent("stand-in.sparseimage") } - private var mountPoint: URL { runRoot.appendingPathComponent("drive") } - private var image: URL { mountPoint.appendingPathComponent("plug.sparsebundle") } - private var target: URL { URL(fileURLWithPath: "/Volumes/PlugPOC") } - - private func detachAll() async { - await POC.detachQuietly(target.path, force: true) - await POC.detachQuietly(mountPoint.path, force: true) - } - - func run() async throws { - guard !clean else { - await detachAll() - try? FileManager.default.removeItem(at: runRoot) - print("Cleaned up.") - return - } - - do { - try await exercise() - } catch { - await detachAll() - throw error - } - await detachAll() - } - - private func exercise() async throws { - await detachAll() - try POC.recreateDirectory(runRoot) - try FileManager.default.createDirectory(at: mountPoint, withIntermediateDirectories: true) - - print("Preparation") - try await POC.createSparseImage(at: outerImage, sizeGB: 20, volumeName: "PlugStandIn") - try await POC.attach(outerImage, mountpoint: mountPoint) - try await POC.createSparsebundle( - at: image, sizeGB: 10, volumeName: "PlugPOC", bandMB: bandMB) - - var failed = 0 - // Rounds that MEASURED NOTHING: the write did not start, finished before - // the floor was pulled out, or the image could not be checked. Without - // this counter a run without a single measurement ended with the sentence - // "The image survived every floor pull". - var unmeasured = 0 - var executed = 0 - for round in 1...rounds { - executed += 1 - print("") - print("--- round \(round) ---") - try await POC.attach(image, mountpoint: target) - - // A background write, so the floor can be pulled out from under it. This - // write IS MEANT to fail - interrupting it halfway is the whole point of - // the test. - let writer = Task.detached { [target] in - writeUntilItBreaks( - to: target.appendingPathComponent("load-\(round).bin"), megabytes: 1500) - } - try await Task.sleep(nanoseconds: 3_000_000_000) - - print("Pulling the floor out (detaching the mount stand-in)") - await POC.detachQuietly(mountPoint.path, force: true) - // "I never started writing" is NOT "write interrupted". The former means - // the round had nothing to interrupt, i.e. it measured nothing. - switch await writer.value { - case .neverStarted(let reason): - print(" THE WRITE NEVER STARTED: \(reason)") - print(" round MEASURES NOTHING - there was nothing to interrupt") - unmeasured += 1 - case .interrupted(let megabytes): - print(" write interrupted after \(megabytes) MB, as expected") - case .completed(let megabytes): - print(" WARNING: the \(megabytes) MB write finished BEFORE the floor was pulled out") - print(" round MEASURES NOTHING - the floor disappeared only after the write") - unmeasured += 1 - } - await POC.detachQuietly(target.path, force: true) - - print("Restoring the layer and checking the image") - // We do not check the image in place. After a forced detach a device can - // linger in the system as a zombie; attaching then returns a dead handle, - // and fsck_apfs reports "failed to read container superblock" with an - // all-zero UUID. It looks like irreversible damage, but it is only an - // unreadable device - an earlier version of this test, on that basis, - // declared three times in a row the loss of a backup that was intact. - // - // A copy under a fresh path is immune to this artefact: new file, new - // device, no old handle has anything to do with it. - await POC.detachQuietly(target.path, force: true) - await POC.purgeStaleDevices(forImage: image) - try await Task.sleep(nanoseconds: 2_000_000_000) - try await POC.attach(outerImage, mountpoint: mountPoint) - try await Task.sleep(nanoseconds: 1_000_000_000) - - let copy = runRoot.appendingPathComponent("check-\(round).sparsebundle") - try? FileManager.default.removeItem(at: copy) - try FileManager.default.copyItem(at: image, to: copy) - - guard let device = try await attachWithoutMounting(copy) else { - print(" RESULT: the image cannot even be attached - lost") - failed += 1 - break - } - - let log = runRoot.appendingPathComponent("fsck-\(round).log") - switch await checkImage(device: device, writingTo: log, repair: false) { - case .consistent: - print(" RESULT: consistent") - case .notChecked(let reason): - // NOT "lost": we do not have a single result. A backup gets rebuilt from - // zero on the strength of that sentence, so a guess has no right to say it. - print(" RESULT: COULD NOT CHECK (\(reason))") - print(" image consistency REMAINS UNCHECKED - the round measures nothing") - unmeasured += 1 - case .inconsistent: - print(" RESULT: inconsistent - trying to repair") - switch await checkImage(device: device, writingTo: log, repair: true, append: true) { - case .consistent: - print(" repair succeeded - the backup can be saved") - case .inconsistent: - print(" repair failed - backup lost (log: \(log.path))") - failed += 1 - case .notChecked(let reason): - print(" the repair COULD NOT be started (\(reason))") - print(" unknown whether the backup can be saved (log: \(log.path))") - unmeasured += 1 - } - } - await POC.detachQuietly(device, force: true) - try? FileManager.default.removeItem(at: copy) - } - - print("") - print("===================================") - for line in Self.summary( - requestedRounds: rounds, executedRounds: executed, lost: failed, unmeasured: unmeasured) - { - print(line) - } - } - - /// Attaches the copy without mounting and returns the device with the APFS - /// container. `41504653` is the Apple_APFS partition type in the output of - /// `hdiutil attach -nomount`. - private func attachWithoutMounting(_ copy: URL) async throws -> String? { - guard - let result = try? await POC.run( - "/usr/bin/hdiutil", ["attach", copy.path, "-nomount"], timeout: 300) - else { return nil } - for line in result.stdout.components(separatedBy: .newlines) where line.contains("41504653") { - let device = line.prefix(while: { !$0.isWhitespace }) - if !device.isEmpty { return String(device) } - } - return nil - } - - /// `fsck_apfs` on a damaged image can run for hours - hence `timeout: nil`. - /// A killed fsck reports a failure that cannot be told apart from a real - /// inconsistency, and that is the whole quantity measured here. - private func checkImage( - device: String, writingTo log: URL, repair: Bool, append: Bool = false - ) async -> ImageCheck { - let result = try? await ProcessRunner.run( - Self.fsck, [repair ? "-y" : "-n", device], timeout: nil) - let text = (result?.stdout ?? "") + (result?.stderr ?? "") - let previous = append ? ((try? String(contentsOf: log, encoding: .utf8)) ?? "") : "" - try? (previous + text).write(to: log, atomically: true, encoding: .utf8) - return Self.classify(fsck: result) - } - - /// "Could not check" is NOT the same as "inconsistent". - /// - /// The same pattern as in `BackupImageService.verifyLocked()`: a `fsck_apfs` - /// that never ran (missing binary, killed process, pulled device) gave - /// `false` exactly like a `fsck_apfs` that found damage - the harness then - /// reported "backup lost" and counted an irreversible loss without having a - /// single result. A false alarm about losing the whole backup is more - /// dangerous here than no answer, because the backup gets rebuilt from zero - /// on the strength of it. - static func classify(fsck result: ProcessResult?) -> ImageCheck { - guard let result else { - return .notChecked("fsck_apfs could not be started or returned no result") - } - return result.succeeded ? .consistent : .inconsistent - } - - /// The last sentence of the run - the only one anybody will remember. - /// - /// Pure and extracted, because this is where the bug was: as long as only - /// losses counted, a run WITHOUT A SINGLE measurement ended with the - /// sentence "The image survived every floor pull". The harness has no right - /// to declare that something survived if it does not know whether it even - /// tried to kill it. - static func summary(requestedRounds: Int, executedRounds: Int, lost: Int, unmeasured: Int) - -> [String] - { - var lines = [ - "Rounds: \(requestedRounds) requested, \(executedRounds) executed " - + "irreversible losses: \(lost) rounds without a measurement: \(unmeasured)" - ] - if executedRounds == 0 { - lines.append("THE RUN MEASURED NOTHING: not a single round was executed.") - return lines - } - if lost > 0 { - lines.append("WARNING: the architecture loses the backup when the cloud layer is lost.") - return lines - } - if unmeasured > 0 { - lines.append( - "THE RUN PROVES NOTHING: \(unmeasured) of \(executedRounds) rounds measured nothing") - lines.append( - "(the write did not start, finished before the floor was pulled out, or the image " - + "could not be checked).") - return lines - } - lines.append("The image survived every floor pull.") - return lines - } -} - -/// The result of checking the image with `fsck_apfs`. Three states, because -/// "could not check" and "inconsistent" are two different answers - see -/// `classify`. -enum ImageCheck: Equatable { - case consistent - case inconsistent - case notChecked(String) -} - -/// What happened to the test write. -/// -/// Three states, not two. `writeUntilItBreaks` used to return `Bool`, and -/// `false` meant both "the write was interrupted halfway" (the whole point of -/// the test) and "the write could not start at all" - `createFile` or -/// `FileHandle` failed immediately, for example because the path does not -/// exist. The harness then printed "write interrupted, as expected" and ended -/// with "The image survived every floor pull", without having written a single -/// byte. -/// -/// `completed` is separate too and is NOT a test success either: a write that -/// finished before the floor was pulled out measured nothing (exactly the trap -/// the comment about `arc4random_buf` on the function warns about). -enum WriteProbe: Equatable { - /// We did NOT START the write - the round measures nothing. - case neverStarted(String) - /// The write was running and got interrupted - the expected case. - case interrupted(megabytesWritten: Int) - /// The write reached the end, i.e. the floor disappeared too late or not at all. - case completed(megabytesWritten: Int) -} - -/// Pours random data until the floor disappears. -/// -/// Internal (not `private`), so that a test can check that "I never started -/// writing" and "write interrupted" are two different answers. -/// -/// Writes in chunks through `FileHandle`, not with a single `Data.write`, so -/// that the write really takes time and can be interrupted halfway. -/// -/// The data source is `/dev/urandom` - exactly like `dd if=/dev/urandom` in -/// the shell version. This is not excessive fidelity but a condition for the -/// test to work: urandom sets the pace of the write. The version generating -/// data with `arc4random_buf` pushed 1500 MB in under three seconds, so the -/// write finished BEFORE the floor was pulled out - the harness reported "the -/// image survived" without having checked what it exists for. Measured: with -/// urandom the write is still running when the mount disappears. -func writeUntilItBreaks(to url: URL, megabytes: Int) -> WriteProbe { - guard FileManager.default.createFile(atPath: url.path, contents: nil) else { - return .neverStarted("could not create \(url.path)") - } - guard let handle = FileHandle(forWritingAtPath: url.path) else { - return .neverStarted("could not open for writing \(url.path)") - } - guard let entropy = FileHandle(forReadingAtPath: "/dev/urandom") else { - try? handle.close() - return .neverStarted("could not open /dev/urandom") - } - defer { - try? handle.close() - try? entropy.close() - } - var written = 0 - for _ in 0.. + var loaded: Set + var bootedOut: [String] = [] + var removed: [String] = [] + var logs: [String] = [] + var deletable = true + + init(onDisk: Set, loaded: Set) { + self.onDisk = onDisk + self.loaded = loaded + } + + var probes: AgentRepair.RetiredProbes { + AgentRepair.RetiredProbes( + plistExists: { self.onDisk.contains($0) }, + isLoaded: { self.loaded.contains($0) }, + bootout: { + self.bootedOut.append($0) + self.loaded.remove($0) + }, + removePlist: { + guard self.deletable else { return false } + self.removed.append($0) + self.onDisk.remove($0) + return true + }, + leftovers: { ["~/.cloudmachine/cache"] }, + log: { self.logs.append($0) }) + } + } + + func testRetiredAgentsAreUnloadedAndDeleted() async { + let fake = Fake( + onDisk: ["gdrive-buffer", "gdrive-attach", "buffer-guard"], loaded: ["gdrive-buffer"]) + let removed = await AgentRepair.removeRetiredAgents(fake.probes) + XCTAssertEqual(removed, ["gdrive-buffer", "gdrive-attach", "buffer-guard"]) + XCTAssertEqual(fake.bootedOut, ["gdrive-buffer"], "only a loaded agent is booted out") + XCTAssertEqual(fake.removed, ["gdrive-buffer", "gdrive-attach", "buffer-guard"]) + XCTAssertTrue( + fake.logs.contains { $0.contains("remove by hand") && $0.contains(".cloudmachine/cache") }, + "what is left must be listed, not deleted: \(fake.logs)") + } + + /// The second run - and every launch after it - finds nothing and says nothing. + func testMigrationIsIdempotent() async { + let fake = Fake(onDisk: ["buffer-guard"], loaded: ["buffer-guard"]) + _ = await AgentRepair.removeRetiredAgents(fake.probes) + fake.logs = [] + let again = await AgentRepair.removeRetiredAgents(fake.probes) + XCTAssertEqual(again, []) + XCTAssertEqual(fake.logs, []) + } + + /// A Mac whose old agents were moved away by hand but are still loaded. + func testLoadedAgentWithoutPlistIsStillBootedOut() async { + let fake = Fake(onDisk: [], loaded: ["gdrive-attach"]) + let removed = await AgentRepair.removeRetiredAgents(fake.probes) + XCTAssertEqual(removed, ["gdrive-attach"]) + XCTAssertEqual(fake.bootedOut, ["gdrive-attach"]) + XCTAssertEqual(fake.removed, []) + } + + func testUndeletablePlistIsNotReportedAsRemoved() async { + let fake = Fake(onDisk: ["gdrive-buffer"], loaded: []) + fake.deletable = false + let removed = await AgentRepair.removeRetiredAgents(fake.probes) + XCTAssertEqual(removed, []) + XCTAssertTrue(fake.logs.contains { $0.contains("could not be deleted") }) + } + + func testLeftoversNameTheMountsAndNeverTheManagedRclone() { + let home = URL(fileURLWithPath: "/Users/someone") + let lines = AgentRepair.retiredLeftovers( + home: home, mounted: ["/Volumes/CloudMachine"], exists: { _ in true }) + XCTAssertTrue(lines.first?.contains("/Volumes/CloudMachine is still mounted") == true) + XCTAssertTrue(lines.contains("/Users/someone/.cloudmachine/cache")) + XCTAssertFalse(lines.contains { $0.contains(".cloudmachine/bin") }, "rclone is still used") + XCTAssertFalse(lines.contains { $0.contains(".cloudmachine/vm") }, "the machine's state") } } diff --git a/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift b/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift new file mode 100644 index 0000000..0146912 --- /dev/null +++ b/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift @@ -0,0 +1,39 @@ +import CloudMachineCore +import Foundation + +@testable import CloudMachineApp + +/// A machine in which everything answers and nothing waits to be uploaded. +extension MachineStatus { + static var healthyForTests: MachineStatus { + var machine = MachineStatus() + machine.state = .running + machine.address = "192.168.64.10" + machine.services = [ + "rclone-tm": "active", "punchfs": "active", "tm-bands-bind": "active", "smbd": "active", + ] + machine.reading = VMLayout.Reading( + buffer: VMLayout.BufferStats( + bytesUsed: 60 * VMGuard.gib, uploadsQueued: 0, uploadsInProgress: 0, unsentBytes: 0), + freeBytes: 120 * VMGuard.gib) + machine.bundlePrepared = true + machine.imageCount = 1 + return machine + } +} + +extension AppStatus { + /// Everything fine: a test changes ONE thing and looks at what that does. + @MainActor + static func healthyForTests() -> AppStatus { + let status = AppStatus() + status.dependencyState = .ready + status.remoteConfigured = true + status.machine = .healthyForTests + status.timeMachineState = .registered(url: "smb://tm@192.168.64.10/CloudMachine") + status.backupCycle = BackupCycleStatus( + known: true, lastSuccess: Date().addingTimeInterval(-1800), problems: [], + checkedAt: Date()) + return status + } +} diff --git a/mac-app/Tests/CloudMachineAppTests/AppStatusHealthTests.swift b/mac-app/Tests/CloudMachineAppTests/AppStatusHealthTests.swift index 9877ab3..2beedbd 100644 --- a/mac-app/Tests/CloudMachineAppTests/AppStatusHealthTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/AppStatusHealthTests.swift @@ -7,9 +7,8 @@ import XCTest /// and the headline in the menu bar. /// /// They exist because mutation testing exposed a gap: `healthy` and `headline` had NO -/// test at all, so the fix adding `erroredFiles` and `outOfSpace` to them -/// passed, but its inverse would have passed just as well. Broken -/// and working looked identical in the test suite too. +/// test at all, so a fix adding a condition to them passed, but its inverse +/// would have passed just as well. /// /// Headlines that come from `UploadState` or `BackupHealth.formatAge` (CloudMachineCore) /// are compared with the core's own text: their wording is that module's business, @@ -17,268 +16,231 @@ import XCTest @MainActor final class AppStatusHealthTests: XCTestCase { - /// A state in which everything really works - the reference point. - private func healthy() -> AppStatus { - let status = AppStatus() - status.dependencyState = .ready - status.remoteConfigured = true - status.timeMachineState = .registered(mountPoint: "/Volumes/CloudMachine") - var buffer = BufferStatus() - buffer.mounted = true - buffer.imageAttached = true - // Must be explicit: `BufferStatus` starts from "queue not read", so that - // a fresh, unchecked state does not pass for an empty queue. - buffer.queueKnown = true - buffer.freeDiskGB = 400 - status.buffer = buffer - // CHANGE 23.09.2026: "everything attached" stopped being enough for the green - // badge. The reference point now also has to contain the fact that a backup - // WAS ACTUALLY made - because that is exactly what was missing in the failure for which - // `BackupHealth` was created in the first place. Previously this helper described the state of the - // devices and said nothing about whether the backup succeeded; the test "a healthy state - // is healthy" therefore also passed for a Mac that had not made a backup for - // two days. - status.backupCycle = BackupCycleStatus( - known: true, lastSuccess: Date().addingTimeInterval(-1800), problems: [], - checkedAt: Date()) - return status - } + private func healthyStatus() -> AppStatus { AppStatus.healthyForTests() } - /// REGRESSION 23.09.2026: `rclone rc` did not answer within the time limit, - /// the caller substituted zeros and the menu bar showed "Ready" with 386 bands - /// waiting in the queue. - func testUnreadQueueTakesAwayGreenBadge() { - let status = healthy() - status.buffer.queueKnown = false - XCTAssertFalse(status.healthy, "Unknown = not green.") - XCTAssertNotEqual(status.headline, "Ready") - XCTAssertEqual(status.buffer.uploadState, .queueUnknown) + private func withReading( + _ status: AppStatus, _ change: (inout VMLayout.BufferStats) -> Void + ) { + var machine = status.machine + var buffer = machine.reading!.buffer + change(&buffer) + machine.reading!.buffer = buffer + status.machine = machine } - // MARK: - Open remote control - - /// REGRESSION 11.10.2026 (N-K1): the old mount kept listening on the open - /// TCP port after the upgrade, and the window stayed green - only one line - /// of `drive-status` said so. - func testOpenRemoteControlTakesAwayGreenAndSaysSo() { - let status = healthy() + func testHealthyStateIsHealthy() { + let status = healthyStatus() XCTAssertTrue(status.healthy) - status.remoteControlOpen = true + XCTAssertEqual(status.headline, "Ready") + XCTAssertEqual(status.overallTone, .success) + } + + /// REGRESSION 23.09.2026: rclone did not answer, the caller substituted + /// zeros and the menu bar showed "Ready" with bands waiting in the queue. + func testUnreadQueueTakesAwayGreenBadge() { + let status = healthyStatus() + withReading(status) { $0.unsentBytes = nil } XCTAssertFalse(status.healthy) - XCTAssertEqual(status.headline, "Drive remote control is open - restart the mount") - XCTAssertTrue(status.canRestartMount) - // Backups still work in this state. - XCTAssertTrue(status.canStartBackup) + XCTAssertEqual(status.machine.uploadState, .queueUnknown) + XCTAssertEqual(status.headline, UploadState.queueUnknown.headline) } - func testMountRestartIsNotOfferedDuringABackupOrAnotherAction() { - let status = healthy() - XCTAssertFalse(status.canRestartMount, "Nothing to restart while the port is closed.") + /// An open remote control makes any web page able to delete the backup. + func testOpenRemoteControlTakesAwayGreenAndSaysSo() { + let status = healthyStatus() status.remoteControlOpen = true - status.backupProgress = BackupProgressInfo() - XCTAssertFalse(status.canRestartMount) - status.backupProgress = nil - status.isBusy = true - XCTAssertFalse(status.canRestartMount) + XCTAssertFalse(status.healthy) + XCTAssertEqual(status.headline, "rclone's remote control is open on this Mac") + XCTAssertEqual(status.overallTone, .danger) } - // MARK: - "Back up now" - /// REGRESSION 09.10.2026: the button followed `healthy`, so an OLD backup - /// the very thing it fixes - greyed it out. func testStaleBackupStillAllowsBackingUp() { - let status = healthy() - status.backupCycle = BackupCycleStatus( - known: true, lastSuccess: Date().addingTimeInterval(-2 * 86400), problems: [], - checkedAt: Date()) + let status = healthyStatus() + status.backupCycle.lastSuccess = Date().addingTimeInterval(-3 * 24 * 3600) XCTAssertFalse(status.healthy) XCTAssertTrue(status.canStartBackup) } func testUnreadQueueStillAllowsBackingUp() { - let status = healthy() - status.buffer.queueKnown = false + let status = healthyStatus() + withReading(status) { $0.unsentBytes = nil } XCTAssertTrue(status.canStartBackup) } func testNowhereToBackUpToBlocksTheButton() { - for breakIt in [ - { (s: AppStatus) in s.buffer.mounted = false }, - { (s: AppStatus) in s.buffer.imageAttached = false }, - { (s: AppStatus) in s.timeMachineState = .notRegistered }, - { (s: AppStatus) in s.timeMachineState = .noAnswer }, - { (s: AppStatus) in s.remoteConfigured = false }, - { (s: AppStatus) in s.isBusy = true }, - ] { - let status = healthy() - breakIt(status) - XCTAssertFalse(status.canStartBackup) - } - } + var status = healthyStatus() + status.machine.state = .notRunning("Stopped") + XCTAssertFalse(status.canStartBackup) + XCTAssertTrue(status.canStartMachine) - // MARK: - "Cloud sync queue" row + status = healthyStatus() + status.timeMachineState = .notRegistered + XCTAssertFalse(status.canStartBackup) - /// REGRESSION 09.10.2026: an empty queue with abandoned bands read - /// "Everything uploaded" - for data that exists only on this Mac. - func testAbandonedBandsAreNotEverythingUploaded() { - var buffer = healthy().buffer - buffer.erroredFiles = 3 - XCTAssertNotEqual(buffer.queueSummary, L10n.tr("Everything uploaded")) - XCTAssertTrue(buffer.queueSummary.contains("3")) + status = healthyStatus() + status.machine.services["smbd"] = "failed" + XCTAssertFalse(status.canStartBackup) + + status = healthyStatus() + status.isBusy = true + XCTAssertFalse(status.canStartBackup) } - func testQueueRowStates() { - var buffer = healthy().buffer - XCTAssertEqual(buffer.queueSummary, L10n.tr("Everything uploaded")) - buffer.uploadsQueued = 5 - XCTAssertEqual(buffer.queueSummary, L10n.tr("%@ in progress, %@ queued", "0", "5")) - buffer.queueKnown = false - XCTAssertEqual(buffer.queueSummary, L10n.tr("not read")) + /// An empty queue with abandoned bands is not "Everything uploaded" - the + /// data exists only in the machine. + func testAbandonedBandsAreNotEverythingUploaded() { + let status = healthyStatus() + withReading(status) { $0.erroredFiles = 3 } + XCTAssertNotEqual(status.machine.queueSummary, "Everything uploaded") + XCTAssertTrue(status.machine.queueSummary.contains("3")) } - func testHealthyStateIsHealthy() { - let status = healthy() - XCTAssertTrue(status.healthy) - XCTAssertEqual(status.headline, "Ready") + func testQueueRowStates() { + let status = healthyStatus() + XCTAssertEqual(status.machine.queueSummary, "Everything uploaded") + withReading(status) { + $0.unsentBytes = 5 * VMGuard.gib + $0.uploadsQueued = 160 + $0.uploadsInProgress = 8 + } + XCTAssertEqual(status.machine.queueSummary, "5 GiB in 160 bands, 8 uploading") + withReading(status) { $0.unsentBytes = nil } + XCTAssertEqual(status.machine.queueSummary, "not read") + status.machine = MachineStatus() + XCTAssertEqual(status.machine.queueSummary, "not read", "a fresh state is not a measurement") } - /// THIS is the failure. Bands that rclone did not upload exist only - /// on this Mac - so the backup is not a copy. The interface then showed - /// a green badge and "Ready". + /// Bands rclone did not upload exist only in the machine - the backup is + /// not a copy. func testUnsentFilesTakeAwayGreenBadge() { - let status = healthy() - status.buffer.erroredFiles = 7 - XCTAssertFalse(status.healthy, "Unsent bands MUST NOT pass for a healthy state.") - XCTAssertEqual(status.buffer.uploadState, .failedFiles(7)) - XCTAssertEqual(status.headline, UploadState.failedFiles(7).headline) + let status = healthyStatus() + withReading(status) { $0.erroredFiles = 2 } + XCTAssertFalse(status.healthy) + XCTAssertEqual(status.machine.uploadState, .failedFiles(2)) } - /// rclone reports that it has nowhere left to put data. A stronger signal than - /// any threshold of ours, because it comes from the one who really knows. func testFullBufferTakesAwayGreenBadge() { - let status = healthy() - status.buffer.outOfSpace = true + let status = healthyStatus() + withReading(status) { $0.outOfSpace = true } XCTAssertFalse(status.healthy) - XCTAssertEqual(status.buffer.uploadState, .bufferFull) - XCTAssertEqual(status.headline, UploadState.bufferFull.headline) + XCTAssertEqual(status.machine.uploadState, .bufferFull) } - func testMissingMountTakesAwayGreenBadge() { - let status = healthy() - status.buffer.mounted = false + /// rclone in the machine down: nothing reaches the Drive. + func testStoppedUploadTakesAwayGreenBadge() { + let status = healthyStatus() + status.machine.services["rclone-tm"] = "activating" XCTAssertFalse(status.healthy) - XCTAssertEqual(status.headline, "Buffer is not working") + XCTAssertEqual(status.machine.uploadState, .mountDown) + XCTAssertEqual(status.headline, "A service in the CloudMachine machine is not running") + } + + func testMachineStatesHaveTheirOwnHeadlines() { + let status = healthyStatus() + status.machine = MachineStatus() + status.machine.state = .notCreated + XCTAssertEqual(status.headline, "The CloudMachine machine is not set up yet") + status.machine.state = .notRunning("Stopped") + XCTAssertEqual(status.headline, "The CloudMachine machine is stopped") + status.machine.state = .noAnswer + XCTAssertEqual(status.headline, "The CloudMachine machine does not answer") + XCTAssertFalse(status.healthy) + XCTAssertEqual(status.overallTone, .danger) } - func testDetachedImageTakesAwayGreenBadge() { - let status = healthy() - status.buffer.imageAttached = false + /// `tm-bands-bind` is skipped before `vm-prepare-bundle`; that is not a failure. + func testSkippedBandBindIsNotAFailure() { + let status = healthyStatus() + status.machine.services["tm-bands-bind"] = "inactive" + XCTAssertTrue(status.healthy) + status.machine.services["tm-bands-bind"] = "failed" XCTAssertFalse(status.healthy) - XCTAssertEqual(status.headline, "Backup image not attached") } func testChangedTimeMachineDestinationTakesAwayGreenBadge() { - let status = healthy() + let status = healthyStatus() status.timeMachineState = .notRegistered XCTAssertFalse(status.healthy) XCTAssertEqual(status.headline, "Time Machine does not point to CloudMachine") } - /// The daily limit does NOT require action, but the bands then sit only on this Mac - - /// so the green badge is not deserved. - func testExhaustedDriveLimitTakesAwayGreenBadge() { - let status = healthy() - status.buffer.dailyQuotaExhausted = true - XCTAssertFalse(status.healthy) - XCTAssertFalse(status.buffer.uploadState.needsAttention) - } - - /// No space on the Drive is something DIFFERENT from the daily limit: it will not pass on its own. - func testNoSpaceOnDriveRequiresAction() { - let status = healthy() - status.buffer.driveFull = true - XCTAssertFalse(status.healthy) - XCTAssertTrue(status.buffer.uploadState.needsAttention) - } - func testUnconnectedDriveTakesAwayGreenBadge() { - let status = healthy() + let status = healthyStatus() status.remoteConfigured = false XCTAssertFalse(status.healthy) + XCTAssertEqual(status.headline, "Google Drive not connected") } - /// An upload in progress is NOT a failure - as long as the queue shrinks, everything goes - /// as designed. Without this test a "fix" consisting of raising an alarm - /// on every non-empty queue would have gone unnoticed. + /// An upload in progress is NOT a failure - as long as the queue shrinks, + /// everything goes as designed. func testUploadInProgressIsNotAFailure() { - let status = healthy() - status.buffer.uploadsQueued = 12 + let status = healthyStatus() + withReading(status) { + $0.unsentBytes = 2 * VMGuard.gib + $0.uploadsQueued = 60 + $0.uploadsInProgress = 4 + } XCTAssertTrue(status.healthy) - XCTAssertEqual(status.buffer.uploadState, .flowing(queued: 12)) - XCTAssertEqual(status.headline, UploadState.flowing(queued: 12).headline) + XCTAssertEqual(status.machine.uploadState, .flowing(queued: 64)) + XCTAssertEqual(status.headline, UploadState.flowing(queued: 64).headline) } - // MARK: - Age of the last SUCCESSFUL backup - - /// THE failure. The mount is up, the image attached, the queue empty, the Time - /// Machine destination set - and the last COMPLETED backup is two days old. The panel - /// showed "Healthy / Ready" then, because it never asked about this even once: - /// `grep -rn "BackupHealth" Sources/CloudMachineApp/` gave no hits. + /// THE failure: the machine runs, Time Machine points at it, the queue is + /// empty - and the last COMPLETED backup is two days old. func testOldBackupTakesAwayGreenBadge() { - let status = healthy() - status.backupCycle.lastSuccess = Date().addingTimeInterval(-48 * 3600) - XCTAssertFalse( - status.healthy, - "All the devices can be fine while there has been no backup for two days.") + let status = healthyStatus() + status.backupCycle.lastSuccess = Date().addingTimeInterval(-2 * 24 * 3600) + XCTAssertFalse(status.healthy) XCTAssertEqual( - status.headline, "No completed backup for \(BackupHealth.formatAge(48 * 3600))") + status.headline, + "No completed backup for \(BackupHealth.formatAge(status.backupCycle.age()!))") } - /// The threshold boundary. Just below it is still fine, just above it no longer - - /// without this test a "fix" setting the threshold to 100 years would pass silently. + /// The threshold boundary works both ways. func testBackupAgeThresholdWorksBothWays() { - let justBefore = healthy() - justBefore.backupCycle.lastSuccess = Date().addingTimeInterval( - -(BackupHealth.maxAgeHours * 3600 - 60)) - XCTAssertTrue(justBefore.healthy) - - let justAfter = healthy() - justAfter.backupCycle.lastSuccess = Date().addingTimeInterval( - -(BackupHealth.maxAgeHours * 3600 + 60)) - XCTAssertFalse(justAfter.healthy) + let status = healthyStatus() + let limit = BackupHealth.maxAgeHours * 3600 + status.backupCycle.lastSuccess = Date().addingTimeInterval(-(limit - 60)) + XCTAssertTrue(status.healthy) + status.backupCycle.lastSuccess = Date().addingTimeInterval(-(limit + 60)) + XCTAssertFalse(status.healthy) } /// An unread backup counter is NOT the same as a backup made a moment ago. - /// The default `BackupCycleStatus` has `known == false` precisely so that - /// the panel does not show green before anyone has asked anything. func testUnreadBackupCounterTakesAwayGreenBadge() { - let status = healthy() + let status = healthyStatus() status.backupCycle = BackupCycleStatus() XCTAssertFalse(status.healthy) XCTAssertEqual(status.headline, "Unknown when the last backup was made") } - /// The preferences were read and there is NOT A SINGLE successful backup in them - which is - /// something other than "could not read" and must sound different. func testNoBackupAtAllTakesAwayGreenBadge() { - let status = healthy() - status.backupCycle = BackupCycleStatus(known: true, lastSuccess: nil, checkedAt: Date()) + let status = healthyStatus() + status.backupCycle.lastSuccess = nil XCTAssertFalse(status.healthy) XCTAssertEqual(status.headline, "There is no completed backup at all") } - /// After "Prepare for restart" the image is detached on purpose: the headline - /// says so instead of reporting a failure, and the badge is still not green. - func testDetachedForRestartHasItsOwnHeadline() { - let status = healthy() - status.buffer.imageAttached = false - XCTAssertEqual(status.headline, L10n.tr("Backup image not attached")) - status.preparedForRestart = true - XCTAssertEqual( - status.headline, L10n.tr("Ready for restart - the image is detached on purpose")) - XCTAssertFalse(status.healthy) - // The preparation does not cover an image that is attached again. - status.buffer.imageAttached = true - XCTAssertFalse(status.detachedForRestart) + /// Time Machine has made its first image and the bands are still 875 MB on + /// the machine's disk - `vm-prepare-bundle` is due, and only then. + func testBundleIsDueOnlyForANewUnpreparedImage() { + var machine = MachineStatus.healthyForTests + XCTAssertFalse(machine.bundleNeedsPreparing) + machine.bundlePrepared = false + machine.imageCount = 0 + XCTAssertFalse(machine.bundleNeedsPreparing, "no image yet - Time Machine has to create it") + machine.imageCount = 1 + XCTAssertTrue(machine.bundleNeedsPreparing) + machine.bundlePrepared = nil + XCTAssertFalse(machine.bundleNeedsPreparing, "unread is not 'not prepared'") + } + + func testMachineStateFromLima() { + XCTAssertEqual(MachineState.from(instanceStatus: .none), .noAnswer) + XCTAssertEqual(MachineState.from(instanceStatus: .some(nil)), .notCreated) + XCTAssertEqual(MachineState.from(instanceStatus: .some("Running")), .running) + XCTAssertEqual(MachineState.from(instanceStatus: .some("Stopped")), .notRunning("Stopped")) } } diff --git a/mac-app/Tests/CloudMachineAppTests/AttachStallTests.swift b/mac-app/Tests/CloudMachineAppTests/AttachStallTests.swift deleted file mode 100644 index 14f88b0..0000000 --- a/mac-app/Tests/CloudMachineAppTests/AttachStallTests.swift +++ /dev/null @@ -1,152 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// When a series of refused attaches ends in a restart of the mount - and -/// when it must not. -final class AttachStallTests: XCTestCase { - - private let boot: TimeInterval = 1_792_999_292 - private let start: TimeInterval = 1_793_000_000 - /// The attach agent's tick. - private let tick: TimeInterval = 15 * 60 - - private let refused = CMActionResult( - succeeded: false, - message: - "Could not attach the image: hdiutil: attach failed - Resource temporarily unavailable", - attachTemporarilyUnavailable: true) - private let attached = CMActionResult(succeeded: true, message: "Attached") - - private final class Restarts: @unchecked Sendable { - var count = 0 - } - - /// Runs the attach outcomes one tick apart, carrying the record along. - private func ticks( - _ outcomes: [CMActionResult], from record: AttachStall.Record? = nil, - restartResult: CMActionResult = CMActionResult(succeeded: true, message: "restarted") - ) async -> (record: AttachStall.Record?, restarts: Int) { - let restarts = Restarts() - var record = record - for (i, outcome) in outcomes.enumerated() { - record = await AttachStall.handle( - outcome, record: record, bootTime: boot, now: start + Double(i) * tick, - restart: { - restarts.count += 1 - return restartResult - } - ).record - } - return (record, restarts.count) - } - - func testRecognizesTheHdiutilRefusal() { - XCTAssertTrue( - BackupImageService.isTemporarilyUnavailable( - stderr: - "hdiutil: WARNING: 'hdiutil attach -nobrowse -mountpoint ...' is deprecated.\nhdiutil: attach failed - Resource temporarily unavailable\n" - )) - XCTAssertFalse( - BackupImageService.isTemporarilyUnavailable( - stderr: "hdiutil: attach failed - RPC version wrong")) - } - - /// KNOWN BAD SAMPLE, 25-26 Oct 2026: every attach refused for 26 h, and - /// nothing but a restart of the Mac helped. Three ticks (30 min) are enough. - func testASeriesOfRefusalsRestartsTheMount() async { - let result = await ticks([refused, refused]) - XCTAssertEqual(result.restarts, 0) - let third = await ticks([refused, refused, refused]) - XCTAssertEqual(third.restarts, 1) - } - - /// Refusals just after a start-up are normal (26.10 08:27) - three of them - /// in quick succession are not yet a series. - func testThreeQuickRefusalsAreNotASeries() async { - let quick = AttachStall.addingFailure( - to: AttachStall.addingFailure( - to: AttachStall.addingFailure(to: nil, bootTime: boot, now: start), bootTime: boot, - now: start + 60), bootTime: boot, now: start + 120) - XCTAssertEqual(quick.failures, 3) - XCTAssertFalse(AttachStall.shouldRestart(quick, bootTime: boot, now: start + 120)) - } - - /// A success between the refusals ends the series. - func testASuccessEndsTheSeries() async { - let result = await ticks([refused, refused, attached, refused, refused]) - XCTAssertEqual(result.restarts, 0) - } - - /// Other failures (the buffer not ready, an orphaned mount point) are not - /// what a fresh rclone fixes. - func testOtherFailuresEndTheSeries() async { - let other = CMActionResult(succeeded: false, message: "Drive is not mounted.") - let result = await ticks([refused, refused, other, refused, refused]) - XCTAssertEqual(result.restarts, 0) - } - - /// "Busy with another operation" says nothing about the mount: it neither - /// ends the series nor adds to it. - func testBusyNeitherEndsNorExtendsTheSeries() async { - let busy = CMActionResult(succeeded: false, message: "busy", didNotRun: true) - let result = await ticks([refused, busy, refused, busy, refused]) - XCTAssertEqual(result.restarts, 1) - XCTAssertEqual(result.record?.failures, 0) - } - - /// If a fresh rclone did not help, another one within the cooldown will - /// not either - and every restart cuts the upload short. - func testAtMostOneRestartPerCooldown() async { - let hours = Int(AttachStall.cooldown / tick) - let result = await ticks(Array(repeating: refused, count: hours - 1)) - XCTAssertEqual(result.restarts, 1) - let longer = await ticks(Array(repeating: refused, count: hours + 3)) - XCTAssertEqual(longer.restarts, 2) - } - - /// The cooldown holds across a short success, or restarts would follow - /// every flap. - func testCooldownSurvivesASuccess() async { - let result = await ticks([refused, refused, refused, attached, refused, refused, refused]) - XCTAssertEqual(result.restarts, 1) - } - - /// A refusal of the restart (Time Machine backing up, tmutil silent) - /// touched nothing - the next refused attach asks again, no cooldown. - func testRefusedRestartDoesNotStartTheCooldown() async { - let refusedRestart = CMActionResult(succeeded: false, message: "backing up", didNotRun: true) - let result = await ticks( - [refused, refused, refused, refused], restartResult: refusedRestart) - XCTAssertEqual(result.restarts, 2) - XCTAssertEqual(result.record?.restartedAt, 0) - } - - /// A restart that ran and failed (the queue did not drain) still starts the - /// cooldown: running the same drain every tick would not change its answer. - func testFailedRestartStartsTheCooldown() async { - let failed = CMActionResult(succeeded: false, message: "not drained") - let result = await ticks( - [refused, refused, refused, refused, refused, refused, refused], - restartResult: failed) - XCTAssertEqual(result.restarts, 1) - } - - /// A record from before a restart of the Mac describes a mount that is gone. - func testARecordFromAnotherBootIsIgnored() async { - let old = AttachStall.Record( - bootTime: boot - 86_400, firstFailureAt: start - 7200, failures: 9, restartedAt: start - 600) - let result = await ticks([refused], from: old) - XCTAssertEqual(result.restarts, 0) - XCTAssertEqual(result.record?.failures, 1) - XCTAssertEqual(result.record?.restartedAt, 0) - } - - func testRecordTextRoundTrips() { - let record = AttachStall.Record( - bootTime: boot, firstFailureAt: start, failures: 4, restartedAt: start + 900) - XCTAssertEqual(AttachStall.Record(text: record.text), record) - XCTAssertNil(AttachStall.Record(text: "garbage")) - XCTAssertNil(AttachStall.Record(text: "1 2 3")) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift b/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift index b951bc7..fea051e 100644 --- a/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift @@ -18,70 +18,30 @@ final class BackupHealthTests: XCTestCase { lastSuccess: Date? = nil, lastAttempt: Date? = nil, result: Int? = 0, - mounted: Bool = true, - attached: Bool? = true, + machineRunning: Bool? = true, + servicesHealthy: Bool? = true, destinationRegistered: Bool = true, erroredFiles: Int = 0, outOfSpace: Bool = false, queueReadable: Bool = true, - imageProbeTimedOut: Bool = false + machineFreeGB: Int? = nil, + punchfsNearFileLimit: Bool = false, + remoteControlOpen: Bool = false ) -> BackupHealth.Report { BackupHealth.evaluate( lastSuccess: lastSuccess ?? now.addingTimeInterval(-1800), lastAttempt: lastAttempt ?? now.addingTimeInterval(-1800), result: result, now: now, - mounted: mounted, - attached: attached, + machineRunning: machineRunning, + servicesHealthy: servicesHealthy, destinationRegistered: destinationRegistered, erroredFiles: erroredFiles, outOfSpace: outOfSpace, queueReadable: queueReadable, - imageProbeTimedOut: imageProbeTimedOut) - } - - /// The readability probe did not answer in time. This is NOT "the image is - /// not attached" (the image is in the mount table) and NOT "the image is - /// DEAD" (the device did not answer at all, rather than with an error). The - /// first message would send the person to attach something that is - /// attached; the second - to FORCE-detach a device that may be alive and - /// holding unsent data. - /// - /// Most important, though, is that the watchdog gets here AT ALL: before the - /// fix this read had no time limit, and `StartInterval 1800` without - /// `KeepAlive` means a single hang silenced the watchdog PERMANENTLY. - func testUnansweredProbeIsReportedAsLackOfKnowledge() { - let report = healthyInput(attached: nil, imageProbeTimedOut: true) - XCTAssertFalse(report.healthy, "silence about a state we do not know is a failure here") - XCTAssertTrue( - report.problems.contains { $0.summary.contains("returns data") }, - "the watchdog must FINISH the run and report the lack of knowledge: \(report.problems)") - XCTAssertFalse(report.problems.contains { $0.summary.contains("is not attached") }) - XCTAssertFalse(report.problems.contains { $0.summary.contains("DEAD") }) - } - - /// Two causes of "I do not know" send the person to two different places, - /// so they must not get the same sentence. - func testTwoCausesOfUnknownImageStateHaveDifferentMessages() { - let probe = healthyInput(attached: nil, imageProbeTimedOut: true).problems - let table = healthyInput(attached: nil).problems - XCTAssertNotEqual(probe, table) - XCTAssertFalse(probe.isEmpty) - XCTAssertFalse(table.isEmpty) - } - - /// The image is in the mount table, but reading fails - on 22 Sep 2026 for - /// 15 h no sensor had a name for this state. Now it does. - func testDeadImageAlarms() { - let report = BackupHealth.evaluate( - lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), - result: 0, now: now, mounted: true, attached: true, destinationRegistered: true, - erroredFiles: 0, outOfSpace: false, queueReadable: true, imageDeadErrno: ENXIO) - XCTAssertFalse(report.healthy) - XCTAssertTrue(report.problems.contains { $0.summary.contains("DEAD") }) - XCTAssertFalse( - report.problems.contains { $0.summary.contains("is not attached") }, - "Dead is a different state from detached - one alarm, not two.") + machineFreeGB: machineFreeGB, + punchfsNearFileLimit: punchfsNearFileLimit, + remoteControlOpen: remoteControlOpen) } func testWorkingCycleDoesNotAlarm() { @@ -90,9 +50,8 @@ final class BackupHealthTests: XCTestCase { // MARK: - Known bad samples - /// THIS is the failure the whole previous system did NOT detect: the mount - /// is up, the image attached, the destination registered - and no backup - /// has been made for two days. The interface showed a green badge then. + /// THIS is the failure device checks do NOT detect: the machine runs, the + /// destination is registered - and no backup has been made for two days. func testSilentCycleFailureIsReported() { let report = healthyInput( lastSuccess: now.addingTimeInterval(-48 * 3600), @@ -120,7 +79,7 @@ final class BackupHealthTests: XCTestCase { XCTAssertTrue(report.problems.contains { $0.summary.contains("RESULT=27") }) } - /// Bands rclone did not upload exist only on this Mac. The backup on the + /// Bands rclone did not upload exist only in the machine. The backup on the /// Drive is then INCOMPLETE and may not open. func testUnsentFilesAreReported() { let report = healthyInput(erroredFiles: 12) @@ -128,14 +87,48 @@ final class BackupHealthTests: XCTestCase { XCTAssertTrue(report.problems.contains { $0.summary.contains("12 files") }) } - func testMissingMountIsReported() { + func testStoppedMachineIsReported() { + XCTAssertTrue( + healthyInput(machineRunning: false, servicesHealthy: nil).problems.contains { + $0.code == "machine-stopped" + }) + } + + func testServiceDownIsReported() { XCTAssertTrue( - healthyInput(mounted: false).problems.contains { $0.summary.contains("mount") }) + healthyInput(servicesHealthy: false).problems.contains { $0.code == "machine-service-down" }) + } + + /// A queue nobody could read in a machine that does not answer at all is + /// ONE problem (the machine), not two. + func testSilentMachineIsNotAlsoReportedAsSilentRclone() { + let report = healthyInput(servicesHealthy: nil, queueReadable: false) + XCTAssertTrue(report.problems.contains { $0.code == "machine-no-answer" }) + XCTAssertFalse(report.problems.contains { $0.code == "rclone-rc-no-answer" }) } - func testDetachedImageIsReported() { + func testMachineLowOnSpaceIsReportedBeforeTheGuardBrakes() { + XCTAssertGreaterThan(BackupHealth.machineFreeWarningGB, VMGuard.Thresholds().minFreeGiB) XCTAssertTrue( - healthyInput(attached: false).problems.contains { $0.summary.contains("is not attached") }) + healthyInput(machineFreeGB: BackupHealth.machineFreeWarningGB - 1).problems.contains { + $0.code == "machine-low-space" + }) + XCTAssertTrue(healthyInput(machineFreeGB: BackupHealth.machineFreeWarningGB).healthy) + } + + func testPunchfsNearItsFileLimitIsReported() { + XCTAssertTrue( + healthyInput(punchfsNearFileLimit: true).problems.contains { + $0.code == "punchfs-file-limit" + }) + } + + /// Any web page can delete the backup through it - a failure even on a day + /// when every backup completes. + func testOpenRemoteControlIsReported() { + let report = healthyInput(remoteControlOpen: true) + XCTAssertFalse(report.healthy) + XCTAssertTrue(report.problems.contains { $0.code == "rc-port-open" }) } func testChangedTimeMachineDestinationIsReported() { @@ -163,8 +156,9 @@ final class BackupHealthTests: XCTestCase { /// exceeded". func testAbsenceOfAnyBackupIsReported() { let report = BackupHealth.evaluate( - lastSuccess: nil, lastAttempt: nil, result: 0, now: now, mounted: true, attached: true, - destinationRegistered: true, erroredFiles: 0, outOfSpace: false, queueReadable: true) + lastSuccess: nil, lastAttempt: nil, result: 0, now: now, machineRunning: true, + servicesHealthy: true, destinationRegistered: true, erroredFiles: 0, outOfSpace: false, + queueReadable: true) XCTAssertFalse(report.healthy) XCTAssertTrue(report.problems.contains { $0.summary.contains("NOT A SINGLE") }) } @@ -174,7 +168,8 @@ final class BackupHealthTests: XCTestCase { /// The shape mirrored from the real /// `/Library/Preferences/com.apple.TimeMachine.plist` on a working /// installation: `SnapshotDates` gets an entry only after a COMPLETED - /// backup, `AttemptDates` also counts the ones that failed. + /// backup, `AttemptDates` also counts the ones that failed. The machine's + /// destination carries its `NetworkURL`. private func preferences(volumeName: String = "CloudMachine") -> [String: Any] { [ "Destinations": [ @@ -186,6 +181,7 @@ final class BackupHealthTests: XCTestCase { ], [ "LastKnownVolumeName": volumeName, + "NetworkURL": "smb://tm@192.168.64.10/CloudMachine", "SnapshotDates": [ Date(timeIntervalSince1970: 1_757_600_000), Date(timeIntervalSince1970: 1_757_698_000), @@ -215,9 +211,21 @@ final class BackupHealthTests: XCTestCase { XCTAssertEqual(result, 5) } + /// The machine's share is found by its URL, whatever its volume is called. + func testReadingFindsTheMachineByItsURL() { + let (lastSuccess, _, result) = BackupHealth.dates( + inPreferences: preferences(volumeName: "Renamed")) + XCTAssertEqual(lastSuccess, Date(timeIntervalSince1970: 1_757_698_000)) + XCTAssertEqual(result, 0) + } + func testReadingDoesNotGuessWhenOurDestinationIsMissing() { + var plist = preferences() + plist["Destinations"] = (plist["Destinations"] as! [[String: Any]]).map { + $0.filter { $0.key != "NetworkURL" } + } let (lastSuccess, lastAttempt, result) = BackupHealth.dates( - inPreferences: preferences(), volumeNamed: "SomethingElseEntirely") + inPreferences: plist, volumeNamed: "SomethingElseEntirely") XCTAssertNil(lastSuccess) XCTAssertNil(lastAttempt) XCTAssertNil(result) @@ -372,30 +380,29 @@ final class BackupHealthTests: XCTestCase { // MARK: - Startup grace period /// KNOWN BAD SAMPLE from 21.09, 25.09 and 01.10.2026: the watchdog starts - /// together with the session, a few seconds after rclone starts - nothing is - /// up yet. + /// together with the session - nothing is up yet. private func rightAfterStartup(grace: Bool, lastSuccessAgo: TimeInterval = 1800) -> BackupHealth.Report { BackupHealth.evaluate( lastSuccess: now.addingTimeInterval(-lastSuccessAgo), lastAttempt: now.addingTimeInterval(-lastSuccessAgo), - result: 0, now: now, mounted: false, attached: false, destinationRegistered: false, - erroredFiles: 0, outOfSpace: false, queueReadable: false, + result: 0, now: now, machineRunning: true, servicesHealthy: false, + destinationRegistered: false, erroredFiles: 0, outOfSpace: false, queueReadable: false, withinStartupGrace: grace) } func testRightAfterStartupDevicesNotReadyAreNotAFailure() { let report = rightAfterStartup(grace: true) XCTAssertTrue(report.healthy, "\(report.problems)") - XCTAssertEqual(report.deferred.count, 3, "deferred, not lost: \(report.deferred)") + XCTAssertEqual(report.deferred.count, 2, "deferred, not lost: \(report.deferred)") } /// The same state AFTER the grace period must alarm - otherwise the previous /// test would also pass for a watchdog that is always silent. func testAfterTheGracePeriodTheSameStateAlarms() { let report = rightAfterStartup(grace: false) - XCTAssertEqual(report.problems.count, 3, "\(report.problems)") + XCTAssertEqual(report.problems.count, 2, "\(report.problems)") XCTAssertTrue(report.deferred.isEmpty) } @@ -406,12 +413,24 @@ final class BackupHealthTests: XCTestCase { XCTAssertTrue(report.problems.contains { $0.summary.contains("No successful backup") }) } + /// A machine that has not booted yet is a normal startup state too. + func testRightAfterStartupAStoppedMachineIsDeferred() { + let report = BackupHealth.evaluate( + lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), + result: 0, now: now, machineRunning: false, servicesHealthy: nil, + destinationRegistered: true, erroredFiles: 0, outOfSpace: false, queueReadable: false, + withinStartupGrace: true) + XCTAssertTrue(report.healthy, "\(report.problems)") + XCTAssertEqual(report.deferred.map(\.code), ["machine-stopped"]) + } + /// "I do not know" (a hung tmutil) is not a normal startup state. func testGracePeriodDoesNotSilenceLackOfKnowledge() { let report = BackupHealth.evaluate( lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), - result: 0, now: now, mounted: true, attached: true, destinationRegistered: nil, - erroredFiles: 0, outOfSpace: false, queueReadable: true, withinStartupGrace: true) + result: 0, now: now, machineRunning: true, servicesHealthy: true, + destinationRegistered: nil, erroredFiles: 0, outOfSpace: false, queueReadable: true, + withinStartupGrace: true) XCTAssertFalse(report.healthy) } @@ -427,13 +446,13 @@ final class BackupHealthTests: XCTestCase { BackupHealth.evaluate( lastSuccess: now.addingTimeInterval(-2.5 * 3600), lastAttempt: now.addingTimeInterval(-2 * 3600), - result: 0, now: now, mounted: true, attached: true, destinationRegistered: true, - erroredFiles: 0, outOfSpace: false, queueReadable: true, backupRunning: running) + result: 0, now: now, machineRunning: true, servicesHealthy: true, + destinationRegistered: true, erroredFiles: 0, outOfSpace: false, queueReadable: true, + backupRunning: running) } - /// KNOWN BAD SAMPLE from 01.10.2026 17:02: walking the whole disk after a - /// restart takes hours, and the watchdog reported an attempt that was still - /// in progress. + /// KNOWN BAD SAMPLE from 01.10.2026 17:02: a long run takes hours, and the + /// watchdog reported an attempt that was still in progress. func testRunInProgressIsNotAFailedAttempt() { XCTAssertTrue(attemptTwoHoursAgo(running: true).healthy) } @@ -455,16 +474,26 @@ final class BackupHealthTests: XCTestCase { /// `evaluate` can produce must carry its own stable code - not the /// translated summary it falls back to. func testEveryProblemHasALanguageIndependentCode() { - let report = BackupHealth.evaluate( - lastSuccess: now.addingTimeInterval(-5 * 3600), lastAttempt: nil, result: 3, now: now, - mounted: false, attached: nil, destinationRegistered: nil, erroredFiles: 2, - outOfSpace: true, queueReadable: false, driveFreeBytes: 1_073_741_824, localFreeGB: 1) - XCTAssertFalse(report.problems.isEmpty) - for problem in report.problems { - XCTAssertNotEqual(problem.code, problem.summary, "no own code: \(problem.summary)") + var seen: [String: String] = [:] + for (running, services) in [(true, false), (true, nil), (false, nil), (nil, nil)] + as [(Bool?, Bool?)] + { + let report = BackupHealth.evaluate( + lastSuccess: now.addingTimeInterval(-5 * 3600), lastAttempt: nil, result: 3, now: now, + machineRunning: running, servicesHealthy: services, destinationRegistered: nil, + erroredFiles: 2, outOfSpace: true, queueReadable: false, driveFreeBytes: 1_073_741_824, + localFreeGB: 1, machineFreeGB: 1, punchfsNearFileLimit: true, remoteControlOpen: true) + XCTAssertFalse(report.problems.isEmpty) + XCTAssertEqual( + Set(report.problems.map(\.code)).count, report.problems.count, + "two problems in one report share a code: \(report.problems.map(\.code))") + for problem in report.problems { + XCTAssertNotEqual(problem.code, problem.summary, "no own code: \(problem.summary)") + if let earlier = seen[problem.code] { + XCTAssertEqual(earlier, problem.summary, "one code for two problems: \(problem.code)") + } + seen[problem.code] = problem.summary + } } - XCTAssertEqual( - Set(report.problems.map(\.code)).count, report.problems.count, - "two different problems must not share a code: \(report.problems.map(\.code))") } } diff --git a/mac-app/Tests/CloudMachineAppTests/BackupImageServiceTests.swift b/mac-app/Tests/CloudMachineAppTests/BackupImageServiceTests.swift deleted file mode 100644 index 31488a6..0000000 --- a/mac-app/Tests/CloudMachineAppTests/BackupImageServiceTests.swift +++ /dev/null @@ -1,282 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// Operations on the backup image: mutual exclusion and the verdicts that used -/// to lie - "everything uploaded" with abandoned bands and "image -/// INCONSISTENT" after the device was pulled out from under `fsck`. -final class BackupImageServiceTests: XCTestCase { - - // MARK: - Mutual exclusion - - /// The lock named `BackupImageService.lockName` lives in the real log - /// directory, because `attach`/`detach`/`verify`/`create` use that same lock. - /// We hold it for a fraction of a second and only to check that the - /// operations RESPECT it - none of them gets as far as `hdiutil` then. - private func withHeldImageLock(_ body: () async -> Void) async throws { - let lock = CMLock(name: BackupImageService.lockName) - try XCTSkipUnless( - lock.acquire(), - "the '\(BackupImageService.lockName)' lock is held by something else on this machine") - defer { lock.release() } - await body() - } - - /// The heart of the fix: until 23 September 2026 `withCMLock` was not called - /// from a single place in the repo, so each of these four operations ran with - /// the lock held just as without it. The test checks not only - /// `succeeded == false` (that already came out before, because the buffer is - /// not mounted), but also the content and - more importantly - `disposition`: - /// the operation has to recognise the lock is taken, not bounce off something - /// else along the way. - func testImageOperationsDoNotGetInEachOthersWay() async throws { - try await withHeldImageLock { - for (name, result) in [ - ("create", await BackupImageService.create(sizeGB: 100)), - ("attach", await BackupImageService.attach()), - ("detach", await BackupImageService.detach()), - ("verify", await BackupImageService.verify()), - ] { - XCTAssertFalse(result.succeeded, "\(name) must not report success with the image busy") - XCTAssertTrue( - result.message.contains("another image operation"), - "\(name) has to say that it did NOTHING - got: \(result.message)") - XCTAssertEqual( - result.disposition, .skipped, - "\(name): busy has to be recognisable by the result TYPE, not by the message text") - } - } - } - - /// `attach-image` runs under launchd every 900 s and its exit code lands in - /// `launchd-gdrive-attach.err.log`. A busy image must not be recorded there - /// as a failure - but a real failure MUST be, otherwise we would hide a real - /// error behind exit code 0. - func testBusyIsNotAFailureButAFailureIsStillAFailure() { - XCTAssertEqual( - CMActionResult(succeeded: true, message: "Attached").disposition, .ok) - XCTAssertEqual( - CMActionResult(succeeded: false, message: "busy", didNotRun: true).disposition, - .skipped) - XCTAssertEqual( - CMActionResult(succeeded: false, message: "Could not attach the image").disposition, - .failed, - "no `didNotRun` has to mean a real failure - the default value must not silence errors") - } - - // MARK: - Detach verdict - - private func stats(queued: Int = 0, inProgress: Int = 0, errored: Int = 0) - -> DriveBufferService.QueueStats - { - DriveBufferService.QueueStats( - uploadsInProgress: inProgress, uploadsQueued: queued, files: 10, - erroredFiles: errored, bytesUsed: 1024, outOfSpace: false) - } - - func testEmptyQueueWithoutErrorsMeansEverythingUploaded() { - let result = BackupImageService.detachVerdict(settled: stats()) - XCTAssertTrue(result.succeeded) - XCTAssertTrue(result.message.contains("everything uploaded")) - } - - /// Bands abandoned by rclone drop out of the queue just like uploaded ones, - /// so an empty queue alone reported "Detached, everything uploaded to Google - /// Drive" with data existing ONLY on this Mac. - func testAbandonedBandsAreNotUploaded() { - let result = BackupImageService.detachVerdict(settled: stats(errored: 7)) - XCTAssertFalse(result.succeeded) - XCTAssertFalse(result.message.contains("everything uploaded")) - XCTAssertTrue(result.message.contains("7")) - } - - /// No reading is not success - see `UploadState.queueUnknown`. - func testNoQueueReadingIsNotSuccess() { - let result = BackupImageService.detachVerdict(settled: nil) - XCTAssertFalse(result.succeeded) - } - - // MARK: - Mount table - - /// Until now this list came from parsing `/sbin/mount` output (`" on "` … - /// `" ("`) inside `unmountBrowsedSnapshots()`, so there was nothing to - /// substitute a sample for. A backup snapshot mounts under - /// `/Volumes/.timemachine//.backup/` and keeps the image's - /// device busy, which makes `hdiutil detach` refuse. - func testPicksOnlyBrowsedBackupSnapshots() { - let points = [ - "/", - "/Volumes/CloudMachine", - "/Users/someone/.cloudmachine/drive", - "/Volumes/.timemachine/mac-studio/2026-09-23-101500.backup/CloudMachine", - "/Volumes/.timemachine/mac-studio/2026-09-22-231500.backup/CloudMachine", - // Trap: a similar name, but NOT under the snapshot directory. - "/Volumes/timemachine-copy", - ] - XCTAssertEqual( - BackupImageService.browsedSnapshotMounts(points), - [ - "/Volumes/.timemachine/mac-studio/2026-09-23-101500.backup/CloudMachine", - "/Volumes/.timemachine/mac-studio/2026-09-22-231500.backup/CloudMachine", - ]) - } - - func testNoSnapshotsGivesEmptyList() { - XCTAssertEqual(BackupImageService.browsedSnapshotMounts(["/", "/Volumes/CloudMachine"]), []) - } - - /// The mount table read from the kernel, not from `/sbin/mount`. We check it - /// live, because the whole fix is that this read does NOT start a process - /// and does NOT touch the file system - which a fake would not show. - func testMountTableIsReadableAndContainsRoot() throws { - let points = try XCTUnwrap( - DriveBufferService.mountPoints(), "getmntinfo did not return the mount table") - XCTAssertTrue(points.contains("/"), "every system has the root mounted - got: \(points)") - } - - // MARK: - Attachment state - - /// `.unknown` is NOT `.detached`. `.detached` is a claim ("I checked, it is - /// not there"), while with an unread mount table there was nothing to check. - /// The distinction matters, because on `.detached` `attach` runs - /// `purgeStaleDevices()`, i.e. `detach -force` on a device that may be alive - /// at that moment. - func testUnknownStateIsNeitherAttachedNorDetached() { - let unknown = BackupImageService.Attachment.unknown - XCTAssertNotEqual(unknown, .detached) - XCTAssertNotEqual(unknown, .attached) - XCTAssertFalse( - unknown.isUsable, - "you must not rely on an unknown - Time Machine has no guaranteed destination here") - } - - /// Every state has to give a different sentence. A shared description for - /// `.detached` and `.unknown` would bring back the merging this fix removes - - /// just in the layer a person reads. - func testEveryAttachmentStateHasItsOwnDescription() { - let descriptions = [ - BackupImageService.describe(.attached), - BackupImageService.describe(.detached), - BackupImageService.describe(.dead(errno: ENXIO)), - BackupImageService.describe(.unknown), - // The same state, a DIFFERENT cause: the readability probe did not - // answer in time. The decision is the same (hold off), but the sentence - // for a person must differ - see `attachmentReading()`. - BackupImageService.describe(.unknown, probeTimedOut: true), - ] - XCTAssertEqual( - Set(descriptions).count, descriptions.count, "descriptions repeat: \(descriptions)") - XCTAssertTrue(BackupImageService.describe(.unknown).contains("UNKNOWN")) - XCTAssertTrue( - BackupImageService.describe(.unknown, probeTimedOut: true).contains("probe"), - "the description has to say that the probe did not answer, not the mount table") - } - - // MARK: - Parent device - - /// `fsck_apfs` gets the partition, `hdiutil info` lists the parent device - - /// without this conversion the "did the device survive" check would always - /// answer "no". - func testParentDeviceFromPartition() { - XCTAssertEqual(BackupImageService.parentDevice(of: "/dev/disk7s1"), "/dev/disk7") - XCTAssertEqual(BackupImageService.parentDevice(of: "/dev/disk12s3"), "/dev/disk12") - XCTAssertEqual(BackupImageService.parentDevice(of: "/dev/disk7"), "/dev/disk7") - XCTAssertEqual(BackupImageService.parentDevice(of: "something-else"), "something-else") - } - - // MARK: - Image on the remote - - private let listing = """ - other-data/ - mac-studio.sparsebundle/ - """ - - func testRemoteListingWithImage() { - XCTAssertEqual( - BackupImageService.classifyRemoteListing(succeeded: true, stdout: listing, stderr: ""), - .present) - } - - func testEmptyRemoteListingMeansNoImage() { - XCTAssertEqual( - BackupImageService.classifyRemoteListing(succeeded: true, stdout: "", stderr: ""), - .absent) - } - - /// First run: the remote directory does not exist yet. That is an ANSWER - /// ("there is nothing there"), not the lack of one - otherwise the guard would - /// block `create` in exactly the one case `create` exists for. - func testMissingRemoteDirectoryMeansNoImage() { - XCTAssertEqual( - BackupImageService.classifyRemoteListing( - succeeded: false, stdout: "", - stderr: "2026/09/23 10:00:00 ERROR : : error listing: directory not found"), - .absent) - } - - /// A broken link is NOT proof that the image is absent. Creating the image is - /// irreversible, so lack of certainty has to abort it. - func testNoLinkIsNotProofOfAbsence() { - let result = BackupImageService.classifyRemoteListing( - succeeded: false, stdout: "", - stderr: "Failed to lsf with 2 errors: couldn't connect to Google Drive") - guard case .unknown = result else { - return XCTFail("no answer has to be .unknown, got \(result)") - } - } - - // MARK: - Finding 5: what the detach says about the queue - - /// THE defect. `expireQueuedUploads()` counted only SUCCESSES, so a queue - /// full of items none of which could be sped up came out of it as `0` - - /// exactly like an empty queue. The measured state of this machine at the - /// time of the audit: 462 items, and the log would claim "queue empty". - /// - /// A person reads these lines at the moment of deciding whether the buffer - /// may be deleted - "queue empty" reads there as "nothing is waiting to be - /// uploaded". - func testFullQueueWithoutASingleSuccessIsNotAnEmptyQueue() { - let line = BackupImageService.expiryLogLine( - DriveBufferService.ExpiryOutcome(queued: 462, moved: 0)) - XCTAssertFalse( - line.contains("queue empty"), - "462 queued items are not an empty queue - got: \(line)") - XCTAssertTrue(line.contains("462"), "the number of waiting items must be visible: \(line)") - XCTAssertTrue( - line.contains("NOT ONE could be"), - "the log has to say that the deadlines were NOT moved: \(line)") - } - - /// An empty queue still has to describe itself as empty - otherwise a - /// "fix" consisting of deleting this case would go unnoticed. - func testEmptyQueueStillSaysItIsEmpty() { - XCTAssertTrue( - BackupImageService.expiryLogLine( - DriveBufferService.ExpiryOutcome(queued: 0, moved: 0) - ).contains("queue empty")) - } - - /// A partial failure is not a success either: items without a moved deadline - /// will wait the whole `writeBackSeconds` and the drain will take longer than - /// the line "forced upload of N items" would suggest. - func testPartialMoveSaysHowManyAreLeft() { - let line = BackupImageService.expiryLogLine( - DriveBufferService.ExpiryOutcome(queued: 100, moved: 60)) - XCTAssertTrue(line.contains("60 of 100"), line) - XCTAssertTrue(line.contains("40"), "the missing 40 items must be visible: \(line)") - } - - func testAllMovedGivesThePlainMessage() { - let line = BackupImageService.expiryLogLine( - DriveBufferService.ExpiryOutcome(queued: 12, moved: 12)) - XCTAssertEqual(line, "Detach: forced upload of 12 queued items") - } - - /// No answer from rclone is a third, separate case - it must not be merged - /// with either the empty queue or the failure to move. - func testNoAnswerIsStillASeparateCase() { - let line = BackupImageService.expiryLogLine(nil) - XCTAssertTrue(line.contains("did not answer"), line) - XCTAssertFalse(line.contains("queue empty"), line) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/BackupVisibilityTests.swift b/mac-app/Tests/CloudMachineAppTests/BackupVisibilityTests.swift index b1fd6fb..9ff9122 100644 --- a/mac-app/Tests/CloudMachineAppTests/BackupVisibilityTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/BackupVisibilityTests.swift @@ -7,8 +7,7 @@ import XCTest /// /// - N-W1: the panel suggested `configure-remote --replace-existing` after /// every credential change. The new sign-in (`drive.file`, another -/// `client_id`) does not see the existing image; `create-image` then sees -/// nothing to protect. +/// `client_id`) does not see the existing backup. /// - S4: without `rclone.conf`, `rclone listremotes` exits 0 with nothing, /// which read as "no remote" - and a Mac whose backup lives in /// `mac-studio` got a new folder for good. @@ -28,49 +27,6 @@ final class BackupVisibilityTests: XCTestCase { try super.tearDownWithError() } - private func makeDirectory(_ path: String) throws { - try FileManager.default.createDirectory( - at: directory.appendingPathComponent(path), withIntermediateDirectories: true) - } - - // MARK: - Traces on the local disk - - /// The cache directory carries a suffix when the mount overrides backend - /// options - `gdrive{hekjD}` on this project's own Mac. - func testCachedFoldersAreFoundUnderTheSuffixedRemoteOnly() throws { - try makeDirectory("vfs/gdrive{hekjD}/CloudMachine/mac-studio/mac-studio.sparsebundle") - try makeDirectory("vfs/gdrive2/CloudMachine/someone-else") - try makeDirectory("vfs/gdrive/Other/not-ours") - - XCTAssertEqual(InstallationTraces.cachedFolders(cacheDir: directory), ["mac-studio"]) - XCTAssertEqual( - InstallationTraces.cachedImage( - folder: "mac-studio", imageName: "mac-studio", cacheDir: directory)?.lastPathComponent, - "mac-studio.sparsebundle") - XCTAssertNil( - InstallationTraces.cachedImage( - folder: "home-mac-studio-3", imageName: "home-mac-studio-3", cacheDir: directory)) - } - - func testNoCacheMeansNoTraces() { - XCTAssertEqual(InstallationTraces.cachedFolders(cacheDir: directory), []) - } - - func testRememberedImageIsATraceOnlyForTheSameRemotePath() throws { - let marker = directory.appendingPathComponent("known-image") - InstallationTraces.rememberImage(remotePath: "CloudMachine/mac-studio", file: marker) - - XCTAssertEqual( - InstallationTraces.knownImageTrace( - folder: "mac-studio", imageName: "mac-studio", remotePath: "CloudMachine/mac-studio", - knownImageFile: marker, cacheDir: directory), - marker.path) - XCTAssertNil( - InstallationTraces.knownImageTrace( - folder: "other", imageName: "other", remotePath: "CloudMachine/other", - knownImageFile: marker, cacheDir: directory)) - } - // MARK: - S4: a missing rclone.conf is not "no remote" func testConfigFileAnswerIsRead() throws { @@ -91,39 +47,38 @@ final class BackupVisibilityTests: XCTestCase { RemoteConfigurer.configFileExists(in: ProcessResult(stdout: "", stderr: "x", exitCode: 1))) } - /// THE case: no file, and a mount of this remote has run here. + /// THE case: no file, and this Mac has already set up the machine. func testMissingConfigNextToABackupIsUnknown() { - XCTAssertNil( - RemoteConfigurer.absentAnswer(configFileExists: false, cachedFolders: ["mac-studio"])) - XCTAssertNil( - RemoteConfigurer.absentAnswer(configFileExists: nil, cachedFolders: ["mac-studio"])) + XCTAssertNil(RemoteConfigurer.absentAnswer(configFileExists: false, machineSetUp: true)) + XCTAssertNil(RemoteConfigurer.absentAnswer(configFileExists: nil, machineSetUp: true)) } func testNewMacWithoutConfigHasNoRemote() { - XCTAssertEqual(RemoteConfigurer.absentAnswer(configFileExists: false, cachedFolders: []), false) - XCTAssertEqual(RemoteConfigurer.absentAnswer(configFileExists: nil, cachedFolders: []), false) - // A file rclone read and found no remote in is an answer. XCTAssertEqual( - RemoteConfigurer.absentAnswer(configFileExists: true, cachedFolders: ["mac-studio"]), false) + RemoteConfigurer.absentAnswer(configFileExists: false, machineSetUp: false), false) + XCTAssertEqual(RemoteConfigurer.absentAnswer(configFileExists: nil, machineSetUp: false), false) + // A file rclone read and found no remote in is an answer. + XCTAssertEqual(RemoteConfigurer.absentAnswer(configFileExists: true, machineSetUp: true), false) } - /// Whatever rclone says, a cached `mac-studio` keeps this Mac in it. - func testCachedLegacyFolderKeepsTheLegacyName() { - for remote in [false, nil] as [Bool?] { - let evidence = DriveFolder.legacyEvidence( - remoteConfigured: remote, cachedFolders: ["mac-studio"]) - XCTAssertEqual(evidence, true) - XCTAssertEqual( - DriveFolder.decide( - existing: nil, requested: nil, legacyEvidence: evidence, - machineKey: "home-mac-studio-3"), - .assign("mac-studio")) - } - XCTAssertEqual(DriveFolder.legacyEvidence(remoteConfigured: false, cachedFolders: []), false) - XCTAssertNil(DriveFolder.legacyEvidence(remoteConfigured: nil, cachedFolders: [])) + /// What the old and the new token see is `bands/` in the machine's folder. + func testBackupPresenceIsTheBandsDirectory() { + XCTAssertEqual( + RemoteConfigurer.classifyRemoteListing(succeeded: true, stdout: "bands/\n", stderr: ""), + .present) XCTAssertEqual( - DriveFolder.legacyEvidence(remoteConfigured: false, cachedFolders: ["home-mac-studio-3"]), - false) + RemoteConfigurer.classifyRemoteListing(succeeded: true, stdout: "other/\n", stderr: ""), + .absent) + XCTAssertEqual( + RemoteConfigurer.classifyRemoteListing( + succeeded: false, stdout: "", stderr: "error: directory not found"), + .absent) + if case .unknown = RemoteConfigurer.classifyRemoteListing( + succeeded: false, stdout: "", stderr: "couldn't connect") + { + } else { + XCTFail("an error other than a missing folder is not an answer") + } } // MARK: - N-W1: replacing the connection @@ -157,8 +112,8 @@ final class BackupVisibilityTests: XCTestCase { configured: true, replaceExisting: true, acceptScopeChange: true, remoteName: "gdrive")) } - func testNewTokenMustSeeTheImageTheOldOneSaw() { - // The old token saw the image: the new one has to as well. + func testNewTokenMustSeeTheBackupTheOldOneSaw() { + // The old token saw the backup: the new one has to as well. XCTAssertNil( RemoteConfigurer.replacementVerdict(before: .present, after: .present, remoteName: "gdrive")) XCTAssertEqual( @@ -172,11 +127,4 @@ final class BackupVisibilityTests: XCTestCase { RemoteConfigurer.replacementVerdict(before: .absent, after: .absent, remoteName: "gdrive")) } - /// `create-image` with an image this Mac remembers but Drive does not show. - func testRememberedButInvisibleImageIsNotCreatedAgain() { - XCTAssertNil(BackupImageService.absentImageVerdict(trace: nil)) - let refusal = BackupImageService.absentImageVerdict(trace: "/tmp/known-image") - XCTAssertEqual(refusal?.succeeded, false) - XCTAssertTrue(refusal?.message.contains("/tmp/known-image") == true) - } } diff --git a/mac-app/Tests/CloudMachineAppTests/BufferGuardDecisionTests.swift b/mac-app/Tests/CloudMachineAppTests/BufferGuardDecisionTests.swift deleted file mode 100644 index 180e62a..0000000 --- a/mac-app/Tests/CloudMachineAppTests/BufferGuardDecisionTests.swift +++ /dev/null @@ -1,705 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// Buffer watchdog decisions walked through the WHOLE path - via `step()`, -/// state change included - and not only through the pure helper functions. -/// -/// Reason: all three failures fixed on 23.09.2026 (the discarded `stopbackup` -/// result, the shared resume branch for lack of space on Drive, the made-up -/// zero from a failed `statfs`) lived in a SEQUENCE of steps, not in a single -/// expression. A test of the pure predicate would have passed for each of -/// them. That is why `BufferGuardService` now has injectable `Probes` - the -/// same trick as `preferencesFile` in `BackupHealth.currentReport`. -/// -/// The five failures fixed on 25.09.2026 (the buffer measure, the third state -/// of the size, dead disk protection during a pause, a pause for one run, an -/// unreadable log read as "no problem") lived in the same place and also -/// needed the whole sequence: each of them shows up only in the SECOND or -/// THIRD step, after a state change. -final class BufferGuardDecisionTests: XCTestCase { - - /// Records what the watchdog did and lets us control what it "sees". - /// A class, not a struct, because several closures read and change the same values. - private final class Fake: @unchecked Sendable { - private let lock = NSLock() - - /// UNSENT BACKLOG in GB. The fake translates it into queue ITEMS, because - /// that is exactly how the watchdog sees it (`backlogGB` estimates gigabytes - /// from the number of 32 MiB items). Giving it directly in GB let the fake - /// pretend that rclone reports bytes - and it does not. - private var _backlogGB = 0 - /// rclone cache size. Kept SEPARATE from the backlog, because the whole fix - /// rests on that distinction: with `--vfs-cache-max-age 9999h` the cache sits - /// at the limit permanently (281 measurements in production, minimum 99 GB), - /// regardless of how much is left to upload. Hence 100 by default. - private var _cacheGB: Int? = 100 - /// Whether rclone's remote control answers. `false` = `vfs/stats` returns - /// `nil`, i.e. the production run of 23.09.2026. - private var _statsAvailable = true - private var _outOfSpace = false - private var _freeGB: Int? = 500 - private var _running: Bool? = true - /// `nil` = the rclone log CANNOT BE READ (`-rw-r-----` permissions, moved to - /// `.1` at start-up). - private var _quotaHit: Bool? = false - private var _stalled: Bool? = false - private var _driveFreeBytes: UInt64? = 1_000 * 1_073_741_824 - private var _stopSucceeds = true - private var _stopCalls = 0 - private var _startCalls = 0 - private var _log: [String] = [] - /// What the watchdog reported about the jam. `Bool?`, because "I do not - /// know" MUST reach the report as "I do not know" - otherwise it clears the - /// jam marker. - private var _stallReports: [Bool?] = [] - - private func read(_ body: () -> T) -> T { - lock.lock() - defer { lock.unlock() } - return body() - } - private func write(_ body: () -> Void) { - lock.lock() - defer { lock.unlock() } - body() - } - - var backlogGB: Int { - get { read { _backlogGB } } - set { write { _backlogGB = newValue } } - } - var cacheGB: Int? { - get { read { _cacheGB } } - set { write { _cacheGB = newValue } } - } - var statsAvailable: Bool { - get { read { _statsAvailable } } - set { write { _statsAvailable = newValue } } - } - var outOfSpace: Bool { - get { read { _outOfSpace } } - set { write { _outOfSpace = newValue } } - } - var freeGB: Int? { - get { read { _freeGB } } - set { write { _freeGB = newValue } } - } - var running: Bool? { - get { read { _running } } - set { write { _running = newValue } } - } - var quotaHit: Bool? { - get { read { _quotaHit } } - set { write { _quotaHit = newValue } } - } - var stalled: Bool? { - get { read { _stalled } } - set { write { _stalled = newValue } } - } - var driveFreeBytes: UInt64? { - get { read { _driveFreeBytes } } - set { write { _driveFreeBytes = newValue } } - } - var stopSucceeds: Bool { - get { read { _stopSucceeds } } - set { write { _stopSucceeds = newValue } } - } - var stopCalls: Int { read { _stopCalls } } - var startCalls: Int { read { _startCalls } } - var log: [String] { read { _log } } - var stallReports: [Bool?] { read { _stallReports } } - - func probes() -> BufferGuardService.Probes { - BufferGuardService.Probes( - queueStats: { [self] in - guard statsAvailable else { return nil } - return DriveBufferService.QueueStats( - uploadsInProgress: 0, - // 1 GiB of backlog is 32 bands of 32 MiB - the same way - // `BufferGuardService.backlogGB` computes it. - uploadsQueued: max(0, backlogGB) * 32, - files: 0, erroredFiles: 0, - bytesUsed: UInt64(max(0, cacheGB ?? 0)) * 1_073_741_824, - outOfSpace: outOfSpace) - }, - cacheSizeGB: { [self] _ in cacheGB }, - freeGB: { [self] in freeGB }, - backupRunning: { [self] in running }, - progressPercent: { 0 }, - hitStorageQuota: { [self] in quotaHit }, - uploadStalled: { [self] in stalled }, - driveFreeBytes: { [self] in driveFreeBytes }, - stopBackup: { [self] in - write { _stopCalls += 1 } - return stopSucceeds - }, - startBackup: { [self] in - write { _startCalls += 1 } - return true - }, - // Reporting a jam touches a marker file in the user's directory and shows - // a notification - in the test we only record WHAT it heard. - reportStall: { [self] value in write { _stallReports.append(value) } }, - log: { [self] line in write { _log.append(line) } }) - } - } - - /// Thresholds as in production after 25.09.2026: computed from the UNSENT - /// backlog and lying BELOW the cache size (100 GB). - private let thresholds = BufferGuardService.Thresholds( - highGB: 50, lowGB: 10, minFreeGB: 80, minDriveFreeGB: 30) - - /// Brings the watchdog to the `.running` state - the starting point for the rest. - private func supervising(_ fake: Fake) async -> BufferGuardService { - let watchdog = BufferGuardService(thresholds: thresholds, probes: fake.probes()) - fake.backlogGB = 2 - fake.running = true - await watchdog.step() - let state = await watchdog.currentState() - XCTAssertEqual( - state, .running, "Starting point: the watchdog has to supervise a running backup.") - return watchdog - } - - /// Brings the watchdog to a backlog pause - the starting point for the pause tests. - private func paused(_ fake: Fake) async -> BufferGuardService { - let watchdog = await supervising(fake) - fake.backlogGB = 200 - await watchdog.step() - let state = await watchdog.currentState() - XCTAssertEqual(state, .pausedForBuffer, "Starting point: the watchdog has to be paused.") - return watchdog - } - - // MARK: - Finding 1: the buffer measure and the thresholds - - /// The thresholds MUST lie below the cache size, because they refer to the - /// unsent backlog - i.e. the part of the cache that rclone cannot evict. The - /// old pair (150/40) referred to the size of the WHOLE cache, which is why the - /// pause threshold was unreachable without reaching for another measure, and - /// the resume threshold unreachable at all. - func testThresholdsReferToTheBacklogAndLieBelowTheBufferSize() { - let defaults = BufferGuardService.Thresholds() - XCTAssertEqual(defaults.highGB, DriveBufferService.cacheSizeGB / 2) - XCTAssertEqual(defaults.lowGB, DriveBufferService.cacheSizeGB / 10) - XCTAssertLessThan( - defaults.highGB, DriveBufferService.cacheSizeGB, - "A pause threshold above the cache size is reachable only by measuring a DIFFERENT quantity.") - XCTAssertLessThan( - defaults.lowGB, defaults.highGB, - "Without hysteresis the watchdog would switch state on almost every tick.") - } - - /// The backlog is computed from queue ITEMS, because `vfs/stats` does not - /// report unsent bytes. Check against a production number: 462 items from - /// 23.09.2026, which the owner estimated at "about 15 GB". - func testBacklogEstimateIsComputedFromQueueItems() { - func stats(queued: Int, inProgress: Int = 0, cacheGB: Int = 100) - -> DriveBufferService.QueueStats - { - DriveBufferService.QueueStats( - uploadsInProgress: inProgress, uploadsQueued: queued, files: 0, erroredFiles: 0, - bytesUsed: UInt64(cacheGB) * 1_073_741_824, outOfSpace: false) - } - - XCTAssertEqual(BufferGuardService.backlogGB(stats: stats(queued: 462)), 14) - XCTAssertEqual(BufferGuardService.backlogGB(stats: stats(queued: 32)), 1) - XCTAssertEqual(BufferGuardService.backlogGB(stats: stats(queued: 0)), 0) - // An item being uploaded is not on Drive yet either. - XCTAssertEqual(BufferGuardService.backlogGB(stats: stats(queued: 16, inProgress: 16)), 1) - // A full cache with an empty queue is ZERO backlog - that is the whole - // difference between the old and the new measure. - XCTAssertEqual(BufferGuardService.backlogGB(stats: stats(queued: 0, cacheGB: 100)), 0) - XCTAssertNil( - BufferGuardService.backlogGB(stats: nil), - "No answer from rclone is not zero items.") - } - - /// THE failure, the one from the journal: ONE PAUSE line and ZERO RESUME lines. - /// - /// The 40 GB resume threshold referred to the cache size, and that sits at the - /// 100 GB limit all the time - also when the queue is already empty, because - /// rclone keeps long-uploaded data in the cache (`--vfs-cache-max-age 9999h`). - /// So the resume condition had no way of being met and the watchdog stayed - /// paused until the process restarted. - func testResumesWhenTheQueueIsEmptyEvenThoughTheCacheSitsAtTheLimit() async { - let fake = Fake() - let watchdog = await paused(fake) - - // The upload caught up: queue empty. The cache is still full - and that is - // exactly the state in which the old version never resumed. - fake.backlogGB = 0 - fake.cacheGB = 100 - await watchdog.step() - - let state = await watchdog.currentState() - XCTAssertEqual( - state, .running, - "An empty queue means the upload caught up - a full cache has no right to hold the pause.") - XCTAssertEqual(fake.startCalls, 1) - } - - // MARK: - Finding 6: the buffer size needs a third state - - /// THE failure, reproduced from production step by step (23.09.2026 03:34). - /// - /// rclone does not answer -> the watchdog falls back to the directory walk -> - /// the walk returns 155 GB, because it counts DISK SPACE TAKEN (a measure the - /// cache limit can exceed) -> 155 >= threshold -> irreversible pause. An hour - /// later the monitor wrote "rclone remote control is not answering", i.e. the - /// pause rested on a number taken from the fact that there was no measurement. - /// - /// After the fix this number may appear in the LOG, but not in the decision. - func testNoAnswerFromRcloneDoesNotPauseTheBackup() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.statsAvailable = false - fake.cacheGB = 155 // exactly the number from that single PAUSE line - fake.freeGB = 300 // the disk is in no danger, so a pause could come ONLY from this number - await watchdog.step() - - let state = await watchdog.currentState() - XCTAssertEqual( - state, .running, - "No answer from rclone turned into a number from another measure triggered a pause.") - XCTAssertEqual(fake.stopCalls, 0) - XCTAssertTrue( - fake.log.contains { $0.contains("rclone remote control is not answering") }, - "...but staying silent is not allowed either: the watchdog has just lost the ability to pause the backup." - ) - XCTAssertTrue( - fake.log.contains { $0.contains("155 GB") && $0.contains("DISK SPACE") }, - "Since we give this number, it has to be named as SOMETHING ELSE than the backlog.") - } - - /// Reported once per episode - as for `freeGB()`. With a failure lasting - /// 53 hours a line every 30 seconds would flood the log. - func testNoAnswerWarningIsLoggedOnce() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.statsAvailable = false - await watchdog.step() - await watchdog.step() - await watchdog.step() - - let warnings = fake.log.filter { $0.contains("rclone remote control is not answering") } - XCTAssertEqual(warnings.count, 1, "got: \(fake.log)") - } - - /// The other side of the same lie. When the directory walk FAILED, it returned - /// `0`, and zero looked like an empty buffer - i.e. it lifted a pause put in - /// place because the buffer was full. - func testNoAnswerFromRcloneDoesNotLiftThePause() async { - let fake = Fake() - let watchdog = await paused(fake) - - fake.statsAvailable = false - fake.cacheGB = nil // the directory walk failed too - fake.freeGB = 900 // everything else favours resuming - await watchdog.step() - - let state = await watchdog.currentState() - XCTAssertEqual( - state, .pausedForBuffer, - "Resuming needs proof that the upload caught up - a missing measurement is no proof.") - XCTAssertEqual(fake.startCalls, 0) - } - - // MARK: - Finding 1a: disk protection works in EVERY state - - /// THE failure. `stats?.outOfSpace`, the threshold and `lowDisk` sat ONLY in - /// the `.running` branch. After one pause the watchdog stopped looking at the - /// disk, and the pause branch checked only the resume condition - so rclone - /// could shout "I have nowhere to put data", and at the same moment the - /// watchdog resumed Time Machine because the queue happened to drain. - func testRcloneOutOfSpaceCannotBeIgnoredDuringAPause() async { - let fake = Fake() - let watchdog = await paused(fake) - - fake.backlogGB = 0 // the queue drained, i.e. the resume condition is met - fake.freeGB = 900 - fake.outOfSpace = true // ...but rclone has nowhere to put data - await watchdog.step() - - let state = await watchdog.currentState() - XCTAssertEqual( - state, .pausedForBuffer, - "outOfSpace is a harder fact than our threshold and does not stop being one during a pause.") - XCTAssertEqual(fake.startCalls, 0) - } - - /// The same in a pause for lack of space on Google Drive: proof from - /// `rclone about` has no right to lift the pause when the BUFFER is against the - /// wall. - func testRcloneOutOfSpaceCannotBeIgnoredDuringAGoogleDrivePause() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.quotaHit = true - await watchdog.step() - var state = await watchdog.currentState() - XCTAssertEqual(state, .pausedForQuota) - - fake.quotaHit = false // the entries in the rclone log have aged - fake.backlogGB = 0 - fake.freeGB = 900 - fake.driveFreeBytes = 500 * 1_073_741_824 // space on Drive really did appear - fake.outOfSpace = true // but the buffer has nowhere to put data - await watchdog.step() - - state = await watchdog.currentState() - XCTAssertEqual(state, .pausedForQuota, "Proof about Drive is not proof about the buffer.") - XCTAssertEqual(fake.startCalls, 0) - } - - /// Running out of disk space has to pause also when no backup is running at - /// the moment: macOS starts another one every hour, and the `.idle` branch - /// used to look only at whether a backup had started. - func testLowDiskSpacePausesAlsoWhenNoBackupIsRunning() async { - let fake = Fake() - let watchdog = BufferGuardService(thresholds: thresholds, probes: fake.probes()) - fake.running = false // keeping watch, not supervising - fake.freeGB = 10 // below minFreeGB - await watchdog.step() - - let state = await watchdog.currentState() - XCTAssertEqual( - state, .pausedForBuffer, - "The disk fills up regardless of whether a backup is running this second.") - XCTAssertTrue(fake.log.contains { $0.contains("little free disk space") }) - // There is nothing to pause, so `stopbackup` is not sent - and rightly so: - // its failure would make the watchdog report "Time Machine KEEPS WRITING". - XCTAssertEqual(fake.stopCalls, 0) - } - - // MARK: - Finding 2: a pause lasts as long as we keep it up - - /// THE failure. `tmutil stopbackup` cancels the RUNNING backup and does not - /// touch the schedule, and `stopBackup()` was called only on a state CHANGE. - /// An hour after the pause macOS started another backup, the watchdog did not - /// stop it - and the log said "waiting for the upload". The state lasted 53 - /// hours, the pause of writes one run. - func testPauseIsRepeatedOnEveryTickOfThePause() async { - let fake = Fake() - let watchdog = await paused(fake) - XCTAssertEqual(fake.stopCalls, 1) - - // Time Machine started by itself in its hourly cycle, the backlog is still large. - fake.running = true - await watchdog.step() - XCTAssertEqual(fake.stopCalls, 2, "The next tick MUST repeat the pause.") - await watchdog.step() - XCTAssertEqual(fake.stopCalls, 3) - - let state = await watchdog.currentState() - XCTAssertEqual(state, .pausedForBuffer) - XCTAssertEqual(fake.startCalls, 0) - XCTAssertTrue( - fake.log.contains { $0.contains("repeating the pause") }, - "Repeating the pause is an event worth a trace - it means a backup started during the pause.") - } - - /// ...but we do not repeat without need. When tmutil says plainly that no - /// backup is running, there is nothing to pause - two processes every 30 - /// seconds for 53 hours is over 12 thousand calls for nothing. - func testPauseIsNotRepeatedWhenNoBackupIsRunning() async { - let fake = Fake() - let watchdog = await paused(fake) - XCTAssertEqual(fake.stopCalls, 1) - - fake.running = false - await watchdog.step() - await watchdog.step() - XCTAssertEqual(fake.stopCalls, 1) - - // "I do not know" is NOT "not running" - no answer from tmutil counts as - // a running backup. - fake.running = nil - await watchdog.step() - XCTAssertEqual(fake.stopCalls, 2) - } - - // MARK: - Finding 7: unreadable rclone log - - /// THE failure, in its most dangerous part. `recentLog` returns `nil` for an - /// unopenable file, and `uploadStalled()` turned that into `false`; `false` - /// means "the jam is over", so `reportStall` DELETED the marker and wrote - /// "Upload to Google Drive has resumed" - about an event nobody checked. The - /// rclone log has `-rw-r-----` permissions, and at start-up it is moved to - /// `.1`, so this is not a theoretical case. - func testIDoNotKnowDoesNotClearTheJamMarker() { - XCTAssertEqual( - BufferGuardService.stallAction(stalled: nil, markerExists: true), .doNothing, - "An unreadable log is no proof that the jam is over.") - XCTAssertEqual( - BufferGuardService.stallAction(stalled: nil, markerExists: false), .doNothing) - // Measured answers work as before - otherwise a "fix" consisting of - // switching off the reports would go unnoticed. - XCTAssertEqual(BufferGuardService.stallAction(stalled: true, markerExists: false), .raise) - XCTAssertEqual(BufferGuardService.stallAction(stalled: false, markerExists: true), .clear) - XCTAssertEqual(BufferGuardService.stallAction(stalled: true, markerExists: true), .doNothing) - XCTAssertEqual(BufferGuardService.stallAction(stalled: false, markerExists: false), .doNothing) - } - - /// "I do not know" has to REACH the report as "I do not know". Substituting - /// `false` already in the probe closed the matter before anyone had a chance - /// to think about it. - func testUnreadableLogReachesTheReportAsIDoNotKnow() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.stalled = nil - await watchdog.step() - - XCTAssertEqual(fake.stallReports.count, 2) - XCTAssertEqual(fake.stallReports.first, .some(false)) - XCTAssertNil( - fake.stallReports.last!, "A failed log read has no right to report 'the jam is over'.") - } - - /// An unreadable log does not pause the backup (as it is no proof of a - /// failure), but it must not be kept quiet: the watchdog has just lost the - /// ability to recognise lack of space on Google Drive. - func testUnreadableLogNeitherPausesNorStaysSilent() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.quotaHit = nil - fake.stalled = nil - await watchdog.step() - await watchdog.step() - - let state = await watchdog.currentState() - XCTAssertEqual(state, .running) - XCTAssertEqual(fake.stopCalls, 0) - let warnings = fake.log.filter { $0.contains("cannot read the rclone log") } - XCTAssertEqual(warnings.count, 1, "Once per episode - got: \(fake.log)") - } - - // MARK: - Point 2 of 23.09: a failed pause is NOT a pause - - /// THE failure. `tmutil stopbackup` fails (no permissions or a timeout), and - /// the watchdog moved to `.pausedForBuffer` anyway. Since the pause is called - /// only on a state CHANGE, it never retried it: Time Machine kept writing, the - /// watchdog waited for the drain, the disk filled up completely, and the log - /// said "PAUSE ... waiting for the upload". - func testFailedPauseDoesNotChangeTheStateAndIsRetried() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.stopSucceeds = false - fake.backlogGB = 200 // above the pause threshold - await watchdog.step() - - var state = await watchdog.currentState() - XCTAssertEqual( - state, .running, - "A failed 'tmutil stopbackup' is NOT a pause - Time Machine is still writing.") - XCTAssertEqual(fake.stopCalls, 1) - XCTAssertTrue( - fake.log.contains { $0.contains("FAILED to pause") }, - "A silent failure is worse than a loud one - there has to be a trace in the log.") - - // The next step MUST try again - without that one failed attempt left the - // backup unsupervised until the agent restarted. - await watchdog.step() - XCTAssertEqual(fake.stopCalls, 2, "The watchdog has to retry the pause on every step.") - state = await watchdog.currentState() - XCTAssertEqual(state, .running) - - // When it finally succeeds - only then does the state change. - fake.stopSucceeds = true - await watchdog.step() - state = await watchdog.currentState() - XCTAssertEqual(state, .pausedForBuffer) - XCTAssertEqual(fake.stopCalls, 3) - } - - // MARK: - Point 3 of 23.09: a pause for lack of space on Drive needs PROOF - - /// THE failure. `hitStorageQuota()` looks at entries from the last 30 minutes - /// of the rclone log. After Time Machine is paused no new bands are created, - /// rclone stops trying, the entries age - and the function starts returning - /// `false`, even though Drive has as little space as before. The shared resume - /// branch then looked only at the buffer and LOCAL free space, i.e. at two - /// numbers that know nothing about Google Drive, and lifted the pause - /// immediately. - func testPauseForLackOfSpaceOnDriveDoesNotPassByItself() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.quotaHit = true - await watchdog.step() - var state = await watchdog.currentState() - XCTAssertEqual(state, .pausedForQuota) - - // The log entries have aged, the queue drained, the local disk is empty - - // i.e. EXACTLY the situation in which the old version resumed the backup. - fake.quotaHit = false - fake.backlogGB = 1 - fake.freeGB = 900 - - // 1. rclone does not answer: "I do not know" is NOT consent to resume. - fake.driveFreeBytes = nil - await watchdog.step() - state = await watchdog.currentState() - XCTAssertEqual( - state, .pausedForQuota, - "No answer about Drive capacity has to KEEP the pause, not lift it.") - XCTAssertEqual(fake.startCalls, 0) - - // 2. rclone answers, but there is still practically no space. - fake.driveFreeBytes = 2 * 1_073_741_824 - await watchdog.step() - state = await watchdog.currentState() - XCTAssertEqual(state, .pausedForQuota, "2 GB is no room for further backups.") - XCTAssertEqual(fake.startCalls, 0) - - // 3. Space really did appear - only that is proof. - fake.driveFreeBytes = 500 * 1_073_741_824 - await watchdog.step() - state = await watchdog.currentState() - XCTAssertEqual(state, .running) - XCTAssertEqual(fake.startCalls, 1) - } - - /// A pause because of the BACKLOG needs nothing from Google Drive - otherwise - /// an unreachable rclone would block every resume in the system. - func testBacklogPauseResumesWithoutAskingDrive() async { - let fake = Fake() - let watchdog = await paused(fake) - - fake.backlogGB = 2 - fake.driveFreeBytes = nil // rclone is silent, but this is not that pause - await watchdog.step() - let state = await watchdog.currentState() - XCTAssertEqual(state, .running) - } - - // MARK: - Point 4 of 23.09: failed free-space measurement - - /// THE failure. `freeGB()` returned `0` when `statfs` failed. Zero met the - /// pause condition (`free <= minFreeGB`) immediately and NEVER met the resume - /// condition (`free > minFreeGB`) - the watchdog paused Time Machine based on - /// a number it did not measure, and never resumed it again. - func testFailedFreeSpaceMeasurementDoesNotPauseTheBackup() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.freeGB = nil - fake.backlogGB = 2 // backlog is fine, so the only reason to pause is the disk - await watchdog.step() - - let state = await watchdog.currentState() - XCTAssertEqual( - state, .running, - "No measurement is not a zero measurement - the backup must not be paused on it.") - XCTAssertEqual(fake.stopCalls, 0) - XCTAssertTrue( - fake.log.contains { $0.contains("cannot measure free disk space") }, - "...but it must not be kept quiet either: this is a failure of the disk protection itself.") - } - - /// A measured zero is SOMETHING ELSE than no measurement - and it has to pause. - /// Without this test a "fix" consisting of ignoring free space altogether - /// would go unnoticed. - func testMeasuredZeroStillPausesTheBackup() async { - let fake = Fake() - let watchdog = await supervising(fake) - - fake.freeGB = 0 - await watchdog.step() - - let state = await watchdog.currentState() - XCTAssertEqual(state, .pausedForBuffer) - XCTAssertEqual(fake.stopCalls, 1) - } - - /// No measurement must not PRETEND to be consent to resume either. - func testFailedMeasurementDoesNotResumeTheBackup() async { - let fake = Fake() - let watchdog = await paused(fake) - - fake.backlogGB = 1 - fake.freeGB = nil - await watchdog.step() - let state = await watchdog.currentState() - XCTAssertEqual( - state, .pausedForBuffer, - "Resuming needs proof that the space IS there - a missing measurement is no proof.") - XCTAssertEqual(fake.startCalls, 0) - } - - /// `freeGB()` on the real system has to return the same as `df`, and it has to - /// be an OPTIONAL value. The successful path - the counterpart of the old - /// `testFreeSpaceMatchesStatfs`, which was the only test of this function. - func testFreeSpaceMeasurementMatchesStatfs() { - var stats = statfs() - XCTAssertEqual(statfs("/System/Volumes/Data", &stats), 0) - let expected = Int(UInt64(stats.f_bavail) * UInt64(stats.f_bsize) / 1_073_741_824) - XCTAssertEqual(BufferGuardService.freeGB(), expected) - } - - // MARK: - Pure predicates - - func testResumeNeedsBothConditionsAndAMeasurement() { - // The upload caught up and there is space - the only case that resumes. - XCTAssertTrue( - BufferGuardService.canResumeLocally(backlog: 2, free: 500, thresholds: thresholds)) - // The upload caught up, but the disk is still full. - XCTAssertFalse( - BufferGuardService.canResumeLocally(backlog: 2, free: 10, thresholds: thresholds)) - // The disk is empty, but the backlog has not drained yet. - XCTAssertFalse( - BufferGuardService.canResumeLocally(backlog: 100, free: 500, thresholds: thresholds)) - // No free-space measurement - unknown, so we do not resume. - XCTAssertFalse( - BufferGuardService.canResumeLocally(backlog: 2, free: nil, thresholds: thresholds)) - // No answer about the backlog - the same. - XCTAssertFalse( - BufferGuardService.canResumeLocally(backlog: nil, free: 500, thresholds: thresholds)) - } - - func testDriveSpaceCountsOnlyWhenMeasured() { - XCTAssertFalse(BufferGuardService.driveHasRoom(freeBytes: nil, minGB: 30)) - XCTAssertFalse(BufferGuardService.driveHasRoom(freeBytes: 0, minGB: 30)) - XCTAssertFalse( - BufferGuardService.driveHasRoom(freeBytes: 29 * 1_073_741_824, minGB: 30)) - XCTAssertTrue( - BufferGuardService.driveHasRoom(freeBytes: 30 * 1_073_741_824, minGB: 30)) - } - - /// The "I do not know" branch in the monitor MUST be alive. - /// - /// A review caught the moment when `BackupHealth` compared a non-optional - /// value to `nil` - such a comparison always gives false, so the branch was - /// dead, and the code still compiled and the tests passed. This test checks - /// THE BRANCH ITSELF, not the type: with no measurement a problem has to - /// appear, with a measurement - not. - func testMissingDiskMeasurementIsReportedByTheMonitor() { - let problems = BackupHealth.unmeasuredLocalDiskProblems(localFreeGB: nil) - XCTAssertEqual( - problems.count, 1, "A failed statfs is a failure of the disk protection, not silence.") - // `first`, not `[0]`: if this assertion failed, the index would abort the - // WHOLE run with a fatal error instead of reporting one failed test. - XCTAssertEqual(problems.first?.summary, "Cannot measure free space on the Mac's disk") - - XCTAssertTrue( - BackupHealth.unmeasuredLocalDiskProblems(localFreeGB: 400).isEmpty, - "A successful measurement has no right to report anything.") - // A measured zero is a RESULT, not a lack of one - a low level is reported - // by a separate threshold in `evaluate`, not by this function. - XCTAssertTrue(BackupHealth.unmeasuredLocalDiskProblems(localFreeGB: 0).isEmpty) - } - - /// The Drive free-space threshold is the same one `BackupHealth` treats as a - /// warning - one source of truth, so that the monitor and the watchdog cannot - /// claim different things about the same number. - func testDriveSpaceThresholdMatchesTheMonitor() { - XCTAssertEqual( - BufferGuardService.Thresholds().minDriveFreeGB, BackupHealth.driveFreeWarningGB) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/BufferReadinessTests.swift b/mac-app/Tests/CloudMachineAppTests/BufferReadinessTests.swift deleted file mode 100644 index 245d23e..0000000 --- a/mac-app/Tests/CloudMachineAppTests/BufferReadinessTests.swift +++ /dev/null @@ -1,103 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// Tests of waiting for a ready buffer. -/// -/// Each one replays a specific race that has already happened live - we do -/// not check that the function "works", but that it behaves differently from -/// the version that left Time Machine without a destination on 13 Sep 2026. -final class BufferReadinessTests: XCTestCase { - - /// A clock that moves only when the code would really sleep. Thanks to that - /// the test measures WAITING, not the speed of the machine. - private final class FakeClock { - private(set) var now = Date(timeIntervalSince1970: 1_757_700_000) - func sleep(_ seconds: TimeInterval) { now.addTimeInterval(seconds) } - } - - // MARK: - Readiness condition - - /// This is the second race: the mount is already up, but rclone has not yet - /// read the dirty cache, so the directory is empty. The old version treated - /// that as ready and the attach failed with "No image". - func testMountWithoutImageIsNotReady() { - XCTAssertFalse(BufferReadiness.isReady(mounted: true, imageVisible: false)) - } - - func testImageWithoutMountIsNotReady() { - XCTAssertFalse(BufferReadiness.isReady(mounted: false, imageVisible: true)) - } - - func testBothMeanReady() { - XCTAssertTrue(BufferReadiness.isReady(mounted: true, imageVisible: true)) - } - - // MARK: - Waiting - - /// KNOWN BAD SAMPLE: the buffer comes up after 150 s. The old 120 s limit - /// gave up ten seconds too early, and that is exactly what happened on - /// 13 Sep 2026. - func testWaitsForABufferThatComesUpAfter150s() async { - let clock = FakeClock() - let readyAt = clock.now.addingTimeInterval(150) - - let ready = await BufferReadiness.wait( - now: { clock.now }, - sleep: { clock.sleep($0) }, - probe: { clock.now >= readyAt }) - - XCTAssertTrue(ready, "The buffer came up after 150 s - the wait has to catch it") - } - - /// Proof that the previous test does not pass because the function always - /// returns `true`: a buffer that NEVER comes up must be reported as a failure. - func testGivesUpWhenTheBufferNeverComesUp() async { - let clock = FakeClock() - - let ready = await BufferReadiness.wait( - now: { clock.now }, - sleep: { clock.sleep($0) }, - probe: { false }) - - XCTAssertFalse(ready, "The buffer never came up - this has to be a failure, not silence") - } - - /// The wait has to end roughly at the declared limit, not drag on forever: - /// launchd waits for this process. - func testStopsWaitingAtTheDeclaredLimit() async { - let clock = FakeClock() - let start = clock.now - - _ = await BufferReadiness.wait( - now: { clock.now }, - sleep: { clock.sleep($0) }, - probe: { false }) - - let elapsed = clock.now.timeIntervalSince(start) - XCTAssertGreaterThanOrEqual(elapsed, BufferReadiness.defaultTimeout) - XCTAssertLessThan(elapsed, BufferReadiness.defaultTimeout + BufferReadiness.defaultPoll * 2) - } - - /// A ready buffer must not cost a single sleep - `attach-image` also runs by - /// hand and on every launchd tick. - func testReadyBufferDoesNotWaitAtAll() async { - let clock = FakeClock() - let start = clock.now - - let ready = await BufferReadiness.wait( - now: { clock.now }, - sleep: { clock.sleep($0) }, - probe: { true }) - - XCTAssertTrue(ready) - XCTAssertEqual(clock.now, start, "A ready buffer has to return immediately") - } - - /// The limit has to be greater than the observed 150 s, otherwise the fix is - /// only apparent. Written down explicitly, so that nobody cuts it back to two - /// minutes. - func testLimitIsGreaterThanTheObservedRace() { - XCTAssertGreaterThan(BufferReadiness.defaultTimeout, 150) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/CMLockTests.swift b/mac-app/Tests/CloudMachineAppTests/CMLockTests.swift deleted file mode 100644 index abd9dc9..0000000 --- a/mac-app/Tests/CloudMachineAppTests/CMLockTests.swift +++ /dev/null @@ -1,80 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// Mutual exclusion lock. The tests run on a REAL directory - a file system -/// fake would check nothing here, because the whole mechanism is the atomicity -/// of `mkdir` and whether the written PID can be read back. The directory is -/// private and temporary, so that the test does not touch the production locks -/// in `~/Library/Logs/CloudMachine`. -final class CMLockTests: XCTestCase { - - private var dir: URL! - - override func setUpWithError() throws { - dir = FileManager.default.temporaryDirectory - .appendingPathComponent("CMLockTests-\(UUID().uuidString)") - try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - } - - override func tearDownWithError() throws { - try? FileManager.default.removeItem(at: dir) - } - - private var lockPath: URL { dir.appendingPathComponent("image.lock.d") } - - func testSecondInstanceDoesNotGetAHeldLock() { - let first = CMLock(directory: lockPath) - XCTAssertTrue(first.acquire()) - defer { first.release() } - - let second = CMLock(directory: lockPath) - XCTAssertFalse( - second.acquire(), - "the lock is held by a live process (this test) - a second instance has no right to get it") - } - - func testAfterReleaseTheLockCanBeTakenAgain() { - let first = CMLock(directory: lockPath) - XCTAssertTrue(first.acquire()) - first.release() - XCTAssertFalse( - FileManager.default.fileExists(atPath: lockPath.path), - "release must remove the lock directory, not just forget about it") - - let second = CMLock(directory: lockPath) - XCTAssertTrue(second.acquire()) - second.release() - } - - /// Exactly the state left by a process killed between `createDirectory` and - /// writing the PID: the lock directory exists, the `pid` file does not. A - /// competitor has the right to take it over - because nobody alive claims it. - func testDirectoryWithoutPidIsOrphanedAndCanBeTakenOver() throws { - try FileManager.default.createDirectory(at: lockPath, withIntermediateDirectories: false) - - let lock = CMLock(directory: lockPath) - XCTAssertTrue(lock.acquire()) - defer { lock.release() } - - let pid = try String( - contentsOf: lockPath.appendingPathComponent("pid"), encoding: .utf8) - XCTAssertEqual( - pid.split(separator: "\n").first.map(String.init), "\(getpid())", - "after the takeover the file must hold OUR PID - otherwise the next competitor will consider the lock free" - ) - } - - /// A lock left by a dead process must not stay forever - a watchdog - /// interrupted by SIGKILL does not get to call `release()`. - func testLockOfDeadPidIsTakenOver() throws { - try FileManager.default.createDirectory(at: lockPath, withIntermediateDirectories: false) - // A PID that certainly does not exist: `kill(pid, 0)` refuses with ESRCH. - try "999999\n".write( - to: lockPath.appendingPathComponent("pid"), atomically: true, encoding: .utf8) - - let lock = CMLock(directory: lockPath) - XCTAssertTrue(lock.acquire()) - lock.release() - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/DriveLayerTests.swift b/mac-app/Tests/CloudMachineAppTests/DriveLayerTests.swift index 2dee10a..e1b8d54 100644 --- a/mac-app/Tests/CloudMachineAppTests/DriveLayerTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/DriveLayerTests.swift @@ -2,58 +2,10 @@ import XCTest @testable import CloudMachineCore -/// Tests of the Google Drive layer. They cover parsing and decisions - i.e. -/// the places where bugs were silent and costly, and cannot be seen from the -/// fact that "the backup is being made". +/// The Mac side of Google Drive: the rclone download, free space, and the +/// Drive quota answer. final class DriveLayerTests: XCTestCase { - // MARK: - Parsing hdiutil info - - /// `hdiutil info` groups entries into blocks: all `/dev/diskN` lines after an - /// `image-path` line belong to it, up to the next `image-path`. - private let hdiutilInfo = """ - framework : 595.100.2 - driver : 595.100.2 - ================================================ - image-path : /Users/x/.cloudmachine/drive/other.sparsebundle - image-alias : /Users/x/.cloudmachine/drive/other.sparsebundle - shadow-path : - /dev/disk4\tGUID_partition_scheme\t - /dev/disk4s1\t41504653-0000-11AA-AA11-00306543ECAC\t/Volumes/Other - ================================================ - image-path : /Users/x/.cloudmachine/drive/mac-studio.sparsebundle - image-alias : /Users/x/.cloudmachine/drive/mac-studio.sparsebundle - shadow-path : - /dev/disk7\tEF57347C-0000-11AA-AA11-00306543ECAC\t - /dev/disk7s1\t41504653-0000-11AA-AA11-00306543ECAC\t/Volumes/CloudMachine - """ - - func testParseDevicesFindsOnlyMatchingImage() { - let devices = BackupImageService.parseDevices( - hdiutilInfo: hdiutilInfo, - imagePath: "/Users/x/.cloudmachine/drive/mac-studio.sparsebundle") - XCTAssertEqual(devices, ["/dev/disk7"]) - } - - func testParseDevicesIgnoresOtherImages() { - let devices = BackupImageService.parseDevices( - hdiutilInfo: hdiutilInfo, - imagePath: "/Users/x/.cloudmachine/drive/other.sparsebundle") - XCTAssertEqual(devices, ["/dev/disk4"]) - } - - /// Regression: when the image is not attached, other images' devices must - /// not be returned - detaching them would kill someone's volume. - func testParseDevicesReturnsNothingForUnknownImage() { - let devices = BackupImageService.parseDevices( - hdiutilInfo: hdiutilInfo, imagePath: "/Users/x/nonexistent.sparsebundle") - XCTAssertTrue(devices.isEmpty) - } - - func testParseDevicesHandlesEmptyInput() { - XCTAssertTrue(BackupImageService.parseDevices(hdiutilInfo: "", imagePath: "/x").isEmpty) - } - // MARK: - rclone checksum private let sums = """ @@ -76,486 +28,23 @@ final class DriveLayerTests: XCTestCase { RcloneInstaller.expectedChecksum(sumsContent: sums, zipName: "rclone-v9.9.9-osx-arm64.zip")) } - // MARK: - Mount arguments - - func testMountArgumentsCarryTheNonObviousFlags() { - let args = DriveBufferService.mountArguments() - - // Without this, deleted bands go to Drive's trash and keep counting towards the limit. - XCTAssertTrue(args.contains("--drive-use-trash=false")) - - // After exceeding the daily 750 GB limit rclone is to stop, not spin in - // 403s. - XCTAssertTrue(args.contains("--drive-stop-on-upload-limit")) - - // Without the full cache writes are not buffered, i.e. the whole promise - // of not being interrupted disappears. - XCTAssertTrue(args.contains("--vfs-cache-mode")) - XCTAssertEqual(args[(args.firstIndex(of: "--vfs-cache-mode")! + 1)], "full") - - // The rc interface is the only source of the queue state - without it the - // buffer watchdog is blind. - XCTAssertTrue(args.contains("--rc")) - } - - // MARK: - Remote control interface - - /// REGRESSION 09.10.2026: the interface listened on 127.0.0.1:5572 with no - /// password, so any web page could POST `operations/purge` on the backup - /// folder. It must listen on the private socket and nowhere else. - func testRemoteControlListensOnlyOnThePrivateSocket() { - let args = DriveBufferService.mountArguments() - func value(_ flag: String) -> String? { - args.firstIndex(of: flag).map { args[$0 + 1] } - } - XCTAssertEqual(value("--rc-addr"), "unix://\(DriveBufferService.rcSocket.path)") - XCTAssertEqual( - DriveBufferService.rcSocket.deletingLastPathComponent().path, DriveBufferService.runDir.path) - XCTAssertFalse( - args.contains { - $0.contains("127.0.0.1") || $0.contains(":5572") || $0.contains("localhost") - }, - "the mount still listens on TCP: \(args)") - XCTAssertEqual(args.filter { $0 == "--rc-addr" }.count, 1) - } - - /// The clients must go where the server listens; the old address only while - /// a mount from before the socket is still running. - func testClientsFollowTheMount() { - XCTAssertEqual( - DriveBufferService.rcClientArguments(.socket), - ["--unix-socket", DriveBufferService.rcSocket.path]) - XCTAssertEqual(DriveBufferService.rcClientArguments(.legacyTCP), ["--url", "127.0.0.1:5572"]) - } - - /// rclone cannot bind a longer path - the mount would never start. - func testSocketPathFitsMacOS() { - XCTAssertLessThanOrEqual( - DriveBufferService.rcSocket.path.utf8.count, DriveBufferService.socketPathLimit) - } - - /// The source has one way to the interface. A call built by hand would keep - /// talking to the old address, i.e. a watchdog blind after the next restart. - func testNoCallerBuildsItsOwnRCAddress() throws { - let sources = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() - .appendingPathComponent("Sources") - let files = FileManager.default.enumerator(at: sources, includingPropertiesForKeys: nil)! - .compactMap { $0 as? URL }.filter { $0.pathExtension == "swift" } - XCTAssertFalse(files.isEmpty) - for file in files { - let text = try String(contentsOf: file, encoding: .utf8) - for line in text.split(separator: "\n") - where !line.trimmingCharacters(in: .whitespaces).hasPrefix("//") { - XCTAssertFalse( - line.contains("[\"rc\", "), - "\(file.lastPathComponent): rclone rc called directly: \(line)") - } - } - } - - private func shortTempDir() throws -> URL { - // A socket path must stay under 104 bytes, so not the long scratch paths. - let dir = URL(fileURLWithPath: "/tmp/cm-\(UUID().uuidString.prefix(8))") - try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - return dir - } - - private func bindSocket(_ path: String, listening: Bool) -> Int32 { - let fd = socket(AF_UNIX, SOCK_STREAM, 0) - var address = sockaddr_un() - address.sun_family = sa_family_t(AF_UNIX) - let bytes = Array(path.utf8) - withUnsafeMutableBytes(of: &address.sun_path) { raw in - raw.copyBytes(from: bytes) - raw[bytes.count] = 0 - } - let bound = withUnsafePointer(to: &address) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { - Darwin.bind(fd, $0, socklen_t(MemoryLayout.size)) - } - } - XCTAssertEqual(bound, 0) - if listening { XCTAssertEqual(Darwin.listen(fd, 1), 0) } - return fd - } - - /// A socket left by an rclone that crashed makes the next start fail with - /// "address already in use" - under KeepAlive, a mount that never returns. - func testStaleSocketIsRemovedBeforeStart() throws { - let base = try shortTempDir() - defer { try? FileManager.default.removeItem(at: base) } - let run = base.appendingPathComponent("run") - let sock = run.appendingPathComponent("rc.sock") - try FileManager.default.createDirectory(at: run, withIntermediateDirectories: true) - close(bindSocket(sock.path, listening: false)) - XCTAssertTrue(FileManager.default.fileExists(atPath: sock.path)) - - try DriveBufferService.prepareRCSocketDirectory(run, socket: sock) - XCTAssertFalse(FileManager.default.fileExists(atPath: sock.path)) - } - - /// A socket someone listens on belongs to a running rclone - not ours to take. - func testLiveSocketIsLeftAlone() throws { - let base = try shortTempDir() - defer { try? FileManager.default.removeItem(at: base) } - let run = base.appendingPathComponent("run") - let sock = run.appendingPathComponent("rc.sock") - try FileManager.default.createDirectory(at: run, withIntermediateDirectories: true) - let fd = bindSocket(sock.path, listening: true) - defer { close(fd) } - - try DriveBufferService.prepareRCSocketDirectory(run, socket: sock) - XCTAssertTrue(FileManager.default.fileExists(atPath: sock.path)) - } - - /// Other users must not reach the socket - also when the directory existed - /// with looser permissions. - func testSocketDirectoryIsPrivate() throws { - let base = try shortTempDir() - defer { try? FileManager.default.removeItem(at: base) } - let run = base.appendingPathComponent("run") - try FileManager.default.createDirectory( - at: run, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o755]) - - try DriveBufferService.prepareRCSocketDirectory( - run, socket: run.appendingPathComponent("rc.sock")) - let mode = try FileManager.default.attributesOfItem(atPath: run.path)[.posixPermissions] as? Int - XCTAssertEqual(mode, 0o700) - } - - func testTooLongSocketPathStopsTheStart() { - let run = URL(fileURLWithPath: "/tmp/" + String(repeating: "x", count: 120)) - XCTAssertThrowsError( - try DriveBufferService.prepareRCSocketDirectory( - run, socket: run.appendingPathComponent("rc.sock"))) - } - - func testStatusNamesTheOldOpenInterface() { - XCTAssertTrue( - StatusLines.remoteControl(.legacyTCP, mounted: true).contains("OPEN")) - XCTAssertFalse(StatusLines.remoteControl(.socket, mounted: true).contains("OPEN")) - XCTAssertFalse(StatusLines.remoteControl(.legacyTCP, mounted: false).contains("OPEN")) - } - - /// 02.10.2026: change notifications from Drive (every minute by default) - /// invalidated the `bands` directory after every upload of our own, and - /// reloading it held the lock for ~42 s - the whole mount stood still every - /// minute. - func testMountDoesNotInvalidateTheDirectoryAfterOwnUploads() { - let args = DriveBufferService.mountArguments() - func value(_ flag: String) -> String? { - args.firstIndex(of: flag).map { args[$0 + 1] } - } - XCTAssertEqual(value("--poll-interval"), "0") - XCTAssertEqual(value("--dir-cache-time"), "9999h") - } - - /// The band size was chosen by measurement (`cloudmachine-poc amplification`). A change - /// only takes effect when the image is created, so it must not be missed. - func testBandSizeIs32MB() { - XCTAssertEqual(BackupImageService.bandSectors * 512, 32 * 1024 * 1024) - } - - // MARK: - Watchdog thresholds - - func testGuardThresholdsAreOrdered() { - let t = BufferGuardService.Thresholds() - XCTAssertLessThan( - t.lowGB, t.highGB, - "The resume threshold has to be lower than the pause threshold, otherwise the watchdog will oscillate." - ) - } - /// Free space has to be counted pessimistically, like `df`. The "important /// usage" measure included space taken by snapshots and showed 1202 GB where - /// `df` said 427 GB - the watchdog would have paused too late. + /// `df` said 427 GB. func testFreeSpaceMatchesStatfs() { var stats = statfs() XCTAssertEqual(statfs("/System/Volumes/Data", &stats), 0) let expected = Int(UInt64(stats.f_bavail) * UInt64(stats.f_bsize) / 1_073_741_824) - XCTAssertEqual(BufferGuardService.freeGB(), expected) - } -} - -/// Detection of the Google Drive daily limit. A separate class, because this is -/// the single mistake that stopped a real backup - it deserves its own place. -final class DailyQuotaDetectionTests: XCTestCase { - private let formatter: DateFormatter = { - let f = DateFormatter() - // The sample imitates an rclone log, so it has to look the same on every - // machine. A literal, NOT `DriveBufferService.rcloneLogLocale`: the sample - // generator must not depend on the constant whose correctness we are - // checking - otherwise replacing that constant would switch the generator - // together with the parser and the test would pass in both states. - f.locale = Locale(identifier: "en_US_POSIX") - f.dateFormat = "yyyy/MM/dd HH:mm:ss" - return f - }() - - private func line(_ minutesAgo: Int, _ message: String, now: Date) -> String { - let stamp = formatter.string(from: now.addingTimeInterval(-Double(minutesAgo) * 60)) - return "\(stamp) ERROR : \(message)" - } - - /// THIS is the bug. rclone describes the momentary throttle with the message - /// "Received upload limit error", indistinguishable by text from the daily - /// limit - and retries it by itself. Catching it paused the backup after - /// uploading 109 GiB of the 750 GB allowed per day. - func testTransientRateLimitIsNotTheDailyQuota() { - let now = Date() - let log = [ - line( - 2, - "Received upload limit error: googleapi: Error 403: User rate limit exceeded., userRateLimitExceeded", - now: now), - line(2, "bands/cf9: vfs cache: failed to upload try #1, will retry in 1m0s", now: now), - ].joined(separator: "\n") - XCTAssertFalse(DriveBufferService.logMentionsUploadLimit(log, now: now, within: 30)) - } - - func testRealQuotaErrorIsDetected() { - let now = Date() - let log = line( - 1, - "googleapi: Error 403: The user has exceeded their Drive storage quota, storageQuotaExceeded", - now: now) - XCTAssertTrue(DriveBufferService.logMentionsUploadLimit(log, now: now, within: 30)) - } - - /// Without a time window an alarm once raised would never go out - the entry - /// stays in the log, so the backup would fall into a pause-resume-pause cycle. - func testOldQuotaErrorIsIgnored() { - let now = Date() - let log = line(120, "googleapi: Error 403: storageQuotaExceeded", now: now) - XCTAssertFalse(DriveBufferService.logMentionsUploadLimit(log, now: now, within: 30)) - } - - // MARK: - Finding 15a: a person's calendar must not silence the parser - - /// KNOWN BAD CALENDAR - the one `Locale.current` returns on a Thai Mac. - /// - /// A `DateFormatter` with a fixed `dateFormat` takes the calendar from the - /// locale, so "2026/09/25" parses WITHOUT AN ERROR as Buddhist year 2026, i.e. - /// Gregorian 1483. The date lands 543 years before the window, `stamp < cutoff` - /// ends the loop on the first line and the real disk limit ceases to exist. - func testBuddhistCalendarErasedLimitDetection() { - let now = Date() - let log = line( - 1, - "googleapi: Error 403: The user has exceeded their Drive storage quota, storageQuotaExceeded", - now: now) - XCTAssertFalse( - DriveBufferService.logMentionsUploadLimit( - log, now: now, within: 30, locale: Locale(identifier: "th_TH@calendar=buddhist")), - "this describes the DEFECT, not an expectation - the fix lives in the default locale") - XCTAssertTrue( - DriveBufferService.logMentionsUploadLimit(log, now: now, within: 30), - "the default parser has to read the same log regardless of the person's settings") - } - - func testEmptyLogIsNotAQuotaError() { - XCTAssertFalse(DriveBufferService.logMentionsUploadLimit("", now: Date(), within: 30)) - } -} - -/// Recognising an upload JAM by rclone's behaviour. -/// -/// All ratios below are MEASURED on this machine's production `rclone.log`, not -/// made up. The error text is identical in both cases - if they could be told -/// apart by content, this class would not have to exist. -final class UploadStallDetectionTests: XCTestCase { - private let formatter: DateFormatter = { - let f = DateFormatter() - // The sample imitates an rclone log, so it has to look the same on every - // machine. A literal, NOT `DriveBufferService.rcloneLogLocale`: the sample - // generator must not depend on the constant whose correctness we are - // checking - otherwise replacing that constant would switch the generator - // together with the parser and the test would pass in both states. - f.locale = Locale(identifier: "en_US_POSIX") - f.dateFormat = "yyyy/MM/dd HH:mm:ss" - return f - }() - - /// Builds a sample with the given ratio of successes to errors, in the window. - private func sample(errors: Int, successes: Int, minutesAgo: Int, now: Date) -> String { - let stamp = formatter.string(from: now.addingTimeInterval(-Double(minutesAgo) * 60)) - var lines: [String] = [] - for _ in 0.. BackupHealth.Report { BackupHealth.evaluate( lastSuccess: now.addingTimeInterval(-5 * 3600), lastAttempt: nil, result: 0, now: now, - mounted: true, attached: false, destinationRegistered: nil, erroredFiles: 3, + machineRunning: true, servicesHealthy: false, destinationRegistered: nil, erroredFiles: 3, outOfSpace: false, queueReadable: true) } diff --git a/mac-app/Tests/CloudMachineAppTests/ImageProbeTests.swift b/mac-app/Tests/CloudMachineAppTests/ImageProbeTests.swift deleted file mode 100644 index 758aee9..0000000 --- a/mac-app/Tests/CloudMachineAppTests/ImageProbeTests.swift +++ /dev/null @@ -1,283 +0,0 @@ -import Foundation -import XCTest - -@testable import CloudMachineCore - -/// The image readability probe. Every verdict has a sample that forces it - -/// otherwise a probe always saying "readable" would pass all the tests. -final class ImageProbeTests: XCTestCase { - - private let manifest = URL(fileURLWithPath: "/Volumes/X/backup_manifest.plist") - private let other = URL(fileURLWithPath: "/Volumes/X/other.plist") - - func testReadingAByteMeansAlive() { - let verdict = ImageProbe.probe(regularFiles: { [manifest] }, readFirstByte: { _ in nil }) - XCTAssertEqual(verdict, .readable) - } - - /// Exactly the case of 22 Sep 2026: listing works, reading gives ENXIO. - func testENXIOMeansDead() { - let verdict = ImageProbe.probe(regularFiles: { [manifest] }, readFirstByte: { _ in ENXIO }) - XCTAssertEqual(verdict, .dead(errno: ENXIO)) - } - - func testEIOAlsoMeansDead() { - let verdict = ImageProbe.probe(regularFiles: { [manifest] }, readFirstByte: { _ in EIO }) - XCTAssertEqual(verdict, .dead(errno: EIO)) - } - - /// An error specific to a file (no permission) is not a volume failure - - /// the probe should try the next file, not declare death. - func testEACCESOnOneFileDoesNotMeanDead() { - let verdict = ImageProbe.probe( - regularFiles: { [manifest, other] }, - readFirstByte: { $0 == self.manifest ? EACCES : nil }) - XCTAssertEqual(verdict, .readable) - } - - func testOnlyFilesWithFileErrorsMeansNothingToProbe() { - let verdict = ImageProbe.probe(regularFiles: { [manifest] }, readFirstByte: { _ in EACCES }) - XCTAssertEqual(verdict, .nothingToProbe) - } - - /// A fresh volume before the first backup - there is nothing to read, so we - /// do NOT alarm. An alarm without proof is worse than no alarm. - func testEmptyDirectoryMeansNothingToProbe() { - let verdict = ImageProbe.probe(regularFiles: { [] }, readFirstByte: { _ in ENXIO }) - XCTAssertEqual(verdict, .nothingToProbe) - } - - // MARK: - Listing can fail too - - /// A CHANGE compared with the previous version of this test, which was - /// called `testNieczytelneListowanieToBrakProbki` and checked that EVERY - /// listing error gives `.nothingToProbe`. It encoded a state that turned out - /// to be a hole: `BackupImageService.attachment` maps `.nothingToProbe` to - /// `.attached`, so a dead image passed for a live one, and the - /// `gdrive-attach` agent did not reattach it for hours. What decides now is - /// the SOURCE of the error, not the mere fact of an error: an error without - /// a recognized device errno still proves nothing about the volume and stays - /// `.nothingToProbe`. - func testListingErrorWithoutDeviceErrnoMeansNothingToProbe() { - struct Boom: Error {} - let verdict = ImageProbe.probe(regularFiles: { throw Boom() }, readFirstByte: { _ in nil }) - XCTAssertEqual(verdict, .nothingToProbe) - } - - /// After `--dir-cache-time 5m` expires, listing stops running from the - /// kernel cache and fails with the same ENXIO as reading. Then it is proof - /// of death. - func testENXIOOnListingMeansDead() { - let verdict = ImageProbe.probe( - regularFiles: { throw POSIXError(.ENXIO) }, readFirstByte: { _ in nil }) - XCTAssertEqual(verdict, .dead(errno: ENXIO)) - } - - /// This is what the same error looks like when Foundation throws it: - /// `contentsOfDirectory` wraps the errno in `NSCocoaErrorDomain` and hides - /// the original under `NSUnderlyingErrorKey`. The probe must recognize both - /// forms, because the live path (`regularFiles(in:)`) goes exactly through - /// Foundation. - func testENXIOWrappedInNSErrorAlsoMeansDead() { - let underlying = NSError(domain: NSPOSIXErrorDomain, code: Int(ENXIO)) - let cocoa = NSError( - domain: NSCocoaErrorDomain, code: 256, - userInfo: [NSUnderlyingErrorKey: underlying]) - let verdict = ImageProbe.probe( - regularFiles: { throw cocoa }, readFirstByte: { _ in nil }) - XCTAssertEqual(verdict, .dead(errno: ENXIO)) - } - - /// No permission on the directory is a property of the directory, not a - /// volume failure. - func testEACCESOnListingMeansNothingToProbe() { - let verdict = ImageProbe.probe( - regularFiles: { throw POSIXError(.EACCES) }, readFirstByte: { _ in nil }) - XCTAssertEqual(verdict, .nothingToProbe) - } - - // MARK: - Live path on a real directory - - func testRealReadOnATemporaryDirectory() async throws { - let dir = FileManager.default.temporaryDirectory - .appendingPathComponent("ImageProbeTests-\(UUID().uuidString)") - try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - defer { try? FileManager.default.removeItem(at: dir) } - - var verdict = await ImageProbe.probe(volume: dir) - XCTAssertEqual(verdict, .nothingToProbe, "empty directory") - - try FileManager.default.createDirectory( - at: dir.appendingPathComponent("subdirectory"), withIntermediateDirectories: true) - verdict = await ImageProbe.probe(volume: dir) - XCTAssertEqual(verdict, .nothingToProbe, "a subdirectory alone is not a file") - - try Data("x".utf8).write(to: dir.appendingPathComponent("file")) - verdict = await ImageProbe.probe(volume: dir) - XCTAssertEqual(verdict, .readable) - } - - func testEmptyFileIsAlsoReadable() async throws { - let dir = FileManager.default.temporaryDirectory - .appendingPathComponent("ImageProbeTests-\(UUID().uuidString)") - try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - defer { try? FileManager.default.removeItem(at: dir) } - try Data().write(to: dir.appendingPathComponent("empty")) - let verdict = await ImageProbe.probe(volume: dir) - XCTAssertEqual(verdict, .readable) - } - - // MARK: - Time limit - - /// A probe on a FUSE-T volume can NEVER return - a `read()` stuck in the - /// kernel (state "U" in `ps`) can be neither cancelled nor killed. These - /// tests substitute exactly such a probe: instead of reading, it waits on a - /// semaphore that we release only at the end of the test. - /// - /// EACH of them has its OWN deadline, independent of the limit in the - /// probe. If the limit disappeared from the code, the `await` on the probe - /// would never return and the test would hang until the end of the whole - /// `swift test` - a failure after tens of minutes and without a single - /// sentence about the cause. With a deadline the failure is fast and - /// readable: a `nil` verdict means "the probe did not answer even within - /// this much". - private func verdict( - slot: String, - timeout: TimeInterval, - deadline: TimeInterval = 5, - regularFiles: @escaping @Sendable () throws -> [URL], - readFirstByte: @escaping @Sendable (URL) -> Int32? = { _ in nil } - ) async -> ImageProbe.Verdict? { - let delivered = expectation(description: "probe \(slot) delivered a verdict") - let box = VerdictBox() - Task { - box.set( - await ImageProbe.probe( - slot: slot, timeout: timeout, - regularFiles: regularFiles, readFirstByte: readFirstByte)) - delivered.fulfill() - } - // `XCTWaiter`, not `await fulfillment(of:)`: the latter fails the test by - // itself when the deadline passes, and we want to fail it with our OWN - // sentence saying the probe has no time limit. - _ = XCTWaiter().wait(for: [delivered], timeout: deadline) - return box.value - } - - /// THIS IS THE FIX: a probe that does not answer delivers a verdict in - /// finite time, and the caller survives and keeps working. - func testUnansweredProbeGivesTimedOutAndTheCallerMovesOn() async throws { - let blocked = DispatchSemaphore(value: 0) - // The probe thread sits in "read()" until the end of the test - just like - // on a real dead volume. We release it on exit so it does not stay forever. - defer { blocked.signal() } - - let start = Date() - let result = await verdict( - slot: "test-no-answer", timeout: 0.5, - regularFiles: { - blocked.wait() - return [] - }) - let waited = Date().timeIntervalSince(start) - - XCTAssertEqual( - result, .timedOut, - "a probe without an answer must deliver .timedOut - otherwise the caller hangs along with it" - ) - XCTAssertLessThan( - waited, 3, "the 0.5 s limit must be an UPPER bound on waiting, not a suggestion") - - // The caller not only returned - it STILL WORKS, even though that other - // thread is still sitting in the kernel. This is the part whose absence - // silenced the watchdog permanently. - let directory = FileManager.default.temporaryDirectory - .appendingPathComponent("ImageProbeTests-\(UUID().uuidString)") - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - defer { try? FileManager.default.removeItem(at: directory) } - try Data("x".utf8).write(to: directory.appendingPathComponent("file")) - let afterwards = await ImageProbe.probe(volume: directory) - XCTAssertEqual( - afterwards, .readable, "after giving up on one volume the probe must keep working") - } - - /// "I do not know" is NOT "the image is unreadable". The latter triggers a - /// FORCED detach in `attach-image`, so merging them into one verdict would - /// turn a lack of knowledge into an irreversible operation. - func testTimedOutIsNotTheSameAsDead() { - XCTAssertNotEqual(ImageProbe.Verdict.timedOut, .dead(errno: ENXIO)) - XCTAssertNotEqual(ImageProbe.Verdict.timedOut, .dead(errno: EIO)) - XCTAssertNotEqual(ImageProbe.Verdict.timedOut, .readable) - XCTAssertNotEqual(ImageProbe.Verdict.timedOut, .nothingToProbe) - } - - /// The time limit must not swallow the verdict of a probe that answers - /// SLOWLY, but does answer - otherwise a dead image would stop being fixed. - func testSlowButAnsweringProbeGivesItsVerdict() async { - let result = await verdict( - slot: "test-slow", timeout: 3, - regularFiles: { - Thread.sleep(forTimeInterval: 0.3) - return [self.manifest] - }, - readFirstByte: { _ in ENXIO }) - XCTAssertEqual(result, .dead(errno: ENXIO)) - } - - /// One probe per volume. Without it, a panel refreshed every 10 s would - /// leave one hanging thread per run on a stuck volume. - func testSecondProbeOfTheSameVolumeDoesNotStartASecondThread() async { - let blocked = DispatchSemaphore(value: 0) - // Twice, because if one-in-flight stopped working, TWO threads would be - // blocked and each needs its own wake-up. - defer { - blocked.signal() - blocked.signal() - } - let slot = "test-one-in-flight" - let first = await verdict( - slot: slot, timeout: 0.5, - regularFiles: { - blocked.wait() - return [] - }) - XCTAssertEqual(first, .timedOut) - - // The first thread is still sitting in the kernel. The second caller must - // get "I do not know" RIGHT AWAY - that is why the probe limit here is - // absurdly long (30 s) and the test deadline short (2 s): if waiting - // starts at all, the test fails quickly, not after half a minute. - let start = Date() - let second = await verdict( - slot: slot, timeout: 30, deadline: 2, - regularFiles: { - blocked.wait() - return [] - }) - XCTAssertEqual( - second, .timedOut, - "a second probe of the same volume must return 'I do not know' right away, not wait or start a thread" - ) - XCTAssertLessThan(Date().timeIntervalSince(start), 1) - } -} - -/// The verdict carried from the `Task` to the test body. A class with a lock -/// rather than a captured variable: the write and the read happen on -/// different threads. -private final class VerdictBox: @unchecked Sendable { - private let lock = NSLock() - private var stored: ImageProbe.Verdict? - - func set(_ value: ImageProbe.Verdict) { - lock.lock() - stored = value - lock.unlock() - } - - var value: ImageProbe.Verdict? { - lock.lock() - defer { lock.unlock() } - return stored - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/OpenRemoteControlTests.swift b/mac-app/Tests/CloudMachineAppTests/OpenRemoteControlTests.swift deleted file mode 100644 index 4655408..0000000 --- a/mac-app/Tests/CloudMachineAppTests/OpenRemoteControlTests.swift +++ /dev/null @@ -1,219 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// N-K1 (second audit, 10.10.2026): after `brew upgrade` the old mount keeps -/// listening on 127.0.0.1:5572 without a password, for as long as the Mac is -/// not restarted. Only one line of `drive-status` said so; the window and the -/// watchdog were silent. These tests cover the detection, the notification, -/// and the restart that closes the port without a restart of the Mac. -final class OpenRemoteControlTests: XCTestCase { - - // MARK: - Detection - - func testOldMountOnTheOpenPortIsDetected() { - XCTAssertTrue( - DriveBufferService.legacyPortOpen(transport: .legacyTCP, mounted: true, portAnswers: { true }) - ) - } - - func testEveryConditionIsNeeded() { - // A mount of this version: the socket is there. - XCTAssertFalse( - DriveBufferService.legacyPortOpen(transport: .socket, mounted: true, portAnswers: { true })) - // No mount, or not known: nothing to warn about here (other alarms say so). - XCTAssertFalse( - DriveBufferService.legacyPortOpen( - transport: .legacyTCP, mounted: false, portAnswers: { true })) - XCTAssertFalse( - DriveBufferService.legacyPortOpen(transport: .legacyTCP, mounted: nil, portAnswers: { true })) - // The socket removed under a new mount: nothing listens on TCP. - XCTAssertFalse( - DriveBufferService.legacyPortOpen( - transport: .legacyTCP, mounted: true, portAnswers: { false })) - } - - /// The port probe itself, against a listener of our own - never 5572. - func testPortProbeSeesAListenerAndItsAbsence() throws { - let fd = socket(AF_INET, SOCK_STREAM, 0) - XCTAssertGreaterThanOrEqual(fd, 0) - var address = sockaddr_in() - address.sin_len = UInt8(MemoryLayout.size) - address.sin_family = sa_family_t(AF_INET) - address.sin_port = 0 - address.sin_addr.s_addr = inet_addr("127.0.0.1") - var length = socklen_t(MemoryLayout.size) - let bound = withUnsafeMutablePointer(to: &address) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { - Darwin.bind(fd, $0, length) == 0 && Darwin.listen(fd, 1) == 0 - && getsockname(fd, $0, &length) == 0 - } - } - XCTAssertTrue(bound) - let port = UInt16(bigEndian: address.sin_port) - - XCTAssertTrue(DriveBufferService.tcpPortAnswers(port: port)) - close(fd) - XCTAssertFalse(DriveBufferService.tcpPortAnswers(port: port)) - } - - /// N2: a mount table that could not be read is not "the mount is not running". - func testUnknownMountIsNotReportedAsStopped() { - let line = StatusLines.remoteControl(.legacyTCP, mounted: nil) - XCTAssertFalse(line.contains("is not running"), line) - XCTAssertTrue(line.contains("UNKNOWN"), line) - XCTAssertTrue(StatusLines.remoteControl(.legacyTCP, mounted: true).contains("restart-mount")) - } - - // MARK: - Notification - - private func stateFile() -> URL { - URL(fileURLWithPath: NSTemporaryDirectory()) - .appendingPathComponent("cm-open-port-\(UUID().uuidString).json") - } - - private final class Deliveries: @unchecked Sendable { - var count = 0 - var succeed = true - } - - private func report( - _ open: Bool, at now: Date, file: URL, _ deliveries: Deliveries - ) async -> Bool { - await OpenPortAlert.report( - open: open, now: now, stateFile: file, - deliver: { _, _ in - deliveries.count += 1 - return deliveries.succeed - }, log: { _ in }) - } - - /// Once at once, then after 1, 2, 4 days, then weekly - never twice a day. - func testNotifiesOnceThenWithAGrowingGap() async { - let file = stateFile() - defer { try? FileManager.default.removeItem(at: file) } - let deliveries = Deliveries() - let start = Date(timeIntervalSince1970: 1_800_000_000) - let day: TimeInterval = 86_400 - // The watchdog runs every 30 minutes; walk 30 days of runs. - var times: [TimeInterval] = [] - var now = start - while now < start.addingTimeInterval(30 * day) { - if await report(true, at: now, file: file, deliveries) { - times.append(now.timeIntervalSince(start) / day) - } - now = now.addingTimeInterval(1800) - } - XCTAssertEqual(times.first, 0) - let gaps = zip(times.dropFirst(), times).map { $0 - $1 } - XCTAssertEqual(gaps.prefix(4).map { Int($0.rounded()) }, [1, 2, 4, 7]) - XCTAssertTrue(gaps.dropFirst(3).allSatisfy { Int($0.rounded()) == 7 }) - } - - func testUndeliveredNotificationIsRetriedAtTheNextRun() async { - let file = stateFile() - defer { try? FileManager.default.removeItem(at: file) } - let deliveries = Deliveries() - let now = Date() - deliveries.succeed = false - let first = await report(true, at: now, file: file, deliveries) - XCTAssertFalse(first) - deliveries.succeed = true - let retried = await report(true, at: now.addingTimeInterval(1800), file: file, deliveries) - XCTAssertTrue(retried) - let quiet = await report(true, at: now.addingTimeInterval(3600), file: file, deliveries) - XCTAssertFalse(quiet) - XCTAssertEqual(deliveries.count, 2) - } - - func testClosedPortEndsTheEpisode() async { - let file = stateFile() - defer { try? FileManager.default.removeItem(at: file) } - let deliveries = Deliveries() - let now = Date() - _ = await report(true, at: now, file: file, deliveries) - _ = await report(false, at: now.addingTimeInterval(60), file: file, deliveries) - XCTAssertFalse(FileManager.default.fileExists(atPath: file.path)) - // Open again (say, a downgrade): told at once, not after the old gap. - let again = await report(true, at: now.addingTimeInterval(120), file: file, deliveries) - XCTAssertTrue(again) - XCTAssertEqual(deliveries.count, 2) - } - - // MARK: - Restarting the mount - - private final class Calls: @unchecked Sendable { - var log: [String] = [] - } - - private func steps( - _ calls: Calls, backupRunning: Bool? = false, mounted: Bool? = true, - drained: Bool = true, stopped: Bool = true, cameBack: Bool = true - ) -> MountRestart.Steps { - MountRestart.Steps( - backupRunning: { backupRunning }, - mounted: { mounted }, - detachAndDrain: { - calls.log.append("drain") - return CMActionResult(succeeded: drained, message: drained ? "drained" : "not drained") - }, - stopMount: { - calls.log.append("stop") - return stopped - }, - waitForNewMount: { - calls.log.append("wait") - return cameBack - }, - attach: { - calls.log.append("attach") - return CMActionResult(succeeded: true, message: "attached") - }) - } - - func testRestartRunsTheShutdownPathInOrder() async { - let calls = Calls() - let result = await MountRestart.run(steps(calls)) - XCTAssertTrue(result.succeeded, result.message) - XCTAssertEqual(calls.log, ["drain", "stop", "wait", "attach"]) - } - - /// Nothing is touched while Time Machine writes, or when that is unknown. - func testRestartRefusesDuringABackupOrWithoutAnAnswer() async { - for running in [true, nil] as [Bool?] { - let calls = Calls() - let result = await MountRestart.run(steps(calls, backupRunning: running)) - XCTAssertFalse(result.succeeded) - XCTAssertTrue(result.didNotRun, "a refusal touched nothing - AttachStall must not count it") - XCTAssertEqual(calls.log, []) - } - let calls = Calls() - let result = await MountRestart.run(steps(calls, mounted: nil)) - XCTAssertFalse(result.succeeded) - XCTAssertEqual(calls.log, []) - } - - /// THE rule: the mount is stopped only after everything reached Drive. - /// A drain that did not finish leaves the mount alone and the image back. - func testUnfinishedDrainNeverStopsTheMount() async { - let calls = Calls() - let result = await MountRestart.run(steps(calls, drained: false)) - XCTAssertFalse(result.succeeded) - XCTAssertEqual(calls.log, ["drain", "attach"]) - XCTAssertTrue(result.message.contains("not drained")) - } - - func testRefusedStopPutsTheImageBack() async { - let calls = Calls() - let result = await MountRestart.run(steps(calls, stopped: false)) - XCTAssertFalse(result.succeeded) - XCTAssertEqual(calls.log, ["drain", "stop", "attach"]) - } - - func testMountThatDoesNotComeBackIsReported() async { - let calls = Calls() - let result = await MountRestart.run(steps(calls, cameBack: false)) - XCTAssertFalse(result.succeeded) - XCTAssertEqual(calls.log, ["drain", "stop", "wait"]) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/OrphanMountTests.swift b/mac-app/Tests/CloudMachineAppTests/OrphanMountTests.swift deleted file mode 100644 index a0350e1..0000000 --- a/mac-app/Tests/CloudMachineAppTests/OrphanMountTests.swift +++ /dev/null @@ -1,59 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// When a mount without its rclone gets unmounted - and when it must not. -final class OrphanMountTests: XCTestCase { - - private let start: TimeInterval = 1_793_000_000 - - func testOrphanNeedsSureMountAndSureGone() { - XCTAssertEqual( - OrphanMount.orphanedSince(nil, mounted: true, rclone: .gone, now: start), start) - XCTAssertNil(OrphanMount.orphanedSince(nil, mounted: true, rclone: .running, now: start)) - XCTAssertNil(OrphanMount.orphanedSince(nil, mounted: true, rclone: .unknown, now: start)) - XCTAssertNil(OrphanMount.orphanedSince(nil, mounted: false, rclone: .gone, now: start)) - XCTAssertNil(OrphanMount.orphanedSince(nil, mounted: nil, rclone: .gone, now: start)) - } - - func testOrphanKeepsItsFirstSighting() { - XCTAssertEqual( - OrphanMount.orphanedSince(start, mounted: true, rclone: .gone, now: start + 90), start) - } - - /// Doubt in between starts the wait over instead of counting towards it. - func testDoubtResetsTheClock() { - var since = OrphanMount.orphanedSince(nil, mounted: true, rclone: .gone, now: start) - since = OrphanMount.orphanedSince(since, mounted: true, rclone: .unknown, now: start + 30) - since = OrphanMount.orphanedSince(since, mounted: true, rclone: .gone, now: start + 60) - XCTAssertEqual(since, start + 60) - XCTAssertFalse(OrphanMount.shouldUnmount(orphanedSince: since, now: start + 90)) - } - - /// launchd restarting rclone also leaves the mount listed for a moment. - func testWaitsOutTheGrace() { - XCTAssertFalse(OrphanMount.shouldUnmount(orphanedSince: nil, now: start)) - XCTAssertFalse( - OrphanMount.shouldUnmount(orphanedSince: start, now: start + OrphanMount.grace - 1)) - XCTAssertTrue(OrphanMount.shouldUnmount(orphanedSince: start, now: start + OrphanMount.grace)) - } - - func testExitingAndZombieProcessesDoNotCount() { - XCTAssertTrue(OrphanMount.anyAlive(psStates: "Ss\n")) - XCTAssertTrue(OrphanMount.anyAlive(psStates: "UE\nS+\n")) - XCTAssertFalse(OrphanMount.anyAlive(psStates: "UE\n")) - XCTAssertFalse(OrphanMount.anyAlive(psStates: "Z\n")) - XCTAssertFalse(OrphanMount.anyAlive(psStates: "")) - } - - /// The pattern matches the command line `DriveBufferService` builds. - func testPatternMatchesTheMountCommandLine() throws { - let line = - ([CMTooling.managedRclonePath.path] + DriveBufferService.mountArguments()).joined( - separator: " ") - let regex = try NSRegularExpression(pattern: OrphanMount.rclonePattern) - XCTAssertNotNil(regex.firstMatch(in: line, range: NSRange(line.startIndex..., in: line))) - let other = "/opt/homebrew/bin/rclone mount gdrive: /tmp/x" - XCTAssertNil(regex.firstMatch(in: other, range: NSRange(other.startIndex..., in: other))) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/PullPlugHarnessTests.swift b/mac-app/Tests/CloudMachineAppTests/PullPlugHarnessTests.swift deleted file mode 100644 index de4e2f4..0000000 --- a/mac-app/Tests/CloudMachineAppTests/PullPlugHarnessTests.swift +++ /dev/null @@ -1,118 +0,0 @@ -import Foundation -import XCTest - -@testable import CloudMachineCore -@testable import cloudmachine_poc - -/// Whether the `pullplug` harness tells "I did not measure" apart from "I -/// measured and it is OK". -/// -/// The harness measures the behaviour of hdiutil and FUSE-T, but the WAY it -/// reports on it is ordinary code - and it broke exactly like the rest of this -/// project: by merging a missing result with a result. No test here creates a -/// disk image; they touch only the pure parts. -final class PullPlugHarnessTests: XCTestCase { - - // MARK: - Test write: "never started" is not "interrupted" - - /// THE bug. `writeUntilItBreaks` returned `false` also when - /// `createFile`/`FileHandle` failed immediately - and the harness printed - /// "write interrupted, as expected" for it and ended with "The image survived - /// every floor pull", without having written a single byte. - func testWriteThatHadNowhereToStartIsNotInterrupted() { - let nonexistent = URL(fileURLWithPath: "/no/such/directory/load.bin") - let result = writeUntilItBreaks(to: nonexistent, megabytes: 1) - - guard case .neverStarted(let reason) = result else { - return XCTFail("a write that never started must not look interrupted: \(result)") - } - XCTAssertTrue( - reason.contains("load.bin"), "the reason has to name the file in question: \(reason)") - XCTAssertNotEqual(result, .interrupted(megabytesWritten: 0)) - } - - /// The other side of the same fix: a write that GOT THROUGH the whole order - /// is not a test success either - the floor disappeared only after it, so the - /// round measured nothing. It is the same bug the comment about - /// `arc4random_buf` warns about, only seen from the report's side. - func testWriteToTheEndIsRecognizableAsNoMeasurement() throws { - let directory = FileManager.default.temporaryDirectory - .appendingPathComponent("cm-pullplug-\(UUID().uuidString)") - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - defer { try? FileManager.default.removeItem(at: directory) } - - let result = writeUntilItBreaks( - to: directory.appendingPathComponent("load.bin"), megabytes: 1) - XCTAssertEqual(result, .completed(megabytesWritten: 1)) - } - - // MARK: - fsck: "could not check" is not "inconsistent" - - func testFsckExitZeroIsConsistent() { - XCTAssertEqual( - PullPlugCommand.classify(fsck: ProcessResult(stdout: "ok", stderr: "", exitCode: 0)), - .consistent) - } - - func testNonZeroFsckExitIsInconsistency() { - XCTAssertEqual( - PullPlugCommand.classify( - fsck: ProcessResult(stdout: "", stderr: "corrupt", exitCode: 1)), - .inconsistent) - } - - /// The pattern from `BackupImageService.verifyLocked()`: a `fsck_apfs` that - /// never ran (missing binary, killed process, pulled device) gave the same - /// `false` as a `fsck_apfs` that found damage - the harness then reported - /// "backup lost" and counted an irreversible loss without having a single - /// result. - func testMissingFsckResultIsNotInconsistency() { - let result = PullPlugCommand.classify(fsck: nil) - XCTAssertNotEqual(result, .inconsistent, "a missing result must not pretend to be damage") - guard case .notChecked = result else { return XCTFail("got: \(result)") } - } - - // MARK: - The last sentence of the run - - func testRunWithoutLossesOrGapsDeclaresSurvival() { - let lines = PullPlugCommand.summary( - requestedRounds: 3, executedRounds: 3, lost: 0, unmeasured: 0) - XCTAssertTrue( - lines.contains { $0.contains("survived every floor pull") }, "got: \(lines)") - } - - /// The core of point 14: a run in which anything went unmeasured HAS NO - /// RIGHT to declare that the image survived. The harness does not know - /// whether it tried to kill it. - func testRunWithAnUnmeasuredRoundDoesNotDeclareSurvival() { - let lines = PullPlugCommand.summary( - requestedRounds: 3, executedRounds: 3, lost: 0, unmeasured: 1) - XCTAssertFalse( - lines.contains { $0.contains("survived") }, - "a round without a measurement must not end with a sentence about survival: \(lines)") - XCTAssertTrue( - lines.contains { $0.contains("PROVES NOTHING") }, "got: \(lines)") - XCTAssertTrue( - lines.contains { $0.contains("rounds without a measurement: 1") }, "got: \(lines)") - } - - /// Zero executed rounds is not a success either - and with `rounds: 0` the - /// counter of the old summary came out as "irreversible losses: 0", i.e. - /// "survived". - func testRunWithoutASingleRoundDeclaresNothing() { - let lines = PullPlugCommand.summary( - requestedRounds: 3, executedRounds: 0, lost: 0, unmeasured: 0) - XCTAssertFalse(lines.contains { $0.contains("survived") }, "got: \(lines)") - XCTAssertTrue(lines.contains { $0.contains("MEASURED NOTHING") }, "got: \(lines)") - } - - /// A detected loss is a result and has to stay visible even next to gaps - - /// otherwise the "fix" would turn a false success into a hushed-up failure. - func testDetectedLossDoesNotHideBehindMissingMeasurement() { - let lines = PullPlugCommand.summary( - requestedRounds: 3, executedRounds: 3, lost: 1, unmeasured: 1) - XCTAssertTrue( - lines.contains { $0.contains("architecture loses the backup") }, "got: \(lines)") - XCTAssertFalse(lines.contains { $0.contains("survived") }, "got: \(lines)") - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/QueueStatsParsingTests.swift b/mac-app/Tests/CloudMachineAppTests/QueueStatsParsingTests.swift deleted file mode 100644 index 86f3e17..0000000 --- a/mac-app/Tests/CloudMachineAppTests/QueueStatsParsingTests.swift +++ /dev/null @@ -1,134 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// Parsing responses of rclone's remote control interface. Each case here is a -/// different way in which "I do not know" turned into "zero" - and "zero" read -/// on screen as "Everything uploaded to Google Drive". -final class QueueStatsParsingTests: XCTestCase { - - // MARK: - vfs/stats - - private let fullResponse = """ - { - "diskCache": { - "bytesUsed": 12345678, - "erroredFiles": 3, - "files": 40, - "outOfSpace": false, - "uploadsInProgress": 2, - "uploadsQueued": 386 - }, - "metadataCache": { "dirs": 1, "files": 40 } - } - """ - - func testFullResponseGivesCounters() throws { - let stats = try XCTUnwrap(DriveBufferService.parseQueueStats(fullResponse)) - XCTAssertEqual(stats.uploadsInProgress, 2) - XCTAssertEqual(stats.uploadsQueued, 386) - XCTAssertEqual(stats.files, 40) - XCTAssertEqual(stats.erroredFiles, 3) - XCTAssertEqual(stats.bytesUsed, 12_345_678) - XCTAssertFalse(stats.outOfSpace) - } - - /// The heart of the fix. A response without the `diskCache` section fell - /// through to `?? json`, where none of the counters are, and a missing key - /// gave 0 - the result was a full set of zeros, i.e. `queueKnown == true` and - /// "Everything uploaded". - func testResponseWithoutDiskCacheMeansNotKnowing() { - let withoutSection = """ - { "metadataCache": { "dirs": 1, "files": 40 } } - """ - XCTAssertNil(DriveBufferService.parseQueueStats(withoutSection)) - } - - /// The same bug one level down: the section is there, but the counter is not. - func testMissingCounterMeansNotKnowing() { - let withoutErrors = """ - { - "diskCache": { - "bytesUsed": 1, "files": 40, - "uploadsInProgress": 0, "uploadsQueued": 0 - } - } - """ - XCTAssertNil( - DriveBufferService.parseQueueStats(withoutErrors), - "missing erroredFiles does not mean 'zero errors'") - } - - func testEmptyResponseMeansNotKnowing() { - XCTAssertNil(DriveBufferService.parseQueueStats("")) - XCTAssertNil(DriveBufferService.parseQueueStats("connection refused")) - } - - /// `outOfSpace` is the only field whose absence may be made up with a default - /// value - it is a flag, not a counter. - func testMissingOutOfSpaceFlagDoesNotBreakTheReading() throws { - let withoutFlag = """ - { - "diskCache": { - "bytesUsed": 1, "erroredFiles": 0, "files": 2, - "uploadsInProgress": 0, "uploadsQueued": 0 - } - } - """ - let stats = try XCTUnwrap(DriveBufferService.parseQueueStats(withoutFlag)) - XCTAssertFalse(stats.outOfSpace) - } - - // MARK: - Quiet versus idle - - func testEmptyQueueWithAbandonedBandsIsNotQuiet() { - let stats = DriveBufferService.QueueStats( - uploadsInProgress: 0, uploadsQueued: 0, files: 40, - erroredFiles: 5, bytesUsed: 1024, outOfSpace: false) - XCTAssertTrue(stats.isIdle, "rclone really is doing nothing - hdiutil can work") - XCTAssertFalse( - stats.isQuiet, - "but 5 bands did not reach Drive, so 'everything uploaded' would be a lie") - } - - func testEmptyQueueWithoutErrorsIsQuiet() { - let stats = DriveBufferService.QueueStats( - uploadsInProgress: 0, uploadsQueued: 0, files: 40, - erroredFiles: 0, bytesUsed: 1024, outOfSpace: false) - XCTAssertTrue(stats.isIdle) - XCTAssertTrue(stats.isQuiet) - } - - func testOngoingUploadIsNeitherQuietNorIdle() { - let stats = DriveBufferService.QueueStats( - uploadsInProgress: 1, uploadsQueued: 12, files: 40, - erroredFiles: 0, bytesUsed: 1024, outOfSpace: false) - XCTAssertFalse(stats.isIdle) - XCTAssertFalse(stats.isQuiet) - } - - // MARK: - vfs/queue - - func testQueueSkipsItemsAlreadyUploading() throws { - let response = """ - { - "queue": [ - { "id": 1, "name": "bands/0001", "uploading": false, "expiry": 480.2 }, - { "id": 2, "name": "bands/0002", "uploading": true, "expiry": -1 }, - { "id": 3, "name": "bands/0003", "uploading": false, "expiry": 512.9 } - ] - } - """ - let ids = try XCTUnwrap(DriveBufferService.parseQueueIDs(response)) - XCTAssertEqual(ids, [1, 3], "rclone will not speed up an item already uploading anyway") - } - - /// An empty queue and no answer are TWO DIFFERENT THINGS - both used to come - /// out of `expireQueuedUploads()` as `0`, so the log was silent exactly when - /// the deadlines were NOT moved and the drain could take 10 minutes. - func testEmptyQueueIsNotTheSameAsNoAnswer() { - XCTAssertEqual(DriveBufferService.parseQueueIDs(#"{ "queue": [] }"#), []) - XCTAssertNil(DriveBufferService.parseQueueIDs("")) - XCTAssertNil(DriveBufferService.parseQueueIDs("{}")) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/RcloneLogReadabilityTests.swift b/mac-app/Tests/CloudMachineAppTests/RcloneLogReadabilityTests.swift deleted file mode 100644 index 95a6704..0000000 --- a/mac-app/Tests/CloudMachineAppTests/RcloneLogReadabilityTests.swift +++ /dev/null @@ -1,119 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// Two questions the watchdog asks the rclone LOG: has Google Drive run out of -/// space, and is the upload stalled. Until 25.09.2026 both had only two -/// answers, while the file has three states. -/// -/// `recentLog` returns `nil` when the file cannot be opened, and both -/// functions turned that into `false` - i.e. into "no problem". The effects -/// were two and different: the watchdog did not pause the backup when Drive -/// was out of space, and `reportStall(false)` DELETED the jam marker and wrote -/// "Upload to Google Drive has resumed". The rclone log has `-rw-r-----` -/// permissions, and at start-up it is moved to `.1`, so an unreadable log is -/// an expected state. -/// -/// The tests write to their own temporary file. They do not touch the -/// production `~/.cloudmachine/rclone.log`, neither for reading nor for -/// writing - that is why both functions take a path. -final class RcloneLogReadabilityTests: XCTestCase { - - private var directory: URL! - - override func setUpWithError() throws { - try super.setUpWithError() - directory = URL(fileURLWithPath: NSTemporaryDirectory()) - .appendingPathComponent("cm-rclone-log-\(UUID().uuidString)") - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - } - - override func tearDownWithError() throws { - // Permissions have to come back, otherwise the directory cannot be removed. - if let files = try? FileManager.default.contentsOfDirectory(atPath: directory.path) { - for file in files { - try? FileManager.default.setAttributes( - [.posixPermissions: 0o600], ofItemAtPath: directory.appendingPathComponent(file).path) - } - } - try? FileManager.default.removeItem(at: directory) - try super.tearDownWithError() - } - - private func stamp(_ minutesAgo: Double = 0) -> String { - let formatter = DateFormatter() - formatter.dateFormat = "yyyy/MM/dd HH:mm:ss" - formatter.timeZone = TimeZone.current - return formatter.string(from: Date().addingTimeInterval(-minutesAgo * 60)) - } - - private func write(_ text: String, permissions: Int? = nil) throws -> URL { - let file = directory.appendingPathComponent("rclone.log") - try text.write(to: file, atomically: true, encoding: .utf8) - if let permissions { - try FileManager.default.setAttributes( - [.posixPermissions: permissions], ofItemAtPath: file.path) - } - return file - } - - // MARK: - No space on Google Drive - - func testFreshLimitEntryIsAnExplicitYES() throws { - let file = try write( - """ - \(stamp(5)) INFO : band-1: Copied (replaced existing) - \(stamp(2)) ERROR : band-2: Failed to copy: googleapi: Error 403: storageQuotaExceeded - """) - XCTAssertEqual(DriveBufferService.hitStorageQuotaState(logFile: file), true) - } - - func testLogWithoutATraceOfTheLimitIsAnExplicitNO() throws { - let file = try write("\(stamp(2)) INFO : band-1: Copied (new)") - XCTAssertEqual(DriveBufferService.hitStorageQuotaState(logFile: file), false) - } - - /// THE failure. The file is there, but we have no permission for it - and - /// that does NOT mean there is space on Drive. - func testUnreadableLogIsIDoNotKnowNotNoProblem() throws { - let file = try write( - "\(stamp(2)) ERROR : band-2: googleapi: Error 403: storageQuotaExceeded", permissions: 0o000) - XCTAssertNil( - DriveBufferService.hitStorageQuotaState(logFile: file), - "An unopenable file turned into `false` pretends to be the answer 'no problem'.") - XCTAssertNil( - DriveBufferService.uploadStalledState(logFile: file), - "The same question about the jam - the same file and the same lack of an answer.") - } - - /// The log moved to `.1` at rclone start-up, or not yet created after a - /// fresh install. No file is no data, not no problem. - func testMissingLogFileIsAlsoIDoNotKnow() { - let file = directory.appendingPathComponent("no-such-file.log") - XCTAssertNil(DriveBufferService.hitStorageQuotaState(logFile: file)) - XCTAssertNil(DriveBufferService.uploadStalledState(logFile: file)) - } - - // MARK: - Upload jam - - func testJamRecognisedFromTheRatioOfSuccessesToErrors() throws { - // The `minErrors` threshold is 300 and `maxSuccessRatio` 0.1 - a jam is - // hundreds of errors with almost zero traffic. - var lines: [String] = [] - for _ in 0..<400 { lines.append("\(stamp(3)) ERROR : band: Received upload limit error") } - lines.append("\(stamp(3)) INFO : band: Copied (replaced existing)") - let file = try write(lines.joined(separator: "\n")) - XCTAssertEqual(DriveBufferService.uploadStalledState(logFile: file), true) - } - - func testOrdinaryRateThrottlingIsNotAJam() throws { - // As many successes as errors - measured 1:1 on the throttling of 12.09.2026. - var lines: [String] = [] - for _ in 0..<400 { - lines.append("\(stamp(3)) ERROR : band: Received upload limit error") - lines.append("\(stamp(3)) INFO : band: Copied (replaced existing)") - } - let file = try write(lines.joined(separator: "\n")) - XCTAssertEqual(DriveBufferService.uploadStalledState(logFile: file), false) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/RcloneLogRotationTests.swift b/mac-app/Tests/CloudMachineAppTests/RcloneLogRotationTests.swift deleted file mode 100644 index efc5f82..0000000 --- a/mac-app/Tests/CloudMachineAppTests/RcloneLogRotationTests.swift +++ /dev/null @@ -1,97 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// The rclone log of a running mount grew without limit (257 MB on -/// 10.10.2026), because it was only rotated when the mount started. Trimming -/// it while rclone writes must not blind the stall detection, which reads the -/// last 30 minutes of the same file. -/// -/// Own temporary files only - the production log is neither read nor written. -final class RcloneLogRotationTests: XCTestCase { - - private var directory: URL! - private var log: URL { directory.appendingPathComponent("rclone.log") } - private var previous: URL { directory.appendingPathComponent("rclone.log.1") } - - override func setUpWithError() throws { - try super.setUpWithError() - directory = URL(fileURLWithPath: NSTemporaryDirectory()) - .appendingPathComponent("cm-rclone-rotation-\(UUID().uuidString)") - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - } - - override func tearDownWithError() throws { - try? FileManager.default.removeItem(at: directory) - try super.tearDownWithError() - } - - private func stamp(_ minutesAgo: Double) -> String { - let formatter = DateFormatter() - formatter.dateFormat = "yyyy/MM/dd HH:mm:ss" - formatter.timeZone = TimeZone.current - return formatter.string(from: Date().addingTimeInterval(-minutesAgo * 60)) - } - - /// A jam as the detection sees it: hundreds of limit errors, no uploads. - private func jam(minutesAgo: Double, errors: Int = 400) -> String { - (0.. SetupPlan.Inputs { SetupPlan.Inputs( - hasRclone: true, hasFuse: true, remoteConfigured: true, hasFullDiskAccess: true, - agentsInstalled: true, mounted: true, imageExists: true, imageAttached: true, - timeMachineNotPointingHere: false, connectCommand: "connect", - setDestinationCommand: "sudo set") + hasRclone: true, remoteConfigured: true, hasLima: true, machineExists: true, + machineRunning: true, hasFullDiskAccess: true, agentsInstalled: true, + timeMachineNotPointingHere: false, bundleNeedsPreparing: false, + connectCommand: "connect", setDestinationCommand: "sudo set secret", + maskedSetDestinationCommand: "sudo set ****") } private func actions(_ input: SetupPlan.Inputs) -> [SetupStep.Action?] { @@ -25,65 +26,80 @@ final class SetupPlanTests: XCTestCase { XCTAssertEqual(SetupPlan.steps(inputs()), []) } - func testFreshMacStopsAtGoogleBeforeAnythingThatMounts() { + func testFreshMacStopsAtGoogleBeforeTheMachine() { var input = inputs() input.hasRclone = false - input.hasFuse = false input.remoteConfigured = false + input.hasLima = false + input.machineExists = false + input.machineRunning = false input.hasFullDiskAccess = false input.agentsInstalled = false - input.mounted = false - input.imageExists = nil - input.imageAttached = false let steps = SetupPlan.steps(input) - XCTAssertEqual(steps.map(\.action), [.installRclone, .installFuse, nil]) + XCTAssertEqual(steps.map(\.action), [.installRclone, nil]) XCTAssertEqual(steps.last?.command, "connect") } - func testAfterGoogleTheAgentsComeNext() { + func testLimaIsACommandToCopyBeforeTheMachine() { var input = inputs() - input.hasFullDiskAccess = false - input.agentsInstalled = false - input.mounted = false - input.imageExists = nil - input.imageAttached = false - XCTAssertEqual(actions(input), [.grantFullDiskAccess, .installAgents]) + input.hasLima = false + input.machineExists = nil + input.machineRunning = false + let steps = SetupPlan.steps(input) + XCTAssertEqual(steps.map(\.action), [nil]) + XCTAssertEqual(steps.first?.command, "brew install lima") } - func testMountedWithoutImageOffersToCreateIt() { + func testMissingMachineIsSetUp() { var input = inputs() - input.imageExists = false - input.imageAttached = false - XCTAssertEqual(actions(input), [.createImage]) + input.machineExists = false + input.machineRunning = false + XCTAssertEqual(actions(input), [.setUpMachine]) } - func testExistingImageIsAttachedNotCreated() { + func testStoppedMachineIsStartedNotSetUpAgain() { var input = inputs() - input.imageAttached = false - XCTAssertEqual(actions(input), [.attachImage]) + input.machineRunning = false + XCTAssertEqual(actions(input), [.startMachine]) } - func testUnknownImageStateAsksForNothing() { + func testUnknownMachineAsksForNothingMachineRelated() { var input = inputs() - input.imageExists = nil - input.imageAttached = false + input.machineExists = nil + input.machineRunning = false XCTAssertEqual(SetupPlan.steps(input), []) } - func testNotMountedAsksForNoImageStep() { + func testAfterTheMachineTheAgentsComeNext() { var input = inputs() - input.mounted = false - input.imageExists = false - input.imageAttached = false - XCTAssertEqual(SetupPlan.steps(input), []) + input.hasFullDiskAccess = false + input.agentsInstalled = false + input.timeMachineNotPointingHere = true + XCTAssertEqual(actions(input), [.grantFullDiskAccess, .installAgents]) } - func testTimeMachineStepIsACommandToCopy() { + func testTimeMachineStepShowsTheMaskedCommandAndCopiesTheRealOne() { var input = inputs() input.timeMachineNotPointingHere = true let steps = SetupPlan.steps(input) XCTAssertEqual(steps.map(\.action), [nil]) - XCTAssertEqual(steps.first?.command, "sudo set") + XCTAssertEqual(steps.first?.command, "sudo set secret") + XCTAssertEqual(steps.first?.displayedCommand, "sudo set ****") + } + + func testWithoutPasswordOrAddressTheMachineIsSetUpAgain() { + var input = inputs() + input.timeMachineNotPointingHere = true + input.setDestinationCommand = nil + XCTAssertEqual(actions(input), [.setUpMachine]) + } + + func testNewImageIsPreparedOnlyOnceTimeMachinePointsHere() { + var input = inputs() + input.bundleNeedsPreparing = true + XCTAssertEqual(actions(input), [.prepareBundle]) + input.timeMachineNotPointingHere = true + XCTAssertEqual(actions(input), [nil]) } func testConnectStepIsWhereTheFolderIsChosen() { diff --git a/mac-app/Tests/CloudMachineAppTests/ShutdownPrepTests.swift b/mac-app/Tests/CloudMachineAppTests/ShutdownPrepTests.swift deleted file mode 100644 index 679634a..0000000 --- a/mac-app/Tests/CloudMachineAppTests/ShutdownPrepTests.swift +++ /dev/null @@ -1,57 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// The marker left by preparing for a restart: it must hold the attach agent -/// back until the restart, and never beyond it. -final class ShutdownPrepTests: XCTestCase { - - private let boot: TimeInterval = 1_792_500_000.4 - private var prepared: ShutdownPrep.Marker { - ShutdownPrep.Marker(bootTime: 1_792_500_000, preparedAt: 1_792_560_000) - } - - /// KNOWN BAD SAMPLE: five minutes after the preparation, in the same boot - /// session, the agent attached the image again and undid it. - func testHoldsTheAgentBackInTheSameBootSession() { - XCTAssertTrue(ShutdownPrep.isActive(prepared, bootTime: boot, now: 1_792_560_300)) - } - - /// After the restart the agents must bring the image up as usual. - func testDoesNotHoldAfterARestart() { - XCTAssertFalse(ShutdownPrep.isActive(prepared, bootTime: 1_792_563_000, now: 1_792_563_100)) - } - - /// A preparation without a restart must not leave Time Machine without a - /// destination for good. - func testExpiresAfterTheWindow() { - let end = prepared.preparedAt + ShutdownPrep.window - XCTAssertTrue(ShutdownPrep.isActive(prepared, bootTime: boot, now: end - 1)) - XCTAssertFalse(ShutdownPrep.isActive(prepared, bootTime: boot, now: end)) - } - - /// A clock set back must not stretch the hold into the past. - func testDoesNotHoldBeforeThePreparation() { - XCTAssertFalse(ShutdownPrep.isActive(prepared, bootTime: boot, now: prepared.preparedAt - 60)) - } - - /// No marker, or no boot time - never hold: a doubt goes to attaching, - /// because a missing destination is the louder failure. - func testNoMarkerOrNoBootTimeNeverHolds() { - XCTAssertFalse(ShutdownPrep.isActive(nil, bootTime: boot, now: 1_792_560_300)) - XCTAssertFalse(ShutdownPrep.isActive(prepared, bootTime: nil, now: 1_792_560_300)) - } - - func testMarkerTextRoundTrips() { - XCTAssertEqual(ShutdownPrep.Marker(text: prepared.text), prepared) - XCTAssertNil(ShutdownPrep.Marker(text: "garbage")) - XCTAssertNil(ShutdownPrep.Marker(text: "")) - } - - /// The boot time read live is a real moment in the past. - func testReadsTheBootTime() throws { - let boot = try XCTUnwrap(ShutdownPrep.bootTime()) - XCTAssertLessThan(boot, Date().timeIntervalSince1970) - XCTAssertGreaterThan(boot, Date().timeIntervalSince1970 - 400 * 86_400) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/StatusLinesTests.swift b/mac-app/Tests/CloudMachineAppTests/StatusLinesTests.swift index 7fd00e9..a12a3f0 100644 --- a/mac-app/Tests/CloudMachineAppTests/StatusLinesTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/StatusLinesTests.swift @@ -2,61 +2,10 @@ import XCTest @testable import CloudMachineCore -/// The `drive-status` lines. This is the text a person READS when asking "is -/// the backup working" - and until now it broke exactly where missing data was -/// turned into some value. +/// Status lines shared by `vm-status` and the app. This is the text a person +/// READS when asking "is the backup working". final class StatusLinesTests: XCTestCase { - // MARK: - Mount - - func testMountedAndNotMounted() { - XCTAssertEqual(StatusLines.mounted(true), "OK") - XCTAssertEqual(StatusLines.mounted(false), "MISSING") - } - - /// "MISSING" means "I checked and it is not there". When reading the mount - /// table failed, nobody has the right to draw that conclusion - and the - /// decision to attach the image rests on this answer. - func testUnreadTableIsNotAMissingMount() { - let line = StatusLines.mounted(nil) - XCTAssertNotEqual(line, "MISSING") - XCTAssertNotEqual(line, "OK") - XCTAssertTrue(line.contains("UNKNOWN"), "got: \(line)") - } - - // MARK: - Free space - - func testMeasuredFreeSpaceIsANumber() { - XCTAssertEqual(StatusLines.freeDisk(427), "427 GB") - } - - /// Regression from 23 September 2026: after `BufferGuardService.freeGB()` - /// was changed to `Int?` the line printed `Free on disk: Optional(427) GB`. - /// The compiler reported it as a warning, not an error, so neither the build - /// nor the tests stopped it. - func testMissingMeasurementDoesNotPrintOptional() { - let line = StatusLines.freeDisk(nil) - XCTAssertFalse(line.contains("Optional"), "got: \(line)") - XCTAssertFalse(line.contains("nil"), "got: \(line)") - } - - /// Zero is a CONCRETE number at which the guard pauses Time Machine - - /// substituting it for a missing measurement was the original bug, which - /// the other agent fixed by changing the type. The line must not bring it - /// back through the back door. - func testMissingMeasurementIsNotZero() { - XCTAssertNotEqual(StatusLines.freeDisk(nil), StatusLines.freeDisk(0)) - XCTAssertEqual(StatusLines.freeDisk(0), "0 GB") - } - - func testMissingMeasurementIsNAMED() { - let line = StatusLines.freeDisk(nil) - XCTAssertTrue(line.contains("NOT MEASURED"), "got: \(line)") - XCTAssertTrue( - line.contains("buffer guard"), - "the line must say WHAT follows from it - that the disk is not protected. Got: \(line)") - } - // MARK: - Undelivered alarm func testNoUndeliveredAlarmPrintsNothing() { @@ -64,7 +13,7 @@ final class StatusLinesTests: XCTestCase { } /// The point of the fix in `HealthAlert`: a failed notification must be - /// VISIBLE. As long as `drive-status` was silent about it, the alarm existed + /// VISIBLE. As long as the status was silent about it, the alarm existed /// only in the state file. func testUndeliveredAlarmIsVisible() { let when = Date(timeIntervalSince1970: 1_790_000_000) @@ -83,37 +32,4 @@ final class StatusLinesTests: XCTestCase { text.contains(BackupHealth.stamp(when)), "without a date it is unknown whether the alarm is fresh or from a week ago") } - - // MARK: - Cache vs backlog: TWO different quantities - - /// A single line "Buffer: 103 GB of 100G" answered a question it could not - /// answer: whether the upload keeps up. The cache sits at the limit - /// constantly, and only the second line tells about the backlog - that is - /// why there are two. - func testCacheAndBacklogAreSeparateLines() { - XCTAssertEqual(StatusLines.cacheSize(103, limitGB: 100), "103 GB of 100G") - XCTAssertEqual(StatusLines.backlog(14, items: 462), "~14 GB (462 items)") - } - - /// "~" is not decoration: the backlog gigabytes are ESTIMATED from the item - /// count, and the item count is a measurement. A line giving the estimate as - /// a measurement hides how solid the basis for the decision to pause the - /// backup is. - func testBacklogIsMarkedAsEstimateAndGivesTheMeasurement() { - let line = StatusLines.backlog(14, items: 462) - XCTAssertTrue(line.hasPrefix("~"), "got: \(line)") - XCTAssertTrue(line.contains("462"), "got: \(line)") - } - - /// rclone not answering must not look like zero or like "Optional(0)". - func testNoRcloneAnswerIsNamedInBothLines() { - let cache = StatusLines.cacheSize(nil, limitGB: 100) - XCTAssertTrue(cache.contains("NOT MEASURED"), "got: \(cache)") - XCTAssertFalse(cache.contains("0 GB"), "got: \(cache)") - - let backlog = StatusLines.backlog(nil, items: nil) - XCTAssertTrue(backlog.contains("UNKNOWN"), "got: \(backlog)") - XCTAssertFalse(backlog.contains("Optional"), "got: \(backlog)") - XCTAssertFalse(backlog.contains("~0"), "got: \(backlog)") - } } diff --git a/mac-app/Tests/CloudMachineAppTests/TimeMachineDestinationStateTests.swift b/mac-app/Tests/CloudMachineAppTests/TimeMachineDestinationStateTests.swift index 5e2acdd..b7ac95c 100644 --- a/mac-app/Tests/CloudMachineAppTests/TimeMachineDestinationStateTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/TimeMachineDestinationStateTests.swift @@ -4,50 +4,54 @@ import XCTest @testable import CloudMachineApp /// Three answers of `tmutil destinationinfo` MUST give three different panel -/// states. -/// -/// Until 25.09.2026 the panel asked -/// `TimeMachineStatus.currentDestinationMountPoint()`, which returns `nil` both -/// when there is no destination and when tmutil does not answer - both paths -/// ended in the same `.notRegistered`, i.e. the text "Time Machine does not -/// point to CloudMachine". The direction of the mistake was safe (a false -/// alarm), but the sentence is false and tells the person to do the wrong -/// thing: register a destination that is intact. The `backup-health` watchdog -/// has told these two cases apart since 23.09.2026 (`destinationReading()`); -/// the panel was the last place that merged them. +/// states: the machine's share, no share of ours, and NO ANSWER. A missing +/// answer read as "does not point" tells the person to register a destination +/// that is intact. @MainActor final class TimeMachineDestinationStateTests: XCTestCase { - private let target = "/Volumes/CloudMachine" + private let ours = TimeMachineStatus.Destination( + name: "CloudMachine", kind: "Network", url: "smb://tm@192.168.64.10/CloudMachine", id: "ID-1") // MARK: - Translating the tmutil answer into a panel state - func testRegisteredDestinationIsOurImage() { + func testRegisteredDestinationIsTheMachine() { XCTAssertEqual( - TimeMachineState.from(.mountPoint(target), target: target), - .registered(mountPoint: target)) + TimeMachineState.from([ours], address: "192.168.64.10"), + .registered(url: "smb://tm@192.168.64.10/CloudMachine")) } - /// A destination exists, but points elsewhere - there is NO backup on the - /// Drive. - func testDestinationPointingElsewhereIsNotRegistered() { + /// The machine got a new address and Time Machine still points at the old + /// one: that is not a backup to this machine. + func testDestinationAtAnotherAddressIsNotRegistered() { + XCTAssertEqual(TimeMachineState.from([ours], address: "192.168.64.11"), .notRegistered) + } + + /// `192.168.64.1` must not match `192.168.64.10`. + func testAddressPrefixDoesNotMatch() { + XCTAssertEqual(TimeMachineState.from([ours], address: "192.168.64.1"), .notRegistered) + } + + /// Without the machine's address (it did not answer) any CloudMachine share counts. + func testWithoutAddressAnyMachineShareCounts() { XCTAssertEqual( - TimeMachineState.from(.mountPoint("/Volumes/SomeOtherDisk"), target: target), - .notRegistered) + TimeMachineState.from([ours], address: nil), + .registered(url: "smb://tm@192.168.64.10/CloudMachine")) + } + + func testOtherDestinationIsNotRegistered() { + let disk = TimeMachineStatus.Destination( + name: "Backup", kind: "Local", mountPoint: "/Volumes/Backup", id: "ID-2") + XCTAssertEqual(TimeMachineState.from([disk], address: nil), .notRegistered) } /// tmutil answered and there is no destination - THAT is "does not point". func testNoDestinationIsNotRegistered() { - XCTAssertEqual(TimeMachineState.from(.none, target: target), .notRegistered) + XCTAssertEqual(TimeMachineState.from([], address: nil), .notRegistered) } - /// THAT defect. tmutil not answering has no right to look like a changed - /// destination. `tmutil destinationinfo` reaches the mount living on Google - /// Drive and with a sick mount does not answer at all - and then we know - /// nothing about the destination, which is different information from - /// "there is no destination". func testNoTmutilAnswerIsNotNotRegistered() { - let state = TimeMachineState.from(.noAnswer, target: target) + let state = TimeMachineState.from(nil, address: nil) XCTAssertNotEqual( state, .notRegistered, "tmutil not answering must not pretend to be a changed destination") @@ -85,18 +89,7 @@ final class TimeMachineDestinationStateTests: XCTestCase { /// so that the headline talks about the destination, not about something /// earlier. private func statusApartFrom(timeMachine: TimeMachineState) -> AppStatus { - let status = AppStatus() - status.dependencyState = .ready - status.remoteConfigured = true - var buffer = BufferStatus() - buffer.mounted = true - buffer.imageAttached = true - buffer.queueKnown = true - buffer.freeDiskGB = 400 - status.buffer = buffer - status.backupCycle = BackupCycleStatus( - known: true, lastSuccess: Date().addingTimeInterval(-1800), problems: [], - checkedAt: Date()) + let status = AppStatus.healthyForTests() status.timeMachineState = timeMachine return status } diff --git a/mac-app/Tests/CloudMachineAppTests/TimeMachineStatusTests.swift b/mac-app/Tests/CloudMachineAppTests/TimeMachineStatusTests.swift index 80c8e02..e62eda9 100644 --- a/mac-app/Tests/CloudMachineAppTests/TimeMachineStatusTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/TimeMachineStatusTests.swift @@ -67,59 +67,47 @@ final class TimeMachineStatusTests: XCTestCase { // Regression for the bug of 2026-07-29: the code used to guess the mount // point from a hard-coded volume name instead of asking for the actual - // registered destination - after a manual volume rename (e.g. to - // "TimeMachine") the GUI/watchdogs wrongly showed "no volume". One entry in - // destinationinfo, because the function assumes exactly one active - // destination (an architectural guarantee, see its doc comment) - not "the - // first of many". - func testCurrentDestinationMountPoint_returnsRealMountPoint() { - let singleDestinationInfo = """ - ==================================================== - Name : TimeMachine - Kind : Local - Mount Point : /Volumes/TimeMachine - ID : C4B0056F-6CE8-486E-8726-75B45E2E7A56 - """ - XCTAssertEqual( - TimeMachineStatus.currentDestinationMountPoint( - destinationInfoOutput: singleDestinationInfo), - "/Volumes/TimeMachine") - } - - func testCurrentDestinationMountPoint_noDestinations_returnsNil() { - XCTAssertNil(TimeMachineStatus.currentDestinationMountPoint(destinationInfoOutput: "")) - } + /// The machine's share as this Mac's tmutil prints it (27.10.2026): a + /// network destination has no Mount Point while it is not mounted. + private let networkInfo = """ + ==================================================== + Name : CloudMachine + Kind : Network + URL : smb://tm@192.168.64.10/CloudMachine + ID : 469C129A-962F-4353-BD32-C8E797B4D20B + """ - func testDestinationID_matchesCorrectBlock() { - XCTAssertEqual( - TimeMachineStatus.destinationID( - forMountPointContaining: "/Volumes/TimeMachine", destinationInfoOutput: twoDestinationsInfo - ), - "C4B0056F-6CE8-486E-8726-75B45E2E7A56") + func testBlocksAreParsedInOrder() { + let destinations = TimeMachineStatus.destinations(destinationInfoOutput: twoDestinationsInfo) XCTAssertEqual( - TimeMachineStatus.destinationID( - forMountPointContaining: "/Volumes/Mac", destinationInfoOutput: twoDestinationsInfo), - "DEAD2007-8BC5-4D7B-BCF3-A5646B636CCD") + destinations.map(\.id), + [ + "DEAD2007-8BC5-4D7B-BCF3-A5646B636CCD", "C4B0056F-6CE8-486E-8726-75B45E2E7A56", + ]) + XCTAssertEqual(destinations.map(\.mountPoint), ["/Volumes/Mac", "/Volumes/TimeMachine"]) + XCTAssertEqual(destinations.map(\.quotaGB), [300, nil]) } - func testDestinationQuotaGB_parsesGBValue() { + func testNetworkDestinationHasURLAndNoMountPoint() { + let destinations = TimeMachineStatus.destinations(destinationInfoOutput: networkInfo) XCTAssertEqual( - TimeMachineStatus.destinationQuotaGB( - forMountPointContaining: "/Volumes/Mac", destinationInfoOutput: twoDestinationsInfo), - 300) + destinations, + [ + TimeMachineStatus.Destination( + name: "CloudMachine", kind: "Network", url: "smb://tm@192.168.64.10/CloudMachine", + id: "469C129A-962F-4353-BD32-C8E797B4D20B") + ]) } - func testDestinationQuotaGB_missingQuota_returnsNil() { - XCTAssertNil( - TimeMachineStatus.destinationQuotaGB( - forMountPointContaining: "/Volumes/TimeMachine", destinationInfoOutput: twoDestinationsInfo) - ) + /// A `Name` line starts a new destination even without the separator. + func testBlocksWithoutSeparatorsStillSplit() { + let text = twoDestinationsInfo.replacingOccurrences( + of: "====================================================\n", with: "") + XCTAssertEqual(TimeMachineStatus.destinations(destinationInfoOutput: text).count, 2) } - func testAllDestinationIDs_returnsBothIDs() { - XCTAssertEqual( - TimeMachineStatus.allDestinationIDs(destinationInfoOutput: twoDestinationsInfo), - ["DEAD2007-8BC5-4D7B-BCF3-A5646B636CCD", "C4B0056F-6CE8-486E-8726-75B45E2E7A56"]) + func testNoDestinationsParseToNone() { + XCTAssertEqual(TimeMachineStatus.destinations(destinationInfoOutput: ""), []) } // MARK: - What counts as an answer @@ -154,7 +142,7 @@ final class TimeMachineStatusTests: XCTestCase { from: ProcessResult( stdout: "tmutil: No destinations configured.\n", stderr: "", exitCode: code)) XCTAssertNotNil(answer) - XCTAssertNil(TimeMachineStatus.currentDestinationMountPoint(destinationInfoOutput: answer!)) + XCTAssertEqual(TimeMachineStatus.destinations(destinationInfoOutput: answer!), []) } } } diff --git a/mac-app/Tests/CloudMachineAppTests/TimeMachineTimeoutTests.swift b/mac-app/Tests/CloudMachineAppTests/TimeMachineTimeoutTests.swift index 5101921..9a70766 100644 --- a/mac-app/Tests/CloudMachineAppTests/TimeMachineTimeoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/TimeMachineTimeoutTests.swift @@ -5,7 +5,7 @@ import XCTest /// What happens when `tmutil` DOES NOT ANSWER. /// /// Until 23.09.2026 the answer was "nothing": `tmutil` calls had no time -/// limit, so with a dead FUSE-T mount (the ENXIO incident of 22.09) +/// limit, so with a dead destination (the ENXIO incident of 22.09) /// `destinationinfo` entered uninterruptible I/O and `BackupHealth. /// currentReport()` never returned. launchd with `StartInterval` does not /// start a second instance while the first one is alive - the watchdog went @@ -41,13 +41,13 @@ final class TimeMachineTimeoutTests: XCTestCase { /// This distinction is the whole point of the time limit. If a hung tmutil /// reported itself as `destinationRegistered: false`, the watchdog would /// send the person off to change a destination that is set correctly - and - /// the real cause (a dead mount) would be left untouched. + /// the real cause (a hung destination) would be left untouched. func testNoTmutilAnswerIsASeparateFailure() { let now = Date(timeIntervalSince1970: 1_758_000_000) let unknown = BackupHealth.evaluate( lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), - result: 0, now: now, mounted: true, attached: true, destinationRegistered: nil, + result: 0, now: now, machineRunning: true, servicesHealthy: true, destinationRegistered: nil, erroredFiles: 0, outOfSpace: false, queueReadable: true) XCTAssertFalse(unknown.healthy, "Unknown = not healthy.") @@ -66,7 +66,8 @@ final class TimeMachineTimeoutTests: XCTestCase { let changed = BackupHealth.evaluate( lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), - result: 0, now: now, mounted: true, attached: true, destinationRegistered: false, + result: 0, now: now, machineRunning: true, servicesHealthy: true, + destinationRegistered: false, erroredFiles: 0, outOfSpace: false, queueReadable: true) XCTAssertTrue( @@ -80,84 +81,58 @@ final class TimeMachineTimeoutTests: XCTestCase { let fine = BackupHealth.evaluate( lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), - result: 0, now: now, mounted: true, attached: true, destinationRegistered: true, + result: 0, now: now, machineRunning: true, servicesHealthy: true, destinationRegistered: true, erroredFiles: 0, outOfSpace: false, queueReadable: true) XCTAssertTrue(fine.healthy) } - // MARK: - Unread mount table + // MARK: - The machine: no answer is not "stopped" /// The reference point for this group: everything working. private func healthy( - mounted: Bool? = true, attached: Bool? = true, imageDeadErrno: Int32? = nil + machineRunning: Bool? = true, servicesHealthy: Bool? = true ) -> BackupHealth.Report { let now = Date(timeIntervalSince1970: 1_758_000_000) return BackupHealth.evaluate( lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), - result: 0, now: now, mounted: mounted, attached: attached, destinationRegistered: true, - erroredFiles: 0, outOfSpace: false, queueReadable: true, imageDeadErrno: imageDeadErrno) + result: 0, now: now, machineRunning: machineRunning, servicesHealthy: servicesHealthy, + destinationRegistered: true, erroredFiles: 0, outOfSpace: false, queueReadable: true) } - /// THAT silence. `BackupImageService.Attachment` got a fourth case - /// `.unknown` ("the mount table could not be read"), and the caller passed - /// `attachment != .detached` to the watchdog - i.e. `.unknown` went in as - /// `true`, "attached". The watchdog, whose ONLY job is not to claim things - /// it does not know, stayed silent about a state it did not know. This was - /// the only place in this round where "I do not know" went towards silence - /// rather than an alarm. - func testUnreadImageStateIsNotSilence() { - let unknown = healthy(attached: nil) - + func testUnknownMachineIsNotSilence() { + let unknown = healthy(machineRunning: nil, servicesHealthy: nil) XCTAssertFalse(unknown.healthy, "Unknown = not healthy. Silence is not allowed here.") XCTAssertTrue( unknown.problems.contains { - $0.summary == "Unknown whether the backup image is attached" + $0.summary == "Unknown whether the CloudMachine machine is running" }) } - /// ...and it is not the same as a real detachment. The message "is not - /// attached" would send the person off to attach an image that may be - /// attached correctly - the same principle as with a changed destination. - func testUnreadImageStateSoundsDifferentFromDetachment() { - let unknown = healthy(attached: nil) - let detached = healthy(attached: false) - + /// "Is not running" sends a person off to start a machine that may run. + func testUnknownMachineSoundsDifferentFromStoppedMachine() { + let unknown = healthy(machineRunning: nil, servicesHealthy: nil) + let stopped = healthy(machineRunning: false, servicesHealthy: nil) XCTAssertFalse( - unknown.problems.contains { $0.summary == "The backup image is not attached" }, - "Not reading is NOT a detachment.") - XCTAssertTrue(detached.problems.contains { $0.summary == "The backup image is not attached" }) + unknown.problems.contains { $0.summary == "The CloudMachine machine is not running" }) + XCTAssertTrue( + stopped.problems.contains { $0.summary == "The CloudMachine machine is not running" }) XCTAssertFalse( - detached.problems.contains { $0.summary.hasPrefix("Unknown whether the backup image") }, - "A real detachment is a FACT, not an unknown.") + stopped.problems.contains { $0.summary.hasPrefix("Unknown whether") }, + "A stopped machine is a FACT, not an unknown.") } - /// The same for the mount. `isMounted` is `mountedState() ?? false`, so an - /// unread mount table reported itself as "mount is not working" - an alarm - /// about a state nobody measured, sending the person off to fix something - /// that may be fine. - func testUnreadMountSoundsDifferentFromMissingMount() { - let unknown = healthy(mounted: nil) - let missing = healthy(mounted: false) - - XCTAssertFalse(unknown.healthy) + /// Running, but nothing inside answered: its own problem, not "a service is down". + func testSilentMachineSoundsDifferentFromAServiceDown() { + let silent = healthy(servicesHealthy: nil) + let down = healthy(servicesHealthy: false) XCTAssertTrue( - unknown.problems.contains { - $0.summary == "Unknown whether the Google Drive mount is working" + silent.problems.contains { $0.summary == "The CloudMachine machine does not answer" }) + XCTAssertFalse(silent.problems.contains { $0.summary.hasPrefix("A service") }) + XCTAssertTrue( + down.problems.contains { + $0.summary == "A service in the CloudMachine machine is not running" }) - XCTAssertFalse( - unknown.problems.contains { $0.summary == "Google Drive mount is not working" }) - XCTAssertTrue(missing.problems.contains { $0.summary == "Google Drive mount is not working" }) - } - - /// An attached but DEAD image must still be reported - rebuilding the - /// `attached` branch into three states must not lose this case, because it - /// has already cost 15 hours without a backup. - func testDeadImageIsStillReported() { - let dead = healthy(attached: true, imageDeadErrno: 6) - XCTAssertTrue(dead.problems.contains { $0.summary.contains("DEAD (errno 6)") }) - // With an unknown state we do NOT guess that the image is alive. - XCTAssertFalse(healthy(attached: nil).problems.contains { $0.summary.contains("DEAD") }) } /// `runningState()` exists so that "not in progress" and "unknown" can be diff --git a/mac-app/Tests/CloudMachineAppTests/UploadDrainTests.swift b/mac-app/Tests/CloudMachineAppTests/UploadDrainTests.swift deleted file mode 100644 index af8f5d5..0000000 --- a/mac-app/Tests/CloudMachineAppTests/UploadDrainTests.swift +++ /dev/null @@ -1,84 +0,0 @@ -import XCTest - -@testable import CloudMachineCore - -/// Tests of waiting for the upload before `hdiutil attach`. -/// -/// They replay the start-up of 01.10.2026: a backlog of ~600 bands draining -/// over ~6 min. The old wait (a fixed 120 s) gave up halfway and hdiutil -/// started in the middle of the full upload. -final class UploadDrainTests: XCTestCase { - - private final class FakeClock { - private(set) var now = Date(timeIntervalSince1970: 1_757_700_000) - func sleep(_ seconds: TimeInterval) { now.addTimeInterval(seconds) } - } - - /// KNOWN BAD SAMPLE: 600 items drain at ~2 per second, i.e. ~5 min. - /// A fixed 120 s would have let hdiutil go with ~360 items in the queue. - func testWaitsForABacklogThatTakesLongerThanTwoMinutes() async { - let clock = FakeClock() - let start = clock.now - let outcome = await UploadDrain.wait( - now: { clock.now }, sleep: { clock.sleep($0) }, expire: {}, - unsent: { max(0, 600 - Int(clock.now.timeIntervalSince(start) * 2)) }) - XCTAssertEqual(outcome, .idle) - XCTAssertGreaterThanOrEqual(clock.now.timeIntervalSince(start), 300) - } - - /// Daily limit exhausted: the queue stands still. We then do not wait the full - /// 20 min, only `stallTimeout`. - func testGivesUpWhenTheQueueStandsStill() async { - let clock = FakeClock() - let start = clock.now - let outcome = await UploadDrain.wait( - now: { clock.now }, sleep: { clock.sleep($0) }, expire: {}, unsent: { 42 }) - XCTAssertEqual(outcome, .stalled(unsent: 42)) - let elapsed = clock.now.timeIntervalSince(start) - XCTAssertGreaterThanOrEqual(elapsed, UploadDrain.defaultStallTimeout) - XCTAssertLessThan(elapsed, UploadDrain.defaultStallTimeout + UploadDrain.defaultPoll * 2) - } - - /// Upward swings (TM adding writes) are not progress - only a new minimum counts. - func testSwingsWithoutANewMinimumAreNotProgress() async { - let clock = FakeClock() - var flip = false - let outcome = await UploadDrain.wait( - now: { clock.now }, sleep: { clock.sleep($0) }, expire: {}, - unsent: { - flip.toggle() - return flip ? 50 : 60 - }) - XCTAssertEqual(outcome, .stalled(unsent: 50)) - } - - /// Hard ceiling: the queue drains, but more slowly than needed. - func testHardCeilingOnSlowProgress() async { - let clock = FakeClock() - let start = clock.now - let outcome = await UploadDrain.wait( - now: { clock.now }, sleep: { clock.sleep($0) }, expire: {}, - unsent: { 100_000 - Int(clock.now.timeIntervalSince(start)) }) - guard case .timedOut = outcome else { return XCTFail("\(outcome)") } - XCTAssertLessThan( - clock.now.timeIntervalSince(start), UploadDrain.defaultMaxTotal + UploadDrain.defaultPoll * 2) - } - - /// rclone is silent - that is neither progress nor an empty queue. - func testNoAnswerIsNotAnEmptyQueue() async { - let clock = FakeClock() - let outcome = await UploadDrain.wait( - now: { clock.now }, sleep: { clock.sleep($0) }, expire: {}, unsent: { nil }) - XCTAssertEqual(outcome, .noAnswer) - } - - /// Items read from the dirty cache AFTER the first deadline move get the - /// full 10 min - the move has to be repeated. - func testRepeatsMovingTheDeadlines() async { - let clock = FakeClock() - var expiries = 0 - _ = await UploadDrain.wait( - now: { clock.now }, sleep: { clock.sleep($0) }, expire: { expiries += 1 }, unsent: { 7 }) - XCTAssertGreaterThanOrEqual(expiries, 2) - } -} diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift index d5b8d8e..c64ebf5 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -99,10 +99,16 @@ final class VMLayoutTests: XCTestCase { SVC tm-bands-bind inactive SVC smbd active FD 910 1024 - {"diskCache":{"bytesUsed":1073741824,"uploadsQueued":1,"uploadsInProgress":0}} + BUNDLEENV 0 + IMAGES 1 + {"diskCache":{"bytesUsed":1073741824,"uploadsQueued":1,"uploadsInProgress":0,"erroredFiles":2,"outOfSpace":true}} CMFREE 2147483648 """ let report = VMStatus.report(scriptOutput: output) + XCTAssertEqual(report.bundlePrepared, false) + XCTAssertEqual(report.imageCount, 1) + XCTAssertEqual(report.reading?.buffer.erroredFiles, 2) + XCTAssertEqual(report.reading?.buffer.outOfSpace, true) XCTAssertEqual(report.services["punchfs"], "active") XCTAssertEqual(report.services["tm-bands-bind"], "inactive") XCTAssertEqual(report.punchfsOpenFiles, 910) @@ -110,16 +116,41 @@ final class VMLayoutTests: XCTestCase { XCTAssertEqual(report.reading?.freeBytes, 2_147_483_648) } - func testDestinationURLs() { - let info = """ - ==================================================== - Name : CloudMachine - Kind : Network - URL : smb://tm@192.168.64.10/CloudMachine - ID : 1234 - """ + /// `tm-bands-bind` may be skipped (inactive) before `vm-prepare-bundle`, + /// never failed; a unit nobody reported is unknown, not fine. + func testServicesVerdict() { + var services = [ + "rclone-tm": "active", "punchfs": "active", "tm-bands-bind": "inactive", "smbd": "active", + ] + XCTAssertEqual(VMStatus.servicesHealthy(services), true) + services["tm-bands-bind"] = "failed" + XCTAssertEqual(VMStatus.servicesHealthy(services), false) + services["tm-bands-bind"] = "active" + services["smbd"] = "inactive" + XCTAssertEqual(VMStatus.servicesHealthy(services), false) + services["smbd"] = nil + XCTAssertNil(VMStatus.servicesHealthy(services)) + } + + /// An unknown backlog after `prepare-shutdown` must not read as "all uploaded". + func testShutdownMessageNamesTheBacklog() { + XCTAssertTrue(VMShutdown.outcomeMessage(unsentBytes: 0).contains("everything reached")) + let left = VMShutdown.outcomeMessage(unsentBytes: 3 * VMGuard.gib) + XCTAssertTrue(left.contains("3 GiB"), left) + XCTAssertFalse(left.contains("everything reached")) + let unknown = VMShutdown.outcomeMessage(unsentBytes: nil) + XCTAssertFalse(unknown.contains("everything reached")) + XCTAssertTrue(unknown.contains("did not say")) + } + + func testPasswordIsRedactedInStatusLines() { XCTAssertEqual( - VMStatus.destinationURLs(destinationInfo: info), ["smb://tm@192.168.64.10/CloudMachine"]) + VMStatus.redactedPassword("smb://tm:s3cret@192.168.64.10/CloudMachine"), + "smb://tm:***@192.168.64.10/CloudMachine") + XCTAssertEqual( + VMStatus.redactedPassword("smb://tm@192.168.64.10/CloudMachine"), + "smb://tm@192.168.64.10/CloudMachine") + XCTAssertFalse(VMSetup.maskedDestinationCommand(address: "192.168.64.10").contains("s3cret")) } // MARK: - The template ships with the repo @@ -189,30 +220,40 @@ final class VMLayoutTests: XCTestCase { XCTAssertTrue(yaml.contains("systemctl enable --now smb-throttle.service")) } - /// `install-launchd` installs every template at the TOP of `launchd/`. The - /// machine's agents must not be there: a Mac on the attached-image layout - /// would get an agent starting a machine it does not have. - func testMachineAgentsAreNotInstalledByInstallLaunchd() throws { + /// `install-launchd` installs every template in `launchd/`: exactly the + /// four of the machine layout, and none of the retired disk-image agents. + func testInstallLaunchdShipsOnlyTheMachineLayoutsAgents() throws { let launchd = Self.repoRoot.appendingPathComponent("launchd") - let top = try FileManager.default.contentsOfDirectory(atPath: launchd.path) - XCTAssertFalse(top.contains { $0.contains("vm-") }, "\(top)") - let vm = try FileManager.default.contentsOfDirectory( - atPath: launchd.appendingPathComponent("vm").path - ).sorted() + let top = try FileManager.default.contentsOfDirectory(atPath: launchd.path).sorted() XCTAssertEqual( - vm, + top, [ + "com.renacode.cloudmachine.app.plist.template", + "com.renacode.cloudmachine.backup-health.plist.template", "com.renacode.cloudmachine.vm-guard.plist.template", "com.renacode.cloudmachine.vm-start.plist.template", ]) + for retired in AgentRepair.retired { + XCTAssertFalse(top.contains { $0.contains(".\(retired).") }, retired) + } + } + + /// The guest's rclone remote control has no password: Lima must not forward + /// it to the Mac's localhost, where any web page can reach it. + func testTemplateForwardsNoPortsToTheMac() throws { + let template = try String( + contentsOf: Self.repoRoot.appendingPathComponent("vm/cmsmb.yaml"), encoding: .utf8) + XCTAssertTrue(template.contains("portForwards:"), "no portForwards section") + XCTAssertTrue(template.contains("ignore: true")) } func testTimeMachineTargetIsMatchedByWholeAddress() { - let urls = ["smb://tm@192.168.64.10/CloudMachine"] - XCTAssertTrue(VMStatus.targets(urls: urls, address: "192.168.64.10")) - XCTAssertFalse(VMStatus.targets(urls: urls, address: "192.168.64.1")) - XCTAssertFalse(VMStatus.targets(urls: urls, address: "192.168.64.11")) - XCTAssertFalse(VMStatus.targets(urls: [], address: "192.168.64.10")) + let url = "smb://tm@192.168.64.10/CloudMachine" + XCTAssertTrue(VMStatus.isMachineShare(url, address: "192.168.64.10")) + XCTAssertFalse(VMStatus.isMachineShare(url, address: "192.168.64.1")) + XCTAssertFalse(VMStatus.isMachineShare(url, address: "192.168.64.11")) + XCTAssertTrue(VMStatus.isMachineShare(url, address: nil)) + XCTAssertFalse(VMStatus.isMachineShare("smb://nas/Backups", address: nil)) } func testReadingCarriesTheUnsentBacklogFromVfsQueue() { diff --git a/packaging/README.md b/packaging/README.md index fb0a829..316cbd8 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -112,16 +112,17 @@ open a pull request to `homebrew-tap` with the result. - **Removes the quarantine attribute** after install. The app is not notarized, so Gatekeeper would otherwise block it. - **Does not stop launchd agents** on uninstall. Homebrew runs `uninstall` - directives on `brew upgrade` too, and stopping `gdrive-buffer` kills the - rclone process that holds the mount. + directives on `brew upgrade` too, and the agents keep the machine and its + guard going across an upgrade. - **Does not reload agents itself** - but they must be reloaded: after the bundle is replaced, launchd refuses to start them (`spawn failed`, `OS_REASON_CODESIGNING`). Cask steps run in a sandbox without `~/Library/LaunchAgents`, so the app does it when Homebrew reopens it after the upgrade, and the backup watchdog repairs any agent that cannot start. - `drive-status` shows the agents' state. -- **`zap` leaves `~/.cloudmachine` alone.** It holds the upload buffer, which - may contain backups that have not reached Google Drive yet. + `vm-status` shows the agents' state. +- **`zap` leaves `~/.cloudmachine` and `~/.lima` alone.** They hold the + machine and its upload buffer, which may contain backups that have not + reached Google Drive yet. ## Already installed from source? diff --git a/packaging/homebrew/cloudmachine.rb.in b/packaging/homebrew/cloudmachine.rb.in index 11df4af..06e0f7b 100644 --- a/packaging/homebrew/cloudmachine.rb.in +++ b/packaging/homebrew/cloudmachine.rb.in @@ -35,16 +35,12 @@ cask "cloudmachine" do end # Deliberately WITHOUT `uninstall launchctl:` - Homebrew runs `uninstall` - # directives on `brew upgrade` too, and stopping - # com.renacode.cloudmachine.gdrive-buffer kills the rclone that holds the - # mount: the image only comes back after ~20 min, and an interrupted upload - # can cost the volume's root directory. The agents are removed manually - # (caveats). + # directives on `brew upgrade` too, and the agents keep the machine and its + # guard going across an upgrade. The agents are removed manually (caveats). uninstall quit: "com.renacode.cloudmachine" - # Deliberately without ~/.cloudmachine: that is where the upload buffer - # lives, i.e. backups not yet sent to Google Drive. Deleting it destroys the - # backup. + # Deliberately without ~/.cloudmachine and ~/.lima: the machine and its + # upload buffer live there, i.e. backups not yet sent to Google Drive. zap trash: [ "~/Library/Logs/CloudMachine", "~/Library/Preferences/com.renacode.cloudmachine.plist", @@ -55,17 +51,19 @@ cask "cloudmachine" do remaining setup steps for this Mac, with a button for each. See https://github.com/RenaCode/CloudMachine#getting-started - `brew upgrade --cask renacode/tap/cloudmachine` keeps the Google Drive - mount running; the app reloads the background agents when it reopens. - `cloudmachine-agent drive-status` shows "Agents: OK" once they run the - new version. If the window shows "Remote control of the Drive mount is - open", press Restart the mount there (or: cloudmachine-agent restart-mount). + The backup runs in a Lima machine: `brew install lima` first. - Uninstalling does NOT stop the launchd agents or delete the upload buffer - in ~/.cloudmachine (it may hold backups not yet sent to Google Drive). + `brew upgrade --cask renacode/tap/cloudmachine` keeps the machine + running; the app reloads the background agents when it reopens. + `cloudmachine-agent vm-status` shows "Agents: OK" once they run the + new version. + + Uninstalling does NOT stop the launchd agents or delete the machine + (its buffer may hold backups not yet sent to Google Drive). Before `brew uninstall`, run: cloudmachine-agent prepare-shutdown - then remove ~/Library/LaunchAgents/com.renacode.cloudmachine.*.plist. + then `limactl delete cmsmb` and remove + ~/Library/LaunchAgents/com.renacode.cloudmachine.*.plist. Full Disk Access is tied to the release signing certificate: grant it once and it survives upgrades. Moving from a locally built copy needs one re-grant. diff --git a/vm/README.md b/vm/README.md index 6da5568..06aa597 100644 --- a/vm/README.md +++ b/vm/README.md @@ -35,12 +35,31 @@ Each layer is there for a reason: ## Commands - cloudmachine-agent vm-setup # create/start the machine, print the tmutil command + cloudmachine-agent vm-setup --install-agents # create/start the machine, agents, print the tmutil command sudo tmutil setdestination smb://tm:@
/CloudMachine # ...Time Machine creates its empty image; stop the backup... cloudmachine-agent vm-prepare-bundle # 32 MB bands, bands moved to Drive cloudmachine-agent vm-status cloudmachine-agent vm-guard # one check; the launchd agent runs it every minute + cloudmachine-agent prepare-shutdown # before a restart: stop the backup, drain, limactl stop + +`install-launchd` (also run by `--install-agents`) installs four agents: +`app`, `backup-health`, `vm-start` (boots the machine at login) and `vm-guard` +(pauses Time Machine when more than 110 GiB waits to be uploaded or the +machine has less than 40 GiB free, resumes below 40 GiB - only what it paused). + +## No ports to the Mac + +rclone's remote control listens in the machine on `127.0.0.1:5572` without a +password - the guard and the status read the upload queue through it. Lima +forwards a guest's localhost ports to the Mac's by default, and on the Mac's +localhost any web page can send it commands, deleting the backup included. +`cmsmb.yaml` therefore forwards nothing (`portForwards` with `ignore: true`); +Samba is reached over vzNAT. A machine created from an older template keeps +the forwarding until the rule is added to `~/.lima/cmsmb/lima.yaml` and the +machine is restarted (`cloudmachine-agent prepare-shutdown`, then +`limactl start cmsmb`). `vm-status` and the backup watchdog report it while +`127.0.0.1:5572` answers on the Mac. ## Trying a change without touching the live machine diff --git a/vm/cmsmb.yaml b/vm/cmsmb.yaml index 52b0a8e..1bb5f2b 100644 --- a/vm/cmsmb.yaml +++ b/vm/cmsmb.yaml @@ -24,6 +24,17 @@ networks: containerd: system: false user: false +# Nothing in the machine is forwarded to the Mac. Lima forwards the guest's +# localhost ports by default, and rclone's remote control (127.0.0.1:5572, no +# password) would then answer on the Mac's localhost - where any web page can +# send it commands, deleting the backup included. Samba is reached over vzNAT. +portForwards: +- guestIP: "127.0.0.1" + proto: any + ignore: true +- guestIP: "0.0.0.0" + proto: any + ignore: true provision: - mode: data path: /etc/samba/smb.conf From efccecb3f4335be1071242b34dfd7db124a3659b Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 16:35:58 +0100 Subject: [PATCH 6/6] fix(vm): track smb-throttle with the other services vm-status, backup-health and the app now show and check the SMB shaper; it was installed by the template but missing from VMLayout.services. Checked against the live machine after its restart (all five active). The live machine also got the template's portForwards ignore rule: before it, limactl published rclone's rc (127.0.0.1:5572, no auth) on the Mac's localhost; after a restart the port is closed on the Mac and still answers inside the machine. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AU1MhxJW1e9oTB2XvjX8WT --- mac-app/Sources/CloudMachineCore/VMLayout.swift | 2 +- mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift | 4 +++- mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift index a942fe1..559d9b2 100644 --- a/mac-app/Sources/CloudMachineCore/VMLayout.swift +++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift @@ -39,7 +39,7 @@ public enum VMLayout { } /// The systemd units the share depends on, in start order. - public static let services = ["rclone-tm", "punchfs", "tm-bands-bind", "smbd"] + public static let services = ["smb-throttle", "rclone-tm", "punchfs", "tm-bands-bind", "smbd"] /// Host-side state: `~/.cloudmachine/vm`. public static var hostDir: URL { diff --git a/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift b/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift index 0146912..db25520 100644 --- a/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift +++ b/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift @@ -10,7 +10,9 @@ extension MachineStatus { machine.state = .running machine.address = "192.168.64.10" machine.services = [ - "rclone-tm": "active", "punchfs": "active", "tm-bands-bind": "active", "smbd": "active", + "smb-throttle": "active", "rclone-tm": "active", "punchfs": "active", + "tm-bands-bind": "active", + "smbd": "active", ] machine.reading = VMLayout.Reading( buffer: VMLayout.BufferStats( diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift index c64ebf5..f20399b 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -120,7 +120,9 @@ final class VMLayoutTests: XCTestCase { /// never failed; a unit nobody reported is unknown, not fine. func testServicesVerdict() { var services = [ - "rclone-tm": "active", "punchfs": "active", "tm-bands-bind": "inactive", "smbd": "active", + "smb-throttle": "active", "rclone-tm": "active", "punchfs": "active", + "tm-bands-bind": "inactive", + "smbd": "active", ] XCTAssertEqual(VMStatus.servicesHealthy(services), true) services["tm-bands-bind"] = "failed"