From 9fcc5459c241de51da81c874507ba0d7ef22dc59 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 18:27:48 +0100 Subject: [PATCH 1/5] feat(vm): copy the image's metadata to Drive and alarm when it goes stale The bands on Drive cannot be opened without the image's Info.plist, token and mapped/, which lived only on the machine's disk - kept out of Time Machine. tm-meta-sync (service + 15 min timer) copies /srv/tm-local minus bands/ and lock to -tm-meta, never from an image without its Info.plist (rclone sync would wipe the copy), and stamps each success. vm-status shows the copy's age; it and backup-health report one older than an hour (deferred after startup). vm/README gains the restore procedure. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AU1MhxJW1e9oTB2XvjX8WT --- .../CloudMachineCore/BackupHealth.swift | 25 +++++++++ .../CloudMachineCore/L10nPolish+Machine.swift | 6 ++ .../CloudMachineCore/L10nPolish+VM.swift | 8 +++ .../Sources/CloudMachineCore/VMLayout.swift | 25 ++++++++- .../Sources/CloudMachineCore/VMSetup.swift | 2 +- .../Sources/CloudMachineCore/VMStatus.swift | 52 +++++++++++++++++- .../AppStatusFixtures.swift | 1 + .../BackupHealthTests.swift | 22 ++++++++ .../CloudMachineAppTests/VMLayoutTests.swift | 55 ++++++++++++++++++- vm/README.md | 44 +++++++++++++++ vm/cmsmb.yaml | 12 +++- vm/systemd/tm-meta-sync.service | 24 ++++++++ vm/systemd/tm-meta-sync.timer | 11 ++++ 13 files changed, 280 insertions(+), 7 deletions(-) create mode 100644 vm/systemd/tm-meta-sync.service create mode 100644 vm/systemd/tm-meta-sync.timer diff --git a/mac-app/Sources/CloudMachineCore/BackupHealth.swift b/mac-app/Sources/CloudMachineCore/BackupHealth.swift index dff0aec..e38156b 100644 --- a/mac-app/Sources/CloudMachineCore/BackupHealth.swift +++ b/mac-app/Sources/CloudMachineCore/BackupHealth.swift @@ -178,6 +178,9 @@ public enum BackupHealth { machineFreeGB: Int? = nil, punchfsNearFileLimit: Bool = false, remoteControlOpen: Bool = false, + // Seconds since the last copy of the image's metadata to Drive + // (`.infinity` = never); `nil` = not known or nothing to copy yet. + metaSyncAge: TimeInterval? = nil, maxAgeHours: Double = BackupHealth.maxAgeHours, // Whether the startup grace period is in progress - see `startupGraceMinutes`. withinStartupGrace: Bool = false, @@ -359,6 +362,25 @@ public enum BackupHealth { code: "rc-port-open")) } + // The bands on Drive cannot be opened without the image's metadata, which + // otherwise exists only on the machine's disk - kept out of Time Machine. + // A stale copy is a backup that cannot be restored after losing the Mac, + // while every other light is green. Deferred after startup: the stamp + // keeps its age across a night with the Mac off, and the timer's first + // run comes 5 min after the machine boots. + if let metaSyncAge, metaSyncAge > VMLayout.metaSyncMaxAge { + notReadyYet( + Problem( + summary: metaSyncAge.isFinite + ? L10n.tr( + "The image's metadata was last copied to Drive %@ ago", formatAge(metaSyncAge)) + : L10n.tr("The image's metadata has never been copied to Drive"), + detail: L10n.tr( + "Info.plist, token and mapped/ of the backup image exist only on the machine's disk; without them the bands in %@ cannot be restored. The copy (tm-meta-sync, every 15 min) goes to %@. Check: cloudmachine-agent vm-status, then in the machine: journalctl -u tm-meta-sync", + VMLayout.remote, VMLayout.metaRemote), + code: "metadata-sync-stale")) + } + // Space on the Drive. Running out of it stops every upload, and the // machine's buffer then fills up - this has to be known EARLIER, not from // a failure. The threshold is counted in cycles, not in percent: at a @@ -564,6 +586,9 @@ public enum BackupHealth { machineFreeGB: reading.map { Int($0.freeBytes / VMGuard.gib) }, punchfsNearFileLimit: nearFileLimit, remoteControlOpen: VMStatus.remoteControlOpenOnMac(), + metaSyncAge: report.flatMap { + VMStatus.metaSyncOverdue($0) != nil ? $0.metaSync?.ageSeconds : nil + }, maxAgeHours: maxAgeHours, // The REAL clock, not `now`: tests substitute a `now` from the past, and // uptime computed from it would come out negative, i.e. "startup in diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift index 3662183..7af60da 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift @@ -52,6 +52,8 @@ extension L10nPolish { "Zainstaluj agentów w tle (start maszyny, strażnik bufora, czujka kopii)", "Installs rclone through Homebrew - CloudMachine itself uses the one from install-rclone.": "Instaluje rclone przez Homebrew - sam CloudMachine używa tego z install-rclone.", + "Info.plist, token and mapped/ of the backup image exist only on the machine's disk; without them the bands in %@ cannot be restored. The copy (tm-meta-sync, every 15 min) goes to %@. Check: cloudmachine-agent vm-status, then in the machine: journalctl -u tm-meta-sync": + "Info.plist, token i mapped/ obrazu kopii są tylko na dysku maszyny; bez nich pasm w %@ nie da się odtworzyć. Kopia (tm-meta-sync, co 15 min) trafia do %@. Sprawdź: cloudmachine-agent vm-status, potem w maszynie: journalctl -u tm-meta-sync", "Lima says the machine is running, but a command in it did not return. Without it nothing is known about Samba, rclone or the upload. Check: cloudmachine-agent vm-status": "Lima mówi, że maszyna działa, ale polecenie w niej nie wróciło. Bez tego nic nie wiadomo o Sambie, rclone ani wysyłce. Sprawdź: cloudmachine-agent vm-status", "Machine": @@ -132,6 +134,10 @@ extension L10nPolish { "Maszyna jest zatrzymana. rclone nie powiedział, ile zostało do wysłania; to, co nie wyszło, zostaje w buforze maszyny i zostanie wysłane po następnym starcie. Można zrestartować Maca.", "The machine's disk is a file on this disk and grows as the upload buffer fills. With no space left the machine cannot write the buffer, and Time Machine's local snapshots have nowhere to go either.": "Dysk maszyny to plik na tym dysku i rośnie razem z buforem wysyłki. Bez wolnego miejsca maszyna nie zapisze bufora, a lokalne migawki Time Machine też nie mają gdzie trafić.", + "The image's metadata has never been copied to Drive": + "Metadane obrazu kopii nigdy nie trafiły na Drive", + "The image's metadata was last copied to Drive %@ ago": + "Metadane obrazu kopii trafiły na Drive ostatnio %@ temu", "The machine's disk is running out of space (%@ GB)": "Na dysku maszyny kończy się miejsce (%@ GB)", "The new sign-in does NOT see the existing backup (%@), so remote '%@' was NOT replaced and rclone.conf is unchanged. Keep the current connection, or sign in with the credential (client_id) that created the backup.": diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift index a220604..7837755 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift @@ -50,6 +50,14 @@ extension L10nPolish { "Gotowe: %@ ma pasma %@ MB na Google Drive; oryginał leży w %@ w maszynie. Uruchom kopię: tmutil startbackup", "exit code %@, no message": "kod wyjścia %@, bez komunikatu", + "Image metadata: NEVER copied to %@ - check tm-meta-sync in the machine": + "Metadane obrazu: NIGDY nie skopiowane do %@ - sprawdź tm-meta-sync w maszynie", + "Image metadata: copied %@ ago to %@": + "Metadane obrazu: skopiowane %@ temu do %@", + "Image metadata: last copied %@ ago - OVERDUE (%@)": + "Metadane obrazu: ostatnia kopia %@ temu - ZALEGŁA (%@)", + "Image metadata: not copied yet - waits for vm-prepare-bundle": + "Metadane obrazu: jeszcze bez kopii - czeka na vm-prepare-bundle", "Free in machine: %@ GiB": "Wolne w maszynie: %@ GiB", "fsck_apfs -n found problems: %@": diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift index 559d9b2..7e8119f 100644 --- a/mac-app/Sources/CloudMachineCore/VMLayout.swift +++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift @@ -36,10 +36,21 @@ public enum VMLayout { public static let cacheDir = "/var/cache/rclone" public static let backupDir = "/var/lib/cloudmachine/prepare-backup" public static let rcAddress = "127.0.0.1:5572" + /// Touched by `tm-meta-sync.service` after every successful sync. + public static let metaSyncStamp = "/var/lib/cloudmachine/tm-meta-sync.ok" } - /// The systemd units the share depends on, in start order. - public static let services = ["smb-throttle", "rclone-tm", "punchfs", "tm-bands-bind", "smbd"] + /// The systemd units the share depends on, in start order, then the timer + /// that copies the image's metadata to Drive. A name without a suffix is a + /// `.service` - see `unitName(_:)`. + public static let services = [ + "smb-throttle", "rclone-tm", "punchfs", "tm-bands-bind", "smbd", "tm-meta-sync.timer", + ] + + /// The systemd unit for an entry of `services`. + public static func unitName(_ service: String) -> String { + service.contains(".") ? service : "\(service).service" + } /// Host-side state: `~/.cloudmachine/vm`. public static var hostDir: URL { @@ -59,6 +70,16 @@ public enum VMLayout { ?? "\(DriveFolder.remoteName):\(DriveFolder.remoteRoot)/\(DriveFolder.name)-tm" } + /// `gdrive:CloudMachine/-tm-meta`: the image's files outside + /// `bands/` (Info.plist, token, mapped/, Time Machine's plists), copied by + /// `tm-meta-sync` from the machine's disk. The bands alone cannot be opened + /// again; a restore takes both folders (vm/README.md, "Restoring"). + public static var metaRemote: String { remote + "-meta" } + + /// Older than this, the metadata copy is reported. The timer runs every + /// 15 min, so an hour is three runs missed in a row. + public static let metaSyncMaxAge: TimeInterval = 3600 + /// Lima's directory with the machine's disk (`LIMA_HOME`, else `~/.lima`). /// The disk holds rclone's buffer: left in the backup, Time Machine copies /// its own backlog through the machine - on 27.10.2026 that kept the first diff --git a/mac-app/Sources/CloudMachineCore/VMSetup.swift b/mac-app/Sources/CloudMachineCore/VMSetup.swift index 0295399..b215a57 100644 --- a/mac-app/Sources/CloudMachineCore/VMSetup.swift +++ b/mac-app/Sources/CloudMachineCore/VMSetup.swift @@ -131,7 +131,7 @@ public enum VMSetup { messages.append(L10n.tr("SMB password stored in %@ (mode 600).", VMLayout.passwordFile.path)) // 5. Services. - let units = VMLayout.services.map { "\($0).service" }.joined(separator: " ") + let units = VMLayout.services.map(VMLayout.unitName).joined(separator: " ") guard let started = try? await VMLayout.rootShell( "systemctl daemon-reload && systemctl start \(units)", timeout: 300), diff --git a/mac-app/Sources/CloudMachineCore/VMStatus.swift b/mac-app/Sources/CloudMachineCore/VMStatus.swift index 56c4509..a9f30c8 100644 --- a/mac-app/Sources/CloudMachineCore/VMStatus.swift +++ b/mac-app/Sources/CloudMachineCore/VMStatus.swift @@ -16,10 +16,34 @@ public enum VMStatus { public var bundlePrepared: Bool? /// How many images Time Machine has created in the machine; `nil` = not read. public var imageCount: Int? + /// The last successful `tm-meta-sync`; `nil` = not read. + public var metaSync: MetaSync? public init() {} } + public enum MetaSync: Equatable, Sendable { + case never + /// Seconds since the last successful sync, by the machine's own clock. + case age(TimeInterval) + + /// `.infinity` for `never`, so one threshold covers both. + public var ageSeconds: TimeInterval { + switch self { + case .never: return .infinity + case .age(let seconds): return seconds + } + } + } + + /// Whether the metadata copy is overdue. Only for an image that went + /// through `vm-prepare-bundle`: before it, the sync is skipped by design. + /// `nil` = not known (not read, or no prepared image). + public static func metaSyncOverdue(_ report: Report) -> Bool? { + guard report.bundlePrepared == true, let sync = report.metaSync else { return nil } + return sync.ageSeconds > VMLayout.metaSyncMaxAge + } + static let script = VMLayout.services.map { "echo SVC \($0) $(systemctl is-active \($0))" }.joined(separator: "; ") + "; pid=$(systemctl show -p MainPID --value punchfs); " @@ -27,6 +51,7 @@ public enum VMStatus { + "$(awk '/Max open files/{print $4}' /proc/$pid/limits); fi; " + "echo BUNDLEENV $([ -s \(VMLayout.Guest.bundleEnv) ] && echo 1 || echo 0); " + "echo IMAGES $(ls -d \(VMLayout.Guest.localRoot)/*.sparsebundle 2>/dev/null | wc -l); " + + "echo METASYNC $(stat -c %Y \(VMLayout.Guest.metaSyncStamp) 2>/dev/null || echo 0) $(date +%s); " + VMLayout.readingScript static func report(scriptOutput: String) -> Report { @@ -42,6 +67,10 @@ public enum VMStatus { report.bundlePrepared = fields[1] == "1" } else if fields.first == "IMAGES", fields.count >= 2 { report.imageCount = Int(fields[1]) + } else if fields.first == "METASYNC", fields.count >= 3, let last = Int(fields[1]), + let now = Int(fields[2]) + { + report.metaSync = last == 0 ? .never : .age(TimeInterval(max(0, now - last))) } } report.reading = VMLayout.reading(scriptOutput: scriptOutput) @@ -160,10 +189,12 @@ public enum VMStatus { for service in VMLayout.services { let state = report.services[service] ?? L10n.tr("unknown") out.append( - L10n.tr("Service %@: %@", service.padding(toLength: 14, withPad: " ", startingAt: 0), state) + L10n.tr("Service %@: %@", service.padding(toLength: 18, withPad: " ", startingAt: 0), state) ) } if servicesHealthy(report.services) != true { healthy = false } + if let line = metaSyncLine(report) { out.append(line) } + if metaSyncOverdue(report) == true { healthy = false } if let reading = report.reading { out.append( L10n.tr( @@ -215,6 +246,25 @@ public enum VMStatus { return (out, healthy) } + /// The metadata copy, for `vm-status`; `nil` when there is nothing to say. + static func metaSyncLine(_ report: Report) -> String? { + switch (report.bundlePrepared, report.metaSync) { + case (.some(false), _): + return L10n.tr("Image metadata: not copied yet - waits for vm-prepare-bundle") + case (.some(true), .some(.never)): + return L10n.tr( + "Image metadata: NEVER copied to %@ - check tm-meta-sync in the machine", + VMLayout.metaRemote) + case (.some(true), .some(.age(let seconds))): + let ago = BackupHealth.formatAge(seconds) + return seconds > VMLayout.metaSyncMaxAge + ? L10n.tr("Image metadata: last copied %@ ago - OVERDUE (%@)", ago, VMLayout.metaRemote) + : L10n.tr("Image metadata: copied %@ ago to %@", ago, VMLayout.metaRemote) + default: + return nil + } + } + /// `smb://tm:secret@host/...` -> `smb://tm:***@host/...`. tmutil prints /// the URL without the password, but a status line must never be the one /// place it leaks from. diff --git a/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift b/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift index db25520..8932f51 100644 --- a/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift +++ b/mac-app/Tests/CloudMachineAppTests/AppStatusFixtures.swift @@ -13,6 +13,7 @@ extension MachineStatus { "smb-throttle": "active", "rclone-tm": "active", "punchfs": "active", "tm-bands-bind": "active", "smbd": "active", + "tm-meta-sync.timer": "active", ] machine.reading = VMLayout.Reading( buffer: VMLayout.BufferStats( diff --git a/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift b/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift index fea051e..d9e4522 100644 --- a/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift @@ -434,6 +434,28 @@ final class BackupHealthTests: XCTestCase { XCTAssertFalse(report.healthy) } + /// A metadata copy older than an hour is a backup that cannot be restored + /// after losing the Mac; right after startup it waits for the timer's + /// first run. + func testStaleMetadataCopyIsReported() { + func health(metaSyncAge: TimeInterval?, grace: Bool = false) -> BackupHealth.Report { + BackupHealth.evaluate( + lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), + result: 0, now: now, machineRunning: true, servicesHealthy: true, + destinationRegistered: true, erroredFiles: 0, outOfSpace: false, queueReadable: true, + metaSyncAge: metaSyncAge, withinStartupGrace: grace) + } + XCTAssertTrue(health(metaSyncAge: 900).healthy) + XCTAssertTrue(health(metaSyncAge: nil).healthy) + XCTAssertEqual(health(metaSyncAge: 7200).problems.map(\.code), ["metadata-sync-stale"]) + XCTAssertEqual(health(metaSyncAge: .infinity).problems.map(\.code), ["metadata-sync-stale"]) + XCTAssertTrue( + health(metaSyncAge: .infinity).problems[0].summary.contains("never")) + let afterBoot = health(metaSyncAge: 10 * 3600, grace: true) + XCTAssertTrue(afterBoot.healthy) + XCTAssertEqual(afterBoot.deferred.map(\.code), ["metadata-sync-stale"]) + } + func testUptimeCanBeRead() { let uptime = BackupHealth.systemUptime() XCTAssertNotNil(uptime) diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift index f20399b..57c8670 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -122,9 +122,12 @@ final class VMLayoutTests: XCTestCase { var services = [ "smb-throttle": "active", "rclone-tm": "active", "punchfs": "active", "tm-bands-bind": "inactive", - "smbd": "active", + "smbd": "active", "tm-meta-sync.timer": "active", ] XCTAssertEqual(VMStatus.servicesHealthy(services), true) + services["tm-meta-sync.timer"] = "inactive" + XCTAssertEqual(VMStatus.servicesHealthy(services), false) + services["tm-meta-sync.timer"] = "active" services["tm-bands-bind"] = "failed" XCTAssertEqual(VMStatus.servicesHealthy(services), false) services["tm-bands-bind"] = "active" @@ -222,6 +225,56 @@ final class VMLayoutTests: XCTestCase { XCTAssertTrue(yaml.contains("systemctl enable --now smb-throttle.service")) } + /// The units `vm-setup` starts: a timer keeps its suffix, the rest are services. + func testUnitNames() { + XCTAssertEqual(VMLayout.unitName("smbd"), "smbd.service") + XCTAssertEqual(VMLayout.unitName("tm-meta-sync.timer"), "tm-meta-sync.timer") + XCTAssertTrue(VMLayout.services.contains("tm-meta-sync.timer")) + } + + func testMetadataSyncIsReadFromTheMachinesClock() { + func report(_ lines: String) -> VMStatus.Report { VMStatus.report(scriptOutput: lines) } + let synced = report("BUNDLEENV 1\nMETASYNC 1000 1600") + XCTAssertEqual(synced.metaSync, .age(600)) + XCTAssertEqual(VMStatus.metaSyncOverdue(synced), false) + + let stale = report("BUNDLEENV 1\nMETASYNC 1000 \(1000 + 3601)") + XCTAssertEqual(VMStatus.metaSyncOverdue(stale), true) + + let never = report("BUNDLEENV 1\nMETASYNC 0 1600") + XCTAssertEqual(never.metaSync, .never) + XCTAssertEqual(VMStatus.metaSyncOverdue(never), true) + + // Before vm-prepare-bundle the unit is skipped by design. + XCTAssertNil(VMStatus.metaSyncOverdue(report("BUNDLEENV 0\nMETASYNC 0 1600"))) + // Not read is not known - never "fine". + XCTAssertNil(VMStatus.metaSyncOverdue(report("BUNDLEENV 1"))) + XCTAssertEqual(VMLayout.metaRemote, VMLayout.remote + "-meta") + } + + /// 27.10.2026: the image's Info.plist, token and mapped/ were only on the + /// machine's disk, so the bands on Drive could not be restored. The template + /// must install the copy, and the copy must neither carry the bands nor + /// wipe Drive from an image without its Info.plist. + func testTemplateCopiesTheImageMetadataToDrive() throws { + let vm = Self.repoRoot.appendingPathComponent("vm") + let unit = try String( + contentsOf: vm.appendingPathComponent("systemd/tm-meta-sync.service"), encoding: .utf8) + let timer = try String( + contentsOf: vm.appendingPathComponent("systemd/tm-meta-sync.timer"), encoding: .utf8) + let yaml = try String(contentsOf: vm.appendingPathComponent("cmsmb.yaml"), encoding: .utf8) + XCTAssertTrue(unit.contains("${CM_REMOTE}-meta/")) + XCTAssertTrue(unit.contains(#"--exclude "*/bands/**""#)) + XCTAssertTrue(unit.contains(#"--exclude "*/lock""#)) + XCTAssertTrue( + unit.contains(#"ExecCondition=/usr/bin/test -s "/srv/tm-local/${CM_BUNDLE}/Info.plist""#)) + XCTAssertTrue(unit.contains("ExecStartPost=/usr/bin/touch \(VMLayout.Guest.metaSyncStamp)")) + XCTAssertTrue(timer.contains("OnUnitActiveSec=15min")) + XCTAssertTrue(yaml.contains("file: systemd/tm-meta-sync.service")) + XCTAssertTrue(yaml.contains("file: systemd/tm-meta-sync.timer")) + XCTAssertTrue(yaml.contains("tm-meta-sync.timer\n")) + } + /// `install-launchd` installs every template in `launchd/`: exactly the /// four of the machine layout, and none of the retired disk-image agents. func testInstallLaunchdShipsOnlyTheMachineLayoutsAgents() throws { diff --git a/vm/README.md b/vm/README.md index 06aa597..f413f54 100644 --- a/vm/README.md +++ b/vm/README.md @@ -10,6 +10,7 @@ Time Machine --SMB--> Samba /srv/tm-local/.sparsebundle (ext4: imag bands/ = bind of /srv/tm-punch/bands punchfs /srv/tm-punch (hole punch, xattrs) rclone /srv/tm = gdrive:CloudMachine/-tm +tm-meta-sync (every 15 min): /srv/tm-local minus bands/ -> gdrive:CloudMachine/-tm-meta ``` Each layer is there for a reason: @@ -20,6 +21,10 @@ Each layer is there for a reason: - macOS frees bands with a hole punch and keeps Samba's sparse flag and Apple streams in extended attributes; rclone supports neither. `punchfs` fills both gaps (see `punchfs/README.md`). +- The image's small files (Info.plist, token, `mapped/`, Time Machine's + plists) are on the machine's disk, and that disk is kept out of Time + Machine. `tm-meta-sync` copies them to `-tm-meta` every 15 minutes; + without them the bands cannot be opened again (see "Restoring"). - Time Machine creates bands of 875 MB. `vm-prepare-bundle` converts the new, empty image to 32 MB bands, so a day of changes fits Google's upload limit. @@ -48,6 +53,45 @@ Each layer is there for a reason: (pauses Time Machine when more than 110 GiB waits to be uploaded or the machine has less than 40 GiB free, resumes below 40 GiB - only what it paused). +## Restoring + +The backup on Drive is two folders: the bands in `gdrive:CloudMachine/-tm` +and everything else of the image in `gdrive:CloudMachine/-tm-meta` +(`vm-status`: "Image metadata"; the backup watchdog reports a copy older than +an hour). To open it on a new Mac or after the machine's disk is lost: + +1. Install CloudMachine and rclone's configuration for the same Google + account, and give the Mac the same Drive folder name. +2. `cloudmachine-agent vm-setup` - but do NOT point Time Machine at the + machine and do NOT run `vm-prepare-bundle`: either would start a new image. +3. Put the image's files back and the bands under them (`` is the + folder in `-tm-meta`, e.g. `Mac Studio.sparsebundle`): + + ```sh + limactl shell --workdir / cmsmb sudo bash -c ' + set -e + systemctl stop smbd + . /etc/cloudmachine/vm.env + rclone --config /etc/rclone/rclone.conf copy "$CM_REMOTE-meta/" /srv/tm-local/ + cd /srv/tm-local && ls -d *.sparsebundle' # -> + limactl shell --workdir / cmsmb sudo bash -c ' + set -e + b="/srv/tm-local/" + mkdir -p "$b/bands" && chown -R tm:tm /srv/tm-local + printf "CM_BUNDLE=\"%s\"\n" "" > /etc/cloudmachine/bundle.env + systemctl start tm-bands-bind smbd && findmnt "$b/bands"' + ``` + +4. On the Mac, connect to the share (`smb://tm@
/CloudMachine`, + password in `~/.cloudmachine/vm/tm-smb.pass`). Migration Assistant ("From a + Mac, Time Machine backup") or Time Machine's "Browse backups" sees it; for a + read-only look, `hdiutil attach -readonly` the image. + +The metadata is copied every 15 minutes while bands are uploaded continuously, +so restore from a state after a backup finished and its upload drained +(`vm-status`: "Still to upload: 0"). Samba's `user.DOSATTRIB` attributes are +not copied; Samba recreates them. + ## No ports to the Mac rclone's remote control listens in the machine on `127.0.0.1:5572` without a diff --git a/vm/cmsmb.yaml b/vm/cmsmb.yaml index 1bb5f2b..230ed55 100644 --- a/vm/cmsmb.yaml +++ b/vm/cmsmb.yaml @@ -70,6 +70,12 @@ provision: - mode: data path: /etc/systemd/system/smb-throttle.service file: systemd/smb-throttle.service +- mode: data + path: /etc/systemd/system/tm-meta-sync.service + file: systemd/tm-meta-sync.service +- mode: data + path: /etc/systemd/system/tm-meta-sync.timer + file: systemd/tm-meta-sync.timer - mode: data path: /opt/cloudmachine/punchfs/test-punchfs.sh file: punchfs/test-punchfs.sh @@ -128,9 +134,11 @@ provision: # which `vm-setup` copies in, and the bands bind waits for # /etc/cloudmachine/bundle.env, which `vm-prepare-bundle` writes. systemctl daemon-reload - systemctl enable rclone-tm.service punchfs.service tm-bands-bind.service smbd.service + systemctl enable rclone-tm.service punchfs.service tm-bands-bind.service smbd.service \ + tm-meta-sync.timer # The SMB shaper needs nothing from the Mac; start it every boot. systemctl enable --now smb-throttle.service if [ -s /etc/rclone/rclone.conf ] && [ -s /etc/cloudmachine/vm.env ]; then - systemctl start rclone-tm.service punchfs.service tm-bands-bind.service smbd.service + systemctl start rclone-tm.service punchfs.service tm-bands-bind.service smbd.service \ + tm-meta-sync.timer fi diff --git a/vm/systemd/tm-meta-sync.service b/vm/systemd/tm-meta-sync.service new file mode 100644 index 0000000..3c8cd0d --- /dev/null +++ b/vm/systemd/tm-meta-sync.service @@ -0,0 +1,24 @@ +# CloudMachine: the image's files outside bands/ (Info.plist, token, mapped/, +# Time Machine's plists) live only on the machine's ext4 disk, and that disk is +# kept out of Time Machine. Without a copy on Drive the bands alone cannot be +# opened again after the Mac or the machine is lost - see vm/README.md +# ("Restoring"). Copied next to the bands: gdrive:CloudMachine/-tm-meta. +[Unit] +Description=Copy of the Time Machine image's files outside bands/ to Google Drive +After=rclone-tm.service tm-bands-bind.service +ConditionPathExists=/etc/rclone/rclone.conf +ConditionPathExists=/etc/cloudmachine/vm.env +ConditionPathExists=/etc/cloudmachine/bundle.env + +[Service] +Type=oneshot +EnvironmentFile=/etc/cloudmachine/vm.env +EnvironmentFile=/etc/cloudmachine/bundle.env +# `sync` deletes on Drive what is gone locally: an image without its Info.plist +# (a new machine, a disk recreated before a restore) must not wipe the copy. +ExecCondition=/usr/bin/test -s "/srv/tm-local/${CM_BUNDLE}/Info.plist" +ExecStart=/usr/local/bin/rclone sync /srv/tm-local/ ${CM_REMOTE}-meta/ \ + --config /etc/rclone/rclone.conf --exclude "*/bands/**" --exclude "*/lock" \ + --log-file /var/log/tm-meta-sync.log --log-level NOTICE +# Only after a successful sync: vm-status and backup-health read its age. +ExecStartPost=/usr/bin/touch /var/lib/cloudmachine/tm-meta-sync.ok diff --git a/vm/systemd/tm-meta-sync.timer b/vm/systemd/tm-meta-sync.timer new file mode 100644 index 0000000..0f04638 --- /dev/null +++ b/vm/systemd/tm-meta-sync.timer @@ -0,0 +1,11 @@ +# CloudMachine: tm-meta-sync every 15 minutes. A backup's metadata changes at +# the end of each hourly run, so a restore is at most one run behind the bands. +[Unit] +Description=Copy of the Time Machine image's files outside bands/, every 15 min + +[Timer] +OnBootSec=5min +OnUnitActiveSec=15min + +[Install] +WantedBy=timers.target From 1035b956e1a0ae92b0993efc112dfdc533e80190 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 18:27:48 +0100 Subject: [PATCH 2/5] fix(budget): a stored Drive reading counts only for the folder it measured The 14:31 reading (101 MB, before the move to -tm) was still shown at 18:00 while the folder held 215 GiB: younger than six hours, it was never measured again. Readings now carry their folder; another folder's reading, or one stored before this change, is measured again instead of shown. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AU1MhxJW1e9oTB2XvjX8WT --- .../CloudMachineCore/MachineBudget.swift | 20 +++++++++++++++++-- .../MachineBudgetTests.swift | 13 ++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mac-app/Sources/CloudMachineCore/MachineBudget.swift b/mac-app/Sources/CloudMachineCore/MachineBudget.swift index 56a77e1..788493e 100644 --- a/mac-app/Sources/CloudMachineCore/MachineBudget.swift +++ b/mac-app/Sources/CloudMachineCore/MachineBudget.swift @@ -116,17 +116,33 @@ public enum MachineBudget { public struct Usage: Codable, Equatable { public var bytes: UInt64 public var measuredAt: Date + /// The Drive folder measured. `nil` in readings stored before 1.3.16. + public var path: String? + + public init(bytes: UInt64, measuredAt: Date, path: String? = nil) { + self.bytes = bytes + self.measuredAt = measuredAt + self.path = path + } public var gb: Double { Double(bytes) / 1_073_741_824 } } + /// A stored reading counts only for the folder backed up now. On 27.10.2026 + /// the reading taken at 14:31 (101 MB, before the move to `-tm`) + /// was still shown at 18:00, when the folder held 215 GiB: it was younger + /// than `measurementMaxAge`, so nothing measured again. + static func usage(_ stored: Usage?, appliesTo path: String) -> Usage? { + stored?.path == path ? stored : nil + } + static var usageFile: URL { CMPaths.appSupportDir.appendingPathComponent("drive-usage.json") } public static func storedUsage() -> Usage? { guard let data = try? Data(contentsOf: usageFile) else { return nil } let decoder = JSONDecoder() decoder.dateDecodingStrategy = .iso8601 - return try? decoder.decode(Usage.self, from: data) + return usage(try? decoder.decode(Usage.self, from: data), appliesTo: VMLayout.remote) } /// `rclone size --json` output -> bytes. @@ -149,7 +165,7 @@ public enum MachineBudget { CMLogger.log("Could not measure the size of \(path) on Google Drive") return nil } - let usage = Usage(bytes: bytes, measuredAt: now) + let usage = Usage(bytes: bytes, measuredAt: now, path: path) let encoder = JSONEncoder() encoder.dateEncodingStrategy = .iso8601 if let data = try? encoder.encode(usage) { diff --git a/mac-app/Tests/CloudMachineAppTests/MachineBudgetTests.swift b/mac-app/Tests/CloudMachineAppTests/MachineBudgetTests.swift index 30da0fd..173dc43 100644 --- a/mac-app/Tests/CloudMachineAppTests/MachineBudgetTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/MachineBudgetTests.swift @@ -173,4 +173,17 @@ final class MachineBudgetTests: XCTestCase { XCTAssertEqual(MachineBudget.standing(limitGB: 1000, usage: usage(1100), now: now), .over) XCTAssertEqual(MachineBudget.standing(limitGB: nil, usage: usage(1100), now: now), .notSet) } + + /// A reading of another folder (or one stored before readings named their + /// folder) is no reading: it is measured again instead of shown. + func testStoredUsageCountsOnlyForTheCurrentFolder() { + let at = Date(timeIntervalSince1970: 1_800_000_000) + let here = MachineBudget.Usage(bytes: 1, measuredAt: at, path: "gdrive:CloudMachine/a-tm") + XCTAssertEqual(MachineBudget.usage(here, appliesTo: "gdrive:CloudMachine/a-tm"), here) + XCTAssertNil(MachineBudget.usage(here, appliesTo: "gdrive:CloudMachine/b-tm")) + XCTAssertNil( + MachineBudget.usage( + MachineBudget.Usage(bytes: 1, measuredAt: at), appliesTo: "gdrive:CloudMachine/a-tm")) + XCTAssertNil(MachineBudget.usage(nil, appliesTo: "gdrive:CloudMachine/a-tm")) + } } From d9ab3f6f1ab866acafe587b1c9527c8ed3970623 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 18:27:48 +0100 Subject: [PATCH 3/5] docs: the Homebrew tap updates itself renacode-tap is on the bypass list of homebrew-tap's main ruleset since 2026-10-27 and 1.3.15 reached the tap on its own. The hand-made pull request is now only the workaround for a failed push. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AU1MhxJW1e9oTB2XvjX8WT --- README.md | 3 +-- packaging/README.md | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 53ff6f7..36ad062 100644 --- a/README.md +++ b/README.md @@ -125,8 +125,7 @@ It works two ways: ### Upgrading and uninstalling `brew upgrade --cask renacode/tap/cloudmachine` replaces the app; the machine -keeps running. The cask in the tap can be a version behind the latest GitHub -Release: it is updated by a pull request after each release (see +keeps running. Each release updates the cask in the tap by itself (see `packaging/README.md`). If a new version does not show up, run `brew update` first - Homebrew refreshes the tap only now and then. When Homebrew reopens the app, it reloads the background agents so they run the new version; diff --git a/packaging/README.md b/packaging/README.md index 316cbd8..67ec213 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -6,14 +6,15 @@ brew install --cask renacode/tap/cloudmachine The cask lives in [RenaCode/homebrew-tap](https://github.com/RenaCode/homebrew-tap) and is generated: `.github/workflows/release.yml` fills -`homebrew/cloudmachine.rb.in` with the version and the DMG's sha256 and tries -to push it to the tap. Edit the template here, never the copy in the tap. +`homebrew/cloudmachine.rb.in` with the version and the DMG's sha256 and +pushes it to the tap. Edit the template here, never the copy in the tap. -**Current state:** the release publishes the DMG, but the push of the cask to -the tap is rejected - `homebrew-tap` protects `main` with a ruleset that -requires a pull request (`GH013`). After each release the cask goes into the -tap through a pull request opened by hand (1.3.13 and 1.3.14 went that way), so -until it is merged the tap can be a version behind the latest release. +The tap updates itself: `homebrew-tap` protects `main` with a ruleset that +requires a pull request, and the `renacode-tap` app is on that ruleset's bypass +list (since 2026-10-27), so the release job's push goes through. 1.3.15 was +the first release to reach the tap on its own; 1.3.13 and 1.3.14 went in by a +pull request opened by hand, before the bypass. That pull request is now only +the workaround for a failed push (see "The tap repository" below). ## Releases @@ -100,12 +101,16 @@ grant Full Disk Access again. (`actions/create-github-app-token`), so there is no token to renew. The app is `renacode-tap` (since 2026-10-23; before that a fine-grained PAT). -Without the app the release is still published, but the job fails and -prints the cask, so a missing tap update cannot go unnoticed. The job also -fails while the tap's `main` ruleset rejects the push (`GH013`, see the top of this -file): the release and its DMG are there, only the tap is stale. Fill the -template with the release's version and the sha256 from its `.sha256` file and -open a pull request to `homebrew-tap` with the result. +3. Add the app to the bypass list of the `main` ruleset of `homebrew-tap` + (*Settings - Rules - Rulesets*, bypass mode *Always*). Without it the push + is rejected with `GH013` - the ruleset requires a pull request. + +Without the app, or without its bypass, the release is still published, but +the job fails and prints the cask, so a missing tap update cannot go +unnoticed: the release and its DMG are there, only the tap is stale. The +workaround is a pull request by hand: fill the template with the release's +version and the sha256 from its `.sha256` file and open a pull request to +`homebrew-tap` with the result. ## What the cask does and deliberately does not do From 2b80d372363158af36d356d5847d03616af40cf7 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 18:42:25 +0100 Subject: [PATCH 4/5] feat(vm): report Google's daily upload limit; vm-guard --dry-run At the 750 GB limit rclone's write-back retries every upload, at most 5 min apart, and resumes by itself (rclone 1.75.1 vfs/vfscache/writeback) - but nothing reported it: erroredFiles counts cache cleaning, not uploads, and Time Machine kept finishing into the machine's buffer, so every light stayed green while nothing reached Drive. The status reads the end of rclone's log: a refusal newer than the last upload means the limit holds. vm-status and backup-health report it. vm-guard --dry-run prints the guard's decision for every Time Machine state instead of acting, so thresholds can be tried on a throwaway machine without pausing the live backup (Time Machine is global). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AU1MhxJW1e9oTB2XvjX8WT --- .../CloudMachineAgent/VMCommands.swift | 36 +++++++++++++++++-- .../CloudMachineCore/BackupHealth.swift | 18 ++++++++++ .../CloudMachineCore/L10nPolish+Machine.swift | 4 +++ .../CloudMachineCore/L10nPolish+VM.swift | 18 ++++++++++ .../Sources/CloudMachineCore/VMLayout.swift | 1 + .../Sources/CloudMachineCore/VMStatus.swift | 30 ++++++++++++++++ .../BackupHealthTests.swift | 15 ++++++++ .../CloudMachineAppTests/VMLayoutTests.swift | 9 +++++ 8 files changed, 128 insertions(+), 3 deletions(-) diff --git a/mac-app/Sources/CloudMachineAgent/VMCommands.swift b/mac-app/Sources/CloudMachineAgent/VMCommands.swift index bfa98d2..67b236f 100644 --- a/mac-app/Sources/CloudMachineAgent/VMCommands.swift +++ b/mac-app/Sources/CloudMachineAgent/VMCommands.swift @@ -86,10 +86,40 @@ struct VMGuardCommand: AsyncParsableCommand { ) var minFreeGiB: Int = VMGuard.Thresholds().minFreeGiB + @Flag( + name: .long, + help: ArgumentHelp( + L10n.tr( + "Only print what the guard would decide - for trying thresholds on a throwaway machine."))) + var dryRun = false + func run() async throws { - let guardian = VMGuard( - thresholds: .init(highGiB: highGiB, lowGiB: lowGiB, minFreeGiB: minFreeGiB)) - await guardian.step() + let thresholds = VMGuard.Thresholds(highGiB: highGiB, lowGiB: lowGiB, minFreeGiB: minFreeGiB) + guard dryRun else { + await VMGuard(thresholds: thresholds).step() + return + } + // Time Machine is global: a guard tried on `cmtest` would pause the live + // machine's backup. Print the decision for every state it depends on. + guard let reading = await VMLayout.currentReading() else { + print(L10n.tr("No answer from the machine (rclone rc) - the guard would do nothing.")) + throw ExitCode.failure + } + let unsent = reading.buffer.unsentBytes.map { "\($0 / VMGuard.gib)" } ?? "?" + print( + L10n.tr( + "Unsent %@ GiB, free %@ GiB (pause above %@ GiB unsent or below %@ GiB free, resume below %@ GiB).", + unsent, "\(reading.freeBytes / VMGuard.gib)", "\(highGiB)", "\(minFreeGiB)", + "\(lowGiB)")) + for (running, paused) in [(true, false), (true, true), (false, true)] { + let action = VMGuard.decide( + reading: reading, backupRunning: running, pausedByGuard: paused, thresholds: thresholds) + print( + L10n.tr( + "Backup %@, paused by the guard %@: %@", + running ? L10n.tr("running") : L10n.tr("stopped"), + paused ? L10n.tr("yes") : L10n.tr("no"), "\(action)")) + } } } diff --git a/mac-app/Sources/CloudMachineCore/BackupHealth.swift b/mac-app/Sources/CloudMachineCore/BackupHealth.swift index e38156b..cf61893 100644 --- a/mac-app/Sources/CloudMachineCore/BackupHealth.swift +++ b/mac-app/Sources/CloudMachineCore/BackupHealth.swift @@ -181,6 +181,9 @@ public enum BackupHealth { // Seconds since the last copy of the image's metadata to Drive // (`.infinity` = never); `nil` = not known or nothing to copy yet. metaSyncAge: TimeInterval? = nil, + // Seconds since Google's daily upload limit last refused an upload, while + // it still holds (`VMStatus.Report.uploadLimitAge`); `nil`/0 = it does not. + uploadLimitAge: TimeInterval? = nil, maxAgeHours: Double = BackupHealth.maxAgeHours, // Whether the startup grace period is in progress - see `startupGraceMinutes`. withinStartupGrace: Bool = false, @@ -381,6 +384,20 @@ public enum BackupHealth { code: "metadata-sync-stale")) } + // Not a failure that needs fixing - it passes by itself - but while it + // lasts every "successful" backup exists only on this Mac. Without this + // the panel stayed green: Time Machine finishes into the machine's + // buffer, and the guard pauses only at 110 GiB. + if let uploadLimitAge, uploadLimitAge > 0 { + problems.append( + Problem( + summary: L10n.tr("Google Drive's daily upload limit stops the upload"), + detail: L10n.tr( + "Google last refused an upload %@ ago (750 GB a day). Backups since then are only in the machine's buffer on this Mac. rclone retries every 5 min and resumes by itself when the limit passes; the guard pauses Time Machine if more than %@ GiB waits.", + formatAge(uploadLimitAge), "\(VMGuard.Thresholds().highGiB)"), + code: "drive-upload-limit")) + } + // Space on the Drive. Running out of it stops every upload, and the // machine's buffer then fills up - this has to be known EARLIER, not from // a failure. The threshold is counted in cycles, not in percent: at a @@ -589,6 +606,7 @@ public enum BackupHealth { metaSyncAge: report.flatMap { VMStatus.metaSyncOverdue($0) != nil ? $0.metaSync?.ageSeconds : nil }, + uploadLimitAge: report?.uploadLimitAge, maxAgeHours: maxAgeHours, // The REAL clock, not `now`: tests substitute a `now` from the past, and // uptime computed from it would come out negative, i.e. "startup in diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift index 7af60da..c0af30c 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+Machine.swift @@ -178,5 +178,9 @@ extension L10nPolish { "Zdalne sterowanie rclone jest otwarte na tym Macu", "statfs('/System/Volumes/Data') returned an error, so nobody knows whether the machine's disk can still grow on this Mac.": "statfs('/System/Volumes/Data') zwrócił błąd, więc nikt nie wie, czy dysk maszyny może jeszcze rosnąć na tym Macu.", + "Google Drive's daily upload limit stops the upload": + "Dzienny limit wysyłki Google Drive zatrzymał wysyłkę", + "Google last refused an upload %@ ago (750 GB a day). Backups since then are only in the machine's buffer on this Mac. rclone retries every 5 min and resumes by itself when the limit passes; the guard pauses Time Machine if more than %@ GiB waits.": + "Google ostatnio odrzucił wysyłkę %@ temu (750 GB na dobę). Kopie od tego czasu są tylko w buforze maszyny na tym Macu. rclone ponawia co 5 min i sam wznowi, gdy limit minie; strażnik wstrzyma Time Machine, jeśli czeka więcej niż %@ GiB.", ] } diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift index 7837755..bddd4d7 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift @@ -58,6 +58,8 @@ extension L10nPolish { "Metadane obrazu: ostatnia kopia %@ temu - ZALEGŁA (%@)", "Image metadata: not copied yet - waits for vm-prepare-bundle": "Metadane obrazu: jeszcze bez kopii - czeka na vm-prepare-bundle", + "Backup %@, paused by the guard %@: %@": + "Kopia %@, wstrzymana przez strażnika %@: %@", "Free in machine: %@ GiB": "Wolne w maszynie: %@ GiB", "fsck_apfs -n found problems: %@": @@ -106,6 +108,10 @@ extension L10nPolish { "Pakowanie przerobionego obrazu nie powiodło się: %@", "Packing the image failed: %@": "Pakowanie obrazu nie powiodło się: %@", + "No answer from the machine (rclone rc) - the guard would do nothing.": + "Brak odpowiedzi maszyny (rclone rc) - strażnik nic by nie zrobił.", + "Only print what the guard would decide - for trying thresholds on a throwaway machine.": + "Tylko wypisz, co strażnik by zdecydował - do prób progów na jednorazowej maszynie.", "Pause above this many GiB in the buffer.": "Wstrzymaj powyżej tylu GiB w buforze.", "Pause when the machine has less free than this, in GiB.": @@ -170,5 +176,17 @@ extension L10nPolish { "Bufor wysyłki: brak odpowiedzi rclone (rc %@)", "When Time Machine has created its empty image, stop the backup (tmutil stopbackup) and run: cloudmachine-agent vm-prepare-bundle": "Gdy Time Machine utworzy pusty obraz, zatrzymaj kopię (tmutil stopbackup) i uruchom: cloudmachine-agent vm-prepare-bundle", + "Unsent %@ GiB, free %@ GiB (pause above %@ GiB unsent or below %@ GiB free, resume below %@ GiB).": + "Niewysłane %@ GiB, wolne %@ GiB (pauza powyżej %@ GiB niewysłanych lub poniżej %@ GiB wolnego, wznowienie poniżej %@ GiB).", + "Upload: Google's daily limit refused an upload %@ ago - rclone retries every 5 min and resumes by itself": + "Wysyłka: dzienny limit Google odrzucił wysyłkę %@ temu - rclone ponawia co 5 min i sam wznowi", + "no": + "nie", + "running": + "trwa", + "stopped": + "zatrzymana", + "yes": + "tak", ] } diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift index 7e8119f..dfedc38 100644 --- a/mac-app/Sources/CloudMachineCore/VMLayout.swift +++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift @@ -38,6 +38,7 @@ public enum VMLayout { public static let rcAddress = "127.0.0.1:5572" /// Touched by `tm-meta-sync.service` after every successful sync. public static let metaSyncStamp = "/var/lib/cloudmachine/tm-meta-sync.ok" + public static let rcloneLog = "/var/log/rclone-tm.log" } /// The systemd units the share depends on, in start order, then the timer diff --git a/mac-app/Sources/CloudMachineCore/VMStatus.swift b/mac-app/Sources/CloudMachineCore/VMStatus.swift index a9f30c8..18aef80 100644 --- a/mac-app/Sources/CloudMachineCore/VMStatus.swift +++ b/mac-app/Sources/CloudMachineCore/VMStatus.swift @@ -18,6 +18,10 @@ public enum VMStatus { public var imageCount: Int? /// The last successful `tm-meta-sync`; `nil` = not read. public var metaSync: MetaSync? + /// Seconds since Google last refused an upload with its daily limit, + /// when that is newer than the last upload that went through: the limit + /// still holds. `0` = it does not; `nil` = not read. + public var uploadLimitAge: TimeInterval? public init() {} } @@ -52,8 +56,23 @@ public enum VMStatus { + "echo BUNDLEENV $([ -s \(VMLayout.Guest.bundleEnv) ] && echo 1 || echo 0); " + "echo IMAGES $(ls -d \(VMLayout.Guest.localRoot)/*.sparsebundle 2>/dev/null | wc -l); " + "echo METASYNC $(stat -c %Y \(VMLayout.Guest.metaSyncStamp) 2>/dev/null || echo 0) $(date +%s); " + + uploadLimitScript + VMLayout.readingScript + /// Google's daily upload limit (750 GB) is not an error rclone gives up + /// on: its write-back retries every upload, at most 5 min apart, and + /// resumes by itself when Google lets it (rclone 1.75.1, + /// vfs/vfscache/writeback). Nothing counts it either - `erroredFiles` is + /// about cache cleaning - so without this, Time Machine kept "succeeding" + /// into the machine's buffer while nothing reached Drive. Read from the + /// end of rclone's log: whichever came last, a refusal or an upload. + static let uploadLimitScript = + "l=$(tac \(VMLayout.Guest.rcloneLog) 2>/dev/null | grep -m1 -E " + + "'Received upload limit error|vfs cache: upload succeeded'); " + + "case \"$l\" in *'upload limit'*) " + + "echo UPLOADLIMIT $(date -d \"$(echo \"$l\" | cut -c1-19 | tr / -)\" +%s) $(date +%s);; " + + "*) echo UPLOADLIMIT 0 $(date +%s);; esac; " + static func report(scriptOutput: String) -> Report { var report = Report() for line in scriptOutput.split(separator: "\n") { @@ -71,6 +90,10 @@ public enum VMStatus { let now = Int(fields[2]) { report.metaSync = last == 0 ? .never : .age(TimeInterval(max(0, now - last))) + } else if fields.first == "UPLOADLIMIT", fields.count >= 3, let last = Int(fields[1]), + let now = Int(fields[2]) + { + report.uploadLimitAge = last == 0 ? 0 : TimeInterval(max(1, now - last)) } } report.reading = VMLayout.reading(scriptOutput: scriptOutput) @@ -194,6 +217,13 @@ public enum VMStatus { } if servicesHealthy(report.services) != true { healthy = false } if let line = metaSyncLine(report) { out.append(line) } + if let age = report.uploadLimitAge, age > 0 { + out.append( + L10n.tr( + "Upload: Google's daily limit refused an upload %@ ago - rclone retries every 5 min and resumes by itself", + BackupHealth.formatAge(age))) + healthy = false + } if metaSyncOverdue(report) == true { healthy = false } if let reading = report.reading { out.append( diff --git a/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift b/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift index d9e4522..4a9c714 100644 --- a/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift @@ -456,6 +456,21 @@ final class BackupHealthTests: XCTestCase { XCTAssertEqual(afterBoot.deferred.map(\.code), ["metadata-sync-stale"]) } + /// While Google's daily limit holds, backups "succeed" into the machine's + /// buffer only - every other light is green, so this one must not be. + func testDailyUploadLimitIsReported() { + func health(_ age: TimeInterval?) -> BackupHealth.Report { + BackupHealth.evaluate( + lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800), + result: 0, now: now, machineRunning: true, servicesHealthy: true, + destinationRegistered: true, erroredFiles: 0, outOfSpace: false, queueReadable: true, + uploadLimitAge: age) + } + XCTAssertTrue(health(nil).healthy) + XCTAssertTrue(health(0).healthy) + XCTAssertEqual(health(1200).problems.map(\.code), ["drive-upload-limit"]) + } + func testUptimeCanBeRead() { let uptime = BackupHealth.systemUptime() XCTAssertNotNil(uptime) diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift index 57c8670..d57a953 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -252,6 +252,15 @@ final class VMLayoutTests: XCTestCase { XCTAssertEqual(VMLayout.metaRemote, VMLayout.remote + "-meta") } + /// A refusal newer than the last upload means the limit still holds; `0` + /// from the machine means it does not; no line means not read. + func testUploadLimitIsReadFromTheEndOfTheLog() { + XCTAssertEqual(VMStatus.report(scriptOutput: "UPLOADLIMIT 1000 1600").uploadLimitAge, 600) + XCTAssertEqual(VMStatus.report(scriptOutput: "UPLOADLIMIT 0 1600").uploadLimitAge, 0) + XCTAssertNil(VMStatus.report(scriptOutput: "BUNDLEENV 1").uploadLimitAge) + XCTAssertTrue(VMStatus.script.contains("Received upload limit error")) + } + /// 27.10.2026: the image's Info.plist, token and mapped/ were only on the /// machine's disk, so the bands on Drive could not be restored. The template /// must install the copy, and the copy must neither carry the bands nor From b0dfe77e931fadde925d0cf9ee5d4a985288acf0 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Tue, 27 Oct 2026 19:14:24 +0100 Subject: [PATCH 5/5] fix(vm-setup): wait 50 min for a new machine's first boot On 27.10.2026 a new machine's first boot took 30 min: Lima copies every `mode: data` file with `sudo -iu root`, and while cloud-final runs logind cannot open the session, so pam_systemd times out after 2 min per file. `limactl start` gave up after its default 10 min and vm-setup failed on a new Mac or a restore. Later boots take ~12 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AU1MhxJW1e9oTB2XvjX8WT --- mac-app/Sources/CloudMachineCore/VMSetup.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mac-app/Sources/CloudMachineCore/VMSetup.swift b/mac-app/Sources/CloudMachineCore/VMSetup.swift index b215a57..2b316ca 100644 --- a/mac-app/Sources/CloudMachineCore/VMSetup.swift +++ b/mac-app/Sources/CloudMachineCore/VMSetup.swift @@ -58,10 +58,14 @@ public enum VMSetup { return fail(L10n.tr("limactl create failed: %@", VMLayout.errorTail(created))) } } + // The first boot of a new machine takes ~30 min on Ubuntu 24.04 (27.10.2026): + // Lima copies every `mode: data` file with `sudo -iu root`, and while + // cloud-final runs logind cannot open the session - pam_systemd waits 2 min + // per file. Lima gives up after 10 min by default; later boots take seconds. if status != "Running" { CMLogger.log("vm-setup: starting \(VMLayout.instanceName)") let started = try? await VMLayout.limactl( - ["start", "--tty=false", VMLayout.instanceName], timeout: 3600) + ["start", "--tty=false", "--timeout", "50m", VMLayout.instanceName], timeout: 3600) guard started?.succeeded == true else { return fail( L10n.tr(