vm-update: maszyna na szablon z aplikacji; alarm zawieszonej wysyłki

mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift

@@ -19,6 +19,7 @@ struct CloudMachineAgent: AsyncParsableCommand {
       InstallLaunchd.self,
       // Running.
       VMStatusCommand.self,
+      VMUpdateCommand.self,
       VMGuardCommand.self,
       BackupHealthCommand.self,
       PrepareShutdown.self,

mac-app/Sources/CloudMachineAgent/VMCommands.swift

@@ -123,6 +123,19 @@ struct VMGuardCommand: AsyncParsableCommand {
   }
 }
 
+struct VMUpdateCommand: AsyncParsableCommand {
+  static let configuration = CommandConfiguration(
+    commandName: "vm-update",
+    abstract: L10n.tr(
+      "Moves the machine onto this app's template (two restarts, never during a backup)."))
+
+  func run() async throws {
+    let outcome = await VMUpdate.run()
+    print(outcome.message)
+    if !outcome.succeeded { throw ExitCode.failure }
+  }
+}
+
 struct VMStatusCommand: AsyncParsableCommand {
   static let configuration = CommandConfiguration(
     commandName: "vm-status",
@@ -134,6 +147,12 @@ struct VMStatusCommand: AsyncParsableCommand {
     let (lines, healthy) = await VMStatus.lines()
     for line in lines { print(line) }
     print(L10n.tr("Drive folder:     %@", VMLayout.remote))
+    switch await VMUpdate.templateCurrent() {
+    case .some(true): print(L10n.tr("Template:         this app's"))
+    case .some(false):
+      print(L10n.tr("Template:         OLDER than this app's - run: cloudmachine-agent vm-update"))
+    case .none: print(L10n.tr("Template:         not compared"))
+    }
 
     let limit = MachineBudget.limitGB()
     print(

mac-app/Sources/CloudMachineCore/BackupHealth.swift

@@ -184,6 +184,9 @@ public enum BackupHealth {
     // Seconds since Google's daily upload limit last refused an upload, while
     // it still holds (`VMStatus.Report.uploadLimitAge`); `nil`/0 = it does not.
     uploadLimitAge: TimeInterval? = nil,
+    // Seconds uploads have been "in progress" with none going through
+    // (`VMStatus.uploadStalledFor`); `nil` = they do not hang.
+    uploadStalledFor: TimeInterval? = nil,
     maxAgeHours: Double = BackupHealth.maxAgeHours,
     // Whether the startup grace period is in progress - see `startupGraceMinutes`.
     withinStartupGrace: Bool = false,
@@ -398,6 +401,16 @@ public enum BackupHealth {
           code: "drive-upload-limit"))
     }
 
+    if let uploadStalledFor {
+      problems.append(
+        Problem(
+          summary: L10n.tr("The upload to Google Drive hangs"),
+          detail: L10n.tr(
+            "rclone has bands in progress, but none has been uploaded for %@ - usually a broken network connection, which rclone does not report as an error. Backups since then are only in the machine's buffer. When the network comes back the upload resumes by itself (on 28.10.2026 within 80 s); if it does not, restarting the machine (prepare-shutdown, limactl start) drops the dead connections.",
+            formatAge(uploadStalledFor)),
+          code: "upload-stalled"))
+    }
+
     // Space on the Drive. Running out of it stops every upload, and the
     // machine's buffer then fills up - this has to be known EARLIER, not from
     // a failure. The threshold is counted in cycles, not in percent: at a
@@ -607,6 +620,7 @@ public enum BackupHealth {
         VMStatus.metaSyncOverdue($0) != nil ? $0.metaSync?.ageSeconds : nil
       },
       uploadLimitAge: report?.uploadLimitAge,
+      uploadStalledFor: report.flatMap(VMStatus.uploadStalledFor),
       maxAgeHours: maxAgeHours,
       // The REAL clock, not `now`: tests substitute a `now` from the past, and
       // uptime computed from it would come out negative, i.e. "startup in

mac-app/Sources/CloudMachineCore/L10nPolish+VM.swift

@@ -188,5 +188,43 @@ extension L10nPolish {
       "zatrzymana",
     "yes":
       "tak",
+    "%@\nThe previous template is in %@ - copy it back over lima.yaml and run limactl start %@.":
+      "%@\nPoprzedni szablon jest w %@ - skopiuj go z powrotem na lima.yaml i uruchom limactl start %@.",
+    "Could not read this app's machine template (limactl template copy).":
+      "Nie udało się odczytać szablonu maszyny z tej aplikacji (limactl template copy).",
+    "Could not replace %@: %@":
+      "Nie udało się podmienić %@: %@",
+    "Machine %@ already runs this app's template.":
+      "Maszyna %@ już działa na szablonie z tej aplikacji.",
+    "Machine %@ now runs this app's template; the previous one is in %@.":
+      "Maszyna %@ działa teraz na szablonie z tej aplikacji; poprzedni jest w %@.",
+    "Moves the machine onto this app's template (two restarts, never during a backup).":
+      "Przenosi maszynę na szablon z tej aplikacji (dwa restarty, nigdy w trakcie kopii).",
+    "No machine %@ - create it with: cloudmachine-agent vm-setup":
+      "Brak maszyny %@ - utwórz ją: cloudmachine-agent vm-setup",
+    "Template:         OLDER than this app's - run: cloudmachine-agent vm-update":
+      "Szablon:          STARSZY niż w tej aplikacji - uruchom: cloudmachine-agent vm-update",
+    "Template:         not compared":
+      "Szablon:          nie porównany",
+    "Template:         this app's":
+      "Szablon:          z tej aplikacji",
+    "The machine has no %@ - run cloudmachine-agent vm-setup first.":
+      "Maszyna nie ma %@ - najpierw uruchom cloudmachine-agent vm-setup.",
+    "The machine restarted, but not every service is running - check: cloudmachine-agent vm-status. The previous template is in %@.":
+      "Maszyna wstała, ale nie wszystkie usługi działają - sprawdź: cloudmachine-agent vm-status. Poprzedni szablon jest w %@.",
+    "The template changes the machine itself (%@) - that needs a new machine, not an update.":
+      "Szablon zmienia samą maszynę (%@) - to wymaga nowej maszyny, nie aktualizacji.",
+    "The upload to Google Drive hangs":
+      "Wysyłka na Google Drive wisi",
+    "Time Machine is backing up - try again when it has finished.":
+      "Time Machine robi kopię - spróbuj ponownie po jej zakończeniu.",
+    "Upload:           HANGS - bands in progress, none uploaded for %@ (network?); see /var/log/rclone-tm.log in the machine":
+      "Wysyłka:          WISI - pasma w toku, żadne wysłane od %@ (sieć?); patrz /var/log/rclone-tm.log w maszynie",
+    "limactl stop failed - nothing was changed.":
+      "limactl stop się nie udał - nic nie zostało zmienione.",
+    "rclone has bands in progress, but none has been uploaded for %@ - usually a broken network connection, which rclone does not report as an error. Backups since then are only in the machine's buffer. When the network comes back the upload resumes by itself (on 28.10.2026 within 80 s); if it does not, restarting the machine (prepare-shutdown, limactl start) drops the dead connections.":
+      "rclone ma pasma w trakcie wysyłki, ale od %@ żadne nie zostało wysłane - zwykle to zerwane połączenie sieciowe, którego rclone nie zgłasza jako błędu. Kopie od tego czasu są tylko w buforze maszyny. Gdy sieć wróci, wysyłka wznowi się sama (28.10.2026 w 80 s); jeśli nie, restart maszyny (prepare-shutdown, limactl start) zrywa martwe połączenia.",
+    "tmutil did not answer - not restarting the machine while its state is unknown.":
+      "tmutil nie odpowiedział - nie restartuję maszyny, póki jej stan jest nieznany.",
   ]
 }

mac-app/Sources/CloudMachineCore/VMLayout.swift

@@ -324,7 +324,8 @@ public enum VMLayout {
   /// says so. The first start after a Mac restart can take minutes: rclone
   /// re-reads its buffer before it serves anything.
   public static func start() async -> CMActionResult {
-    let started = try? await limactl(["start", "--tty=false", instanceName], timeout: 1800)
+    let started = try? await limactl(
+      ["start", "--tty=false", "--timeout", "50m", instanceName], timeout: 3600)
     guard started?.succeeded == true else {
       return CMActionResult(
         succeeded: false,

mac-app/Sources/CloudMachineCore/VMSetup.swift

@@ -58,7 +58,7 @@ public enum VMSetup {
         return fail(L10n.tr("limactl create failed: %@", VMLayout.errorTail(created)))
       }
     }
-    // The first boot of a new machine takes ~30 min on Ubuntu 24.04 (27.10.2026):
+    // The first boot of a new machine can take ~30 min on Ubuntu 24.04 (27.10.2026; 1 min on 28.10):
     // Lima copies every `mode: data` file with `sudo -iu root`, and while
     // cloud-final runs logind cannot open the session - pam_systemd waits 2 min
     // per file. Lima gives up after 10 min by default; later boots take seconds.

mac-app/Sources/CloudMachineCore/VMStatus.swift

@@ -22,6 +22,9 @@ public enum VMStatus {
     /// when that is newer than the last upload that went through: the limit
     /// still holds. `0` = it does not; `nil` = not read.
     public var uploadLimitAge: TimeInterval?
+    /// Seconds since the last upload that went through, by the machine's
+    /// clock; `.infinity` = none in rclone's log; `nil` = not read.
+    public var lastUploadAge: TimeInterval?
 
     public init() {}
   }
@@ -40,6 +43,21 @@ public enum VMStatus {
     }
   }
 
+  /// Uploads in progress, and none has gone through for this long: they hang.
+  /// A 32 MB band takes seconds; on 27.10.2026, with the machine's network
+  /// cut, eight uploads stayed "in progress" for 47 min on dead connections
+  /// and nothing - no error line, no counter - said so.
+  public static let uploadStallSeconds: TimeInterval = 15 * 60
+
+  /// Seconds the upload has hung, or `nil` when it does not (or not known).
+  /// A log without a single upload says nothing: a new machine has none yet.
+  public static func uploadStalledFor(_ report: Report) -> TimeInterval? {
+    guard let inProgress = report.reading?.buffer.uploadsInProgress, inProgress > 0,
+      let age = report.lastUploadAge, age.isFinite, age > uploadStallSeconds
+    else { return nil }
+    return age
+  }
+
   /// Whether the metadata copy is overdue. Only for an image that went
   /// through `vm-prepare-bundle`: before it, the sync is skipped by design.
   /// `nil` = not known (not read, or no prepared image).
@@ -57,6 +75,9 @@ public enum VMStatus {
     + "echo IMAGES $(ls -d \(VMLayout.Guest.localRoot)/*.sparsebundle 2>/dev/null | wc -l); "
     + "echo METASYNC $(stat -c %Y \(VMLayout.Guest.metaSyncStamp) 2>/dev/null || echo 0) $(date +%s); "
     + uploadLimitScript
+    + "l=$(tac \(VMLayout.Guest.rcloneLog) 2>/dev/null | grep -m1 'vfs cache: upload succeeded'); "
+    + "if [ -n \"$l\" ]; then echo LASTUPLOAD $(date -d \"$(echo \"$l\" | cut -c1-19 | tr / -)\" +%s) $(date +%s); "
+    + "else echo LASTUPLOAD 0 $(date +%s); fi; "
     + VMLayout.readingScript
 
   /// Google's daily upload limit (750 GB) is not an error rclone gives up
@@ -90,6 +111,10 @@ public enum VMStatus {
         let now = Int(fields[2])
       {
         report.metaSync = last == 0 ? .never : .age(TimeInterval(max(0, now - last)))
+      } else if fields.first == "LASTUPLOAD", fields.count >= 3, let last = Int(fields[1]),
+        let now = Int(fields[2])
+      {
+        report.lastUploadAge = last == 0 ? .infinity : TimeInterval(max(0, now - last))
       } else if fields.first == "UPLOADLIMIT", fields.count >= 3, let last = Int(fields[1]),
         let now = Int(fields[2])
       {
@@ -217,6 +242,13 @@ public enum VMStatus {
     }
     if servicesHealthy(report.services) != true { healthy = false }
     if let line = metaSyncLine(report) { out.append(line) }
+    if let stalled = uploadStalledFor(report) {
+      out.append(
+        L10n.tr(
+          "Upload:           HANGS - bands in progress, none uploaded for %@ (network?); see /var/log/rclone-tm.log in the machine",
+          BackupHealth.formatAge(stalled)))
+      healthy = false
+    }
     if let age = report.uploadLimitAge, age > 0 {
       out.append(
         L10n.tr(

mac-app/Sources/CloudMachineCore/VMUpdate.swift

@@ -0,0 +1,171 @@
+import Foundation
+
+/// `vm-update`: brings an existing machine onto the template shipped with this
+/// app.
+///
+/// Lima embeds the template's files into `~/.lima/<machine>/lima.yaml` when the
+/// machine is created, and boots from that copy. A newer template in a newer
+/// release therefore never reaches a machine that already exists - on
+/// 27.10.2026 the live `cmsmb` still ran its morning prototype after 1.3.15
+/// shipped the template. This replaces the machine's copy and restarts it twice:
+/// the first boot rewrites the units (provisioning), the second starts the
+/// services from them. The steps were tried on a throwaway machine and then
+/// moved `cmsmb` in 1:52 min.
+public enum VMUpdate {
+  /// The machine's own copy of the template.
+  public static var instanceYAML: URL {
+    VMLayout.limaHome.appendingPathComponent(VMLayout.instanceName)
+      .appendingPathComponent("lima.yaml")
+  }
+
+  /// Top-level settings the update must not change: a different disk or VM
+  /// type is a new machine, not an update.
+  static let fixedKeys = ["vmType", "arch", "cpus", "memory", "disk"]
+
+  /// The top-level `key: value` lines of a Lima YAML.
+  static func topLevel(_ yaml: String) -> [String: String] {
+    var values: [String: String] = [:]
+    for line in yaml.split(separator: "\n") where !line.hasPrefix(" ") && !line.hasPrefix("#") {
+      let parts = line.split(separator: ":", maxSplits: 1)
+      guard parts.count == 2 else { continue }
+      let value = parts[1].trimmingCharacters(in: .whitespaces)
+      if !value.isEmpty { values[String(parts[0])] = value }
+    }
+    return values
+  }
+
+  /// The fixed keys that differ, as `key: old -> new`. Empty = safe to update.
+  static func fixedDifferences(current: String, new: String) -> [String] {
+    let old = topLevel(current)
+    let fresh = topLevel(new)
+    return fixedKeys.compactMap { key in
+      old[key] == fresh[key] ? nil : "\(key): \(old[key] ?? "-") -> \(fresh[key] ?? "-")"
+    }
+  }
+
+  /// This app's template with its files embedded, as Lima stores it.
+  static func embeddedTemplate() async -> String? {
+    guard let dir = VMLayout.templateDir else { return nil }
+    let out = FileManager.default.temporaryDirectory
+      .appendingPathComponent("cloudmachine-\(UUID().uuidString).yaml")
+    defer { try? FileManager.default.removeItem(at: out) }
+    guard
+      let copied = try? await VMLayout.limactl(
+        ["template", "copy", "--embed", dir.appendingPathComponent("cmsmb.yaml").path, out.path],
+        timeout: 120),
+      copied.succeeded
+    else { return nil }
+    return try? String(contentsOf: out, encoding: .utf8)
+  }
+
+  /// Whether the machine runs this app's template. `nil` = could not tell
+  /// (no machine, no template, limactl did not answer).
+  public static func templateCurrent() async -> Bool? {
+    guard let current = try? String(contentsOf: instanceYAML, encoding: .utf8),
+      let fresh = await embeddedTemplate()
+    else { return nil }
+    return current == fresh
+  }
+
+  public static func run() async -> CMActionResult {
+    func fail(_ message: String) -> CMActionResult {
+      CMLogger.log("vm-update: \(message.replacingOccurrences(of: "\n", with: " "))")
+      return CMActionResult(succeeded: false, message: message)
+    }
+    func step(_ text: String) { CMLogger.log("vm-update: \(text)") }
+
+    // 1. Never under a running backup: the machine stops twice.
+    switch await TimeMachineStatus.runningState() {
+    case .none:
+      return fail(
+        L10n.tr("tmutil did not answer - not restarting the machine while its state is unknown."))
+    case .some(true):
+      return fail(L10n.tr("Time Machine is backing up - try again when it has finished."))
+    case .some(false):
+      break
+    }
+
+    // 2. The two templates.
+    guard let current = try? String(contentsOf: instanceYAML, encoding: .utf8) else {
+      return fail(
+        L10n.tr(
+          "No machine %@ - create it with: cloudmachine-agent vm-setup", VMLayout.instanceName))
+    }
+    guard let fresh = await embeddedTemplate() else {
+      return fail(L10n.tr("Could not read this app's machine template (limactl template copy)."))
+    }
+    if current == fresh {
+      return CMActionResult(
+        succeeded: true,
+        message: L10n.tr("Machine %@ already runs this app's template.", VMLayout.instanceName))
+    }
+    let differences = fixedDifferences(current: current, new: fresh)
+    guard differences.isEmpty else {
+      return fail(
+        L10n.tr(
+          "The template changes the machine itself (%@) - that needs a new machine, not an update.",
+          differences.joined(separator: ", ")))
+    }
+
+    // 3. The units read these files; a machine without them was not set up.
+    guard
+      (try? await VMLayout.rootShell(
+        "test -s \(VMLayout.Guest.vmEnv) && test -s /etc/cloudmachine/tm-ids.env", timeout: 60))?
+        .succeeded == true
+    else {
+      return fail(
+        L10n.tr(
+          "The machine has no %@ - run cloudmachine-agent vm-setup first.", VMLayout.Guest.vmEnv))
+    }
+
+    // 4. Replace the copy, keeping the old one next to it.
+    step("stopping \(VMLayout.instanceName)")
+    guard
+      (try? await VMLayout.limactl(["stop", VMLayout.instanceName], timeout: 600))?.succeeded
+        == true
+    else { return fail(L10n.tr("limactl stop failed - nothing was changed.")) }
+    let stamp = DateFormatter()
+    stamp.dateFormat = "yyyyMMdd-HHmm"
+    let backup = instanceYAML.deletingLastPathComponent()
+      .appendingPathComponent("lima.yaml.before-\(stamp.string(from: Date()))")
+    do {
+      try FileManager.default.copyItem(at: instanceYAML, to: backup)
+      try fresh.write(to: instanceYAML, atomically: true, encoding: .utf8)
+    } catch {
+      _ = await VMLayout.start()
+      return fail(
+        L10n.tr("Could not replace %@: %@", instanceYAML.path, error.localizedDescription))
+    }
+    step("template replaced, old copy in \(backup.lastPathComponent)")
+
+    // 5. Boot to provision, boot again to run the new units.
+    for pass in 1...2 {
+      if pass == 2 {
+        _ = try? await VMLayout.limactl(["stop", VMLayout.instanceName], timeout: 600)
+      }
+      step("start \(pass) of 2")
+      let started = await VMLayout.start()
+      guard started.succeeded else {
+        return fail(
+          L10n.tr(
+            "%@\nThe previous template is in %@ - copy it back over lima.yaml and run limactl start %@.",
+            started.message, backup.path, VMLayout.instanceName))
+      }
+    }
+
+    // 6. The share must be back on the Drive layers.
+    guard let report = await VMStatus.currentReport(),
+      VMStatus.servicesHealthy(report.services) == true
+    else {
+      return fail(
+        L10n.tr(
+          "The machine restarted, but not every service is running - check: cloudmachine-agent vm-status. The previous template is in %@.",
+          backup.path))
+    }
+    return CMActionResult(
+      succeeded: true,
+      message: L10n.tr(
+        "Machine %@ now runs this app's template; the previous one is in %@.",
+        VMLayout.instanceName, backup.path))
+  }
+}

mac-app/Tests/CloudMachineAppTests/BackupHealthTests.swift

@@ -471,6 +471,15 @@ final class BackupHealthTests: XCTestCase {
     XCTAssertEqual(health(1200).problems.map(\.code), ["drive-upload-limit"])
   }
 
+  func testHangingUploadIsReported() {
+    let report = BackupHealth.evaluate(
+      lastSuccess: now.addingTimeInterval(-1800), lastAttempt: now.addingTimeInterval(-1800),
+      result: 0, now: now, machineRunning: true, servicesHealthy: true,
+      destinationRegistered: true, erroredFiles: 0, outOfSpace: false, queueReadable: true,
+      uploadStalledFor: 2820)
+    XCTAssertEqual(report.problems.map(\.code), ["upload-stalled"])
+  }
+
   func testUptimeCanBeRead() {
     let uptime = BackupHealth.systemUptime()
     XCTAssertNotNil(uptime)