Include SSH logs in workspace diagnostics

CHANGELOG.md

@@ -4,15 +4,19 @@
 
 ### Added
 
-- include a Coder support bundle for the selected workspace and agent only when using the environment's
-  _Collect logs_ action in Toolbox
+- include a Coder support bundle for the selected workspace and agent only when using the environment's _Collect logs_
+  action in Toolbox
+- include active and recent local CLI SSH logs in workspace log collection, using the configured or default CLI log
+  directory
 
 ### Changed
 
 - store CLI session tokens in the OS keyring by default on supported platforms; set `useKeyring` to `false` to opt out
-- share the same session token between REST and CLI authentication, passing it through the environment and sanitizing logs
+- share the same session token between REST and CLI authentication, passing it through the environment and sanitizing
+  logs
 - remove stored credentials on explicit logout, and remove old plaintext session files after successful keyring login
-- bound credential and header commands to 60 seconds while preserving unbounded workspace starts and complete progress lines
+- bound credential and header commands to 60 seconds while preserving unbounded workspace starts and complete progress
+  lines
 
 - upgraded the Toolbox plugin API, dropping support for Toolbox versions older than 3.7.2
 - display live workspace start output from the Coder CLI and poll build status and provisioner output while updating,

README.md

@@ -409,6 +409,33 @@ The plugin runs `coder support bundle` using the deployment's existing CLI login
 and its agent, when available. It places `coder-support.zip` inside the environment's diagnostic directory in the
 Toolbox log archive. Collection does not start a stopped workspace.
 
+The same diagnostic directory also includes recent local CLI SSH logs under `coder-ssh/`, even if support bundle
+generation fails. The plugin uses the configured SSH log directory, then `CODER_SSH_LOG_DIR`, or otherwise the CLI's
+default state directory: `$XDG_STATE_HOME/coder/logs` when set to an absolute path, falling back to
+`~/.local/state/coder/logs` on Linux, `~/Library/Application Support/coder/logs` on macOS, or
+`%LOCALAPPDATA%/coder/logs` on Windows.
+
+Only regular files directly inside the SSH log directory with these names are eligible:
+
+- `coder-ssh*.log`: CLI SSH session logs.
+- `coder-ssh*.log.<number>`: rotated CLI SSH logs, where `<number>` contains only digits, such as `.log.1`.
+- `<PID>.log`: legacy SSH logs with a numeric filename, such as `12345.log`.
+
+Collection skips subdirectories, symbolic links, and other filenames, including rotated legacy files such as
+`12345.log.1`. These local logs can include sessions for other workspaces that share the log directory.
+
+The following fixed limits apply and are not configurable:
+
+- **Age:** include files modified within the last **3 days (72 hours)**, based on the file's last-modified time.
+  The identified active SSH session's log for the selected workspace/agent is included regardless of age.
+  Rotated logs remain subject to the age cutoff. If the active log cannot be identified, the cutoff applies to
+  every file.
+- **Size:** copy only the last **50 MiB (52,428,800 bytes) per file**, including the active log. This is a limit on
+  each file's uncompressed contents, not on the total archive size.
+
+These limits apply only to the collected copies; the original log files are unchanged. Missing or unreadable SSH
+logs do not prevent collecting the remaining diagnostics.
+
 The bundle can contain deployment health and configuration, network diagnostics, workspace build logs, template
 source, and agent diagnostics. Available information depends on your permissions and workspace connectivity.
 Review the bundle before sharing, following
@@ -417,7 +444,7 @@ Review the bundle before sharing, following
 Support bundles require a Coder CLI that supports `coder support bundle` (Coder 2.10 or newer). Collection runs until
 the CLI finishes or you cancel log collection. If the CLI is unavailable or unsupported, or authentication or
 connectivity fails, the plugin removes any partial bundle and attempts to include
-`coder-support-error.txt` instead. Toolbox's other diagnostics remain available.
+`coder-support-error.txt` instead. Local SSH logs and Toolbox's other diagnostics remain available.
 
 ### HTTP Request Logging
 

gradle.properties

@@ -1,3 +1,3 @@
-version=0.9.4
+version=0.9.5
 group=com.coder.toolbox
 name=coder-toolbox

src/main/kotlin/com/coder/toolbox/CoderRemoteEnvironment.kt

@@ -38,6 +38,7 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.channels.Channel
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.ensureActive
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.update
@@ -114,6 +115,8 @@ class CoderRemoteEnvironment(
     private val networkMetricsMarshaller = Moshi.Builder().build().adapter(NetworkMetrics::class.java)
     private val proxyCommandHandle = SshCommandProcessHandle(context)
     private var pollJob: Job? = null
+    @Volatile
+    private var sshPid: Long? = null
 
     init {
         if (context.settingsStore.shouldAutoConnect(id)) {
@@ -128,8 +131,13 @@ class CoderRemoteEnvironment(
     internal fun currentSessionId(): SessionId? =
         agent?.let { SessionIdRegistry.findSession(workspace.name, it.name) }
 
-    override val diagnosticInfoCollector: DiagnosticInfoCollector =
-        CoderSupportBundleCollector(context.logger) { outputFile ->
+    override val diagnosticInfoCollector: DiagnosticInfoCollector
+        get() = CoderSupportBundleCollector(
+            context.logger,
+            cli.getSshLogDirectory(),
+            ::currentSessionId,
+            sshPid,
+        ) { outputFile ->
             cli.supportBundle(WorkspaceAddress.from(workspace, agent), outputFile)
         }
 
@@ -312,6 +320,7 @@ class CoderRemoteEnvironment(
         // Toolbox can invoke this hook again while retrying without first reporting a disconnect.
         // Replace the previous poller so one environment never leaves multiple pollers behind.
         pollJob?.cancel()
+        sshPid = null
         pollJob = pollNetworkMetrics()
     }
 
@@ -323,6 +332,8 @@ class CoderRemoteEnvironment(
                 val sessionId = currentSessionId()
                 context.logger.debug(sessionId, "Searching SSH command's PID for workspace $id...")
                 val pid = proxyCommandHandle.findByWorkspaceAndAgent(workspace, currentAgent)
+                ensureActive()
+                sshPid = pid
                 if (pid == null) {
                     context.logger.debug(sessionId, "No SSH command PID was found for workspace $id")
                     delay(POLL_INTERVAL)
@@ -366,6 +377,7 @@ class CoderRemoteEnvironment(
     fun dispose() {
         pollJob?.cancel()
         pollJob = null
+        sshPid = null
         isConnected.update { false }
         val removedSessionId = agent?.let {
             SessionIdRegistry.removeSession(workspace.name, it.name)
@@ -401,6 +413,7 @@ class CoderRemoteEnvironment(
         context.logger.info(sessionId, "Stopping the network metrics poll job for $id")
         pollJob?.cancel()
         pollJob = null
+        sshPid = null
         connectionRequest.update { false }
         isConnected.update { false }
         if (isManual) {

src/main/kotlin/com/coder/toolbox/cli/CoderCLIManager.kt

@@ -129,6 +129,12 @@ class CoderCLIManager(
     val localBinaryPath: Path = context.settingsStore.binPath(deploymentURL)
     val coderConfigPath: Path = context.settingsStore.dataDir(deploymentURL).resolve("config")
 
+    /** Returns the effective SSH log directory for this CLI's current settings. */
+    internal fun getSshLogDirectory(): Path = getSshLogDirectory(
+        context.settingsStore.sshLogDirectory,
+        os = currentOs,
+    )
+
     private fun createDownloadService(): CoderDownloadService {
 
         val okHttpClient = CoderHttpClientBuilder.default(context)
@@ -679,6 +685,45 @@ class CoderCLIManager(
         private const val CODER_SESSION_TOKEN_ENV_VAR = "CODER_SESSION_TOKEN"
 
         internal fun supportsKeyringStorage(os: OS?): Boolean = os == OS.MAC || os == OS.WINDOWS
+
+        /** Resolves the log directory used by `coder ssh`, including its XDG state directory default. */
+        internal fun getSshLogDirectory(
+            configuredDirectory: String?,
+            env: Map<String, String> = System.getenv(),
+            os: OS? = getOS(),
+            home: Path = Path.of(
+                env[if (os == OS.WINDOWS) "USERPROFILE" else "HOME"].orEmpty().ifEmpty {
+                    System.getProperty("user.home")
+                },
+            ),
+        ): Path {
+            val directoryOverride = configuredDirectory?.takeIf { it.isNotBlank() }
+                ?: env["CODER_SSH_LOG_DIR"]?.takeIf { it.isNotEmpty() }
+            if (directoryOverride != null) return Path.of(directoryOverride)
+
+            // The CLI uses github.com/adrg/xdg: XDG_STATE_HOME must be absolute after home expansion.
+            val stateHome = absoluteStatePath(env["XDG_STATE_HOME"], home, os) ?: when (os) {
+                OS.MAC -> home.resolve("Library/Application Support")
+                OS.WINDOWS -> absoluteStatePath(env["LOCALAPPDATA"], home, os) ?: home.resolve("AppData/Local")
+                else -> home.resolve(".local/state")
+            }
+            return stateHome.resolve("coder/logs")
+        }
+
+        private fun absoluteStatePath(value: String?, home: Path, os: OS?): Path? {
+            if (value.isNullOrEmpty()) return null
+            val prefix = if (os == OS.WINDOWS) {
+                "%USERPROFILE%".takeIf { value.startsWith(it) }
+            } else {
+                listOf("~", "\$HOME").firstOrNull { value.startsWith(it) }
+            }
+            val path = if (prefix == null) {
+                Path.of(value)
+            } else {
+                home.resolve(value.removePrefix(prefix).trimStart('/', '\\'))
+            }
+            return path.takeIf { it.isAbsolute }
+        }
     }
 
     private fun globalConfigArgs(): List<String> = listOf("--global-config", coderConfigPath.toString())

src/main/kotlin/com/coder/toolbox/diagnostics/CoderSshLogCollector.kt

@@ -0,0 +1,98 @@
+package com.coder.toolbox.diagnostics
+
+import com.coder.toolbox.session.SessionId
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.withContext
+import java.nio.channels.FileChannel
+import java.nio.file.Files
+import java.nio.file.LinkOption.NOFOLLOW_LINKS
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption.CREATE
+import java.nio.file.StandardOpenOption.READ
+import java.nio.file.StandardOpenOption.TRUNCATE_EXISTING
+import java.nio.file.StandardOpenOption.WRITE
+import java.nio.file.attribute.BasicFileAttributes
+import java.time.Instant
+
+// Include current CLI session filenames and older PID-based logs.
+private val SSH_LOG_FILE = Regex("coder-ssh.*\\.log(?:\\.\\d+)?|\\d+\\.log")
+private const val SSH_LOG_MAX_AGE_SECONDS = 3L * 24 * 60 * 60
+private const val SSH_LOG_MAX_BYTES = 50L * 1024 * 1024
+
+internal class CoderSshLogCollector(
+    private val logger: CoderLogger,
+    private val logDirectory: Path,
+) {
+    suspend fun collect(
+        logsRootFolder: Path,
+        sessionId: SessionId? = null,
+        activeSshPid: Long? = null,
+    ) = withContext(Dispatchers.IO) {
+        // Older CLIs may not have created any session logs.
+        if (Files.notExists(logDirectory)) return@withContext
+
+        val cutoff = Instant.now().minusSeconds(SSH_LOG_MAX_AGE_SECONDS)
+        val entries = Files.newDirectoryStream(logDirectory).use { it.toList() }
+        // CLI proxy log names end in the parent SSH PID; timestamps distinguish reused PIDs.
+        val activeLog = activeSshPid?.let { pid ->
+            entries.filter { entry ->
+                val name = entry.fileName.toString()
+                (name == "$pid.log" || (name.startsWith("coder-ssh") && name.endsWith("-$pid.log"))) &&
+                        Files.isRegularFile(entry, NOFOLLOW_LINKS)
+            }.maxByOrNull { it.fileName.toString() }
+        }
+        for (entry in entries) {
+            ensureActive()
+            // An active session can remain quiet longer than the collection window.
+            collectFile(entry, logsRootFolder.resolve("coder-ssh"), cutoff.takeUnless { entry == activeLog }, sessionId)
+        }
+    }
+
+    @Suppress("TooGenericExceptionCaught")
+    private fun CoroutineScope.collectFile(source: Path, destination: Path, cutoff: Instant?, sessionId: SessionId?) {
+        if (!SSH_LOG_FILE.matches(source.fileName.toString())) return
+        var output: Path? = null
+        try {
+            val attributes = Files.readAttributes(source, BasicFileAttributes::class.java, NOFOLLOW_LINKS)
+            val expired = cutoff != null && attributes.lastModifiedTime().toInstant().isBefore(cutoff)
+            if (!attributes.isRegularFile || expired) return
+
+            FileChannel.open(source, READ, NOFOLLOW_LINKS).use { input ->
+                val size = input.size()
+                if (size > SSH_LOG_MAX_BYTES) {
+                    logger.warn(
+                        sessionId,
+                        "Truncating SSH log $source ($size bytes) to its last $SSH_LOG_MAX_BYTES bytes",
+                    )
+                }
+                Files.createDirectories(destination)
+                val targetPath = destination.resolve(source.fileName)
+                output = targetPath
+                FileChannel.open(targetPath, CREATE, WRITE, TRUNCATE_EXISTING).use { target ->
+                    copyTail(input, target, size)
+                }
+            }
+        } catch (ex: CancellationException) {
+            output?.let { runCatching { Files.deleteIfExists(it) } }
+            throw ex
+        } catch (ex: Exception) {
+            output?.let { runCatching { Files.deleteIfExists(it) } }
+            logger.warn(sessionId, ex, "Could not collect SSH log $source")
+        }
+    }
+
+    private fun CoroutineScope.copyTail(input: FileChannel, target: FileChannel, size: Long) {
+        var remaining = minOf(size, SSH_LOG_MAX_BYTES)
+        var offset = size - remaining
+        while (remaining > 0) {
+            ensureActive()
+            val copied = input.transferTo(offset, remaining, target)
+            if (copied == 0L) break
+            offset += copied
+            remaining -= copied
+        }
+    }
+}

src/main/kotlin/com/coder/toolbox/diagnostics/CoderSupportBundleCollector.kt

@@ -1,39 +1,58 @@
 package com.coder.toolbox.diagnostics
 
+import com.coder.toolbox.session.SessionId
 import com.jetbrains.toolbox.api.remoteDev.deploy.DiagnosticInfoCollector
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.nio.file.Files
 import java.nio.file.Path
 
-/** Collects a Coder support bundle for a workspace and its optional agent. */
+/** Collects a workspace support bundle and recent local Coder SSH logs. */
 internal class CoderSupportBundleCollector(
     private val logger: CoderLogger,
+    sshLogDirectory: Path,
+    private val currentSessionId: () -> SessionId? = { null },
+    private val sshPid: Long? = null,
     private val collectBundle: suspend (Path) -> Unit,
 ) : DiagnosticInfoCollector {
+    private val sshLogs = CoderSshLogCollector(logger, sshLogDirectory)
+
     // Toolbox discards the additional-diagnostics directory if the collector throws.
     @Suppress("TooGenericExceptionCaught")
     override suspend fun collectAdditionalDiagnostics(logsRootFolder: Path) = withContext(Dispatchers.IO) {
-        val bundle = logsRootFolder.resolve("coder-support.zip")
+        val sessionId = currentSessionId()
         try {
             Files.createDirectories(logsRootFolder)
+            collectSupportBundle(logsRootFolder, sessionId)
+            sshLogs.collect(logsRootFolder, sessionId, sshPid)
+        } catch (ex: CancellationException) {
+            throw ex
+        } catch (ex: Exception) {
+            logger.warn(sessionId, ex, "Could not collect Coder workspace diagnostics")
+        }
+    }
+
+    // A failed bundle is replaced by an error report so the SSH logs are still collected.
+    @Suppress("TooGenericExceptionCaught")
+    private suspend fun collectSupportBundle(logsRootFolder: Path, sessionId: SessionId?) {
+        val bundle = logsRootFolder.resolve("coder-support.zip")
+        try {
             collectBundle(bundle)
         } catch (ex: CancellationException) {
             runCatching { Files.deleteIfExists(bundle) }
             throw ex
         } catch (ex: Exception) {
-            logger.warn(ex, "Could not collect the Coder support bundle")
+            runCatching { Files.deleteIfExists(bundle) }
+            logger.warn(sessionId, ex, "Could not collect the Coder support bundle")
             runCatching {
-                Files.deleteIfExists(bundle)
                 Files.writeString(
                     logsRootFolder.resolve("coder-support-error.txt"),
                     "Coder support bundle collection failed (${ex.javaClass.simpleName}). " +
                             "Check deployment connectivity, login, and CLI support for 'coder support bundle' " +
                             "(Coder 2.10 or newer). Other Toolbox logs are still available.\n",
                 )
-            }.onFailure { logger.warn(it, "Could not write the Coder diagnostic failure report") }
+            }.onFailure { logger.warn(sessionId, it, "Could not write the Coder diagnostic failure report") }
         }
-        Unit
     }
 }