#680 [ci] CI の隔離の probe が UNVERIFIED・判定なしでも job が success になる

.github/workflows/ci.yml

@@ -74,21 +74,9 @@ jobs:
         run: |
           # A passing test's stdout is captured by cargo test, and $GITHUB_STEP_SUMMARY
           # is not readable through gh or the REST API, so print the verdict file and
-          # annotate every UNVERIFIED line. This step never fails the job.
-          verdicts="${COMMANDAGENT_PROBE_VERDICT_FILE}"
-          if [ ! -s "$verdicts" ]; then
-            echo "::warning::no isolation probe verdict was recorded in $verdicts: the probes did not run, or the file is empty"
-            exit 0
-          fi
-          echo "isolation probe verdicts ($verdicts):"
-          cat "$verdicts"
-          while IFS= read -r line; do
-            case "$line" in
-              *[Uu][Nn][Vv][Ee][Rr][Ii][Ff][Ii][Ee][Dd]*)
-                echo "::warning::$line"
-                ;;
-            esac
-          done < "$verdicts"
+          # annotate every UNVERIFIED line. This step fails the job if any probe was
+          # UNVERIFIED or if the verdict file is missing or empty.
+          bash scripts/check-probe-verdicts.sh
 
   gui-dashboard:
     name: GUI Dashboard

scripts/check-probe-verdicts.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+report_tiocsti_guidance() {
+  local val=""
+  local tiocsti_str=""
+  if command -v sysctl >/dev/null 2>&1; then
+    val="$(sysctl -n dev.tty.legacy_tiocsti 2>/dev/null || true)"
+  fi
+  if [ -n "$val" ]; then
+    tiocsti_str="$val"
+  else
+    tiocsti_str="could not be read (sysctl unavailable or dev.tty.legacy_tiocsti not present)"
+  fi
+  echo "dev.tty.legacy_tiocsti might not have been set. Check the logs of step 'Enable legacy TIOCSTI for the isolation probes'."
+  echo "Current dev.tty.legacy_tiocsti: $tiocsti_str"
+}
+
+verdicts="${1:-${COMMANDAGENT_PROBE_VERDICT_FILE:-}}"
+
+if [ -z "$verdicts" ]; then
+  echo "::error::isolation probe verdict file path was not provided (pass as first argument or set COMMANDAGENT_PROBE_VERDICT_FILE)"
+  report_tiocsti_guidance
+  exit 1
+fi
+
+if [ ! -e "$verdicts" ]; then
+  echo "::error::isolation probe verdict file does not exist: $verdicts"
+  report_tiocsti_guidance
+  exit 1
+fi
+
+if [ ! -s "$verdicts" ]; then
+  echo "::error::no isolation probe verdict was recorded in $verdicts: the probes did not run, or the file is empty"
+  report_tiocsti_guidance
+  exit 1
+fi
+
+echo "isolation probe verdicts ($verdicts):"
+cat "$verdicts"
+
+has_unverified=0
+while IFS= read -r line || [ -n "$line" ]; do
+  case "$line" in
+    *[Uu][Nn][Vv][Ee][Rr][Ii][Ff][Ii][Ee][Dd]*)
+      echo "::error::$line"
+      has_unverified=1
+      ;;
+  esac
+done < "$verdicts"
+
+if [ "$has_unverified" -ne 0 ]; then
+  report_tiocsti_guidance
+  exit 1
+fi

tests/probe_verdicts_script.rs

@@ -0,0 +1,134 @@
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output};
+
+fn script_path() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("scripts/check-probe-verdicts.sh")
+}
+
+fn run_probe_verifier(arg_path: &Path) -> Output {
+    Command::new("bash")
+        .arg(script_path())
+        .arg(arg_path)
+        .env_remove("COMMANDAGENT_PROBE_VERDICT_FILE")
+        .output()
+        .expect("failed to execute check-probe-verdicts.sh")
+}
+
+fn combined_output(output: &Output) -> String {
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    format!("{stdout}\n{stderr}")
+}
+
+#[test]
+fn unverified_verdict_fails_with_error_annotation() {
+    let temp = tempfile::tempdir().expect("create tempdir");
+    let file_path = temp.path().join("probe-verdicts.txt");
+    std::fs::write(
+        &file_path,
+        "issue667/tiocsti: UNVERIFIED (could not write legacy_tiocsti)\n\
+         issue502/sandbox: unverified: backend not enabled\n",
+    )
+    .expect("write verdicts");
+
+    let output = run_probe_verifier(&file_path);
+    assert!(
+        !output.status.success(),
+        "script must exit non-zero when UNVERIFIED lines are present, got: {:?}",
+        output.status
+    );
+    let out = combined_output(&output);
+    assert!(
+        out.contains("::error::"),
+        "expected '::error::' in output, got:\n{out}"
+    );
+    assert!(
+        out.contains("::error::issue667/tiocsti: UNVERIFIED (could not write legacy_tiocsti)"),
+        "expected line-by-line annotation for uppercase UNVERIFIED, got:\n{out}"
+    );
+    assert!(
+        out.contains("::error::issue502/sandbox: unverified: backend not enabled"),
+        "expected line-by-line annotation for lowercase unverified, got:\n{out}"
+    );
+    assert!(
+        out.contains("dev.tty.legacy_tiocsti"),
+        "expected failure message to mention dev.tty.legacy_tiocsti, got:\n{out}"
+    );
+}
+
+#[test]
+fn empty_verdict_file_fails_with_error_annotation() {
+    let temp = tempfile::tempdir().expect("create tempdir");
+    let file_path = temp.path().join("probe-verdicts.txt");
+    std::fs::write(&file_path, "").expect("write empty verdicts");
+
+    let output = run_probe_verifier(&file_path);
+    assert!(
+        !output.status.success(),
+        "script must exit non-zero for empty verdict file, got: {:?}",
+        output.status
+    );
+    let out = combined_output(&output);
+    assert!(
+        out.contains("::error::"),
+        "expected '::error::' in output for empty file, got:\n{out}"
+    );
+    assert!(
+        out.contains("dev.tty.legacy_tiocsti"),
+        "expected failure message to mention dev.tty.legacy_tiocsti, got:\n{out}"
+    );
+}
+
+#[test]
+fn missing_verdict_file_fails_with_error_annotation() {
+    let temp = tempfile::tempdir().expect("create tempdir");
+    let file_path = temp.path().join("nonexistent-probe-verdicts.txt");
+
+    let output = run_probe_verifier(&file_path);
+    assert!(
+        !output.status.success(),
+        "script must exit non-zero when verdict file is missing, got: {:?}",
+        output.status
+    );
+    let out = combined_output(&output);
+    assert!(
+        out.contains("::error::"),
+        "expected '::error::' in output for missing file, got:\n{out}"
+    );
+    assert!(
+        out.contains("dev.tty.legacy_tiocsti"),
+        "expected failure message to mention dev.tty.legacy_tiocsti, got:\n{out}"
+    );
+}
+
+#[test]
+fn blocked_only_verdicts_succeed_without_error() {
+    let temp = tempfile::tempdir().expect("create tempdir");
+    let file_path = temp.path().join("probe-verdicts.txt");
+    std::fs::write(
+        &file_path,
+        "issue667/tiocsti: blocked; legacy_tiocsti=1\n\
+         issue502/sandbox: blocked; confinement verified\n",
+    )
+    .expect("write verdicts");
+
+    let output = run_probe_verifier(&file_path);
+    assert!(
+        output.status.success(),
+        "script must exit 0 when only BLOCKED verdicts are recorded, got: {:?}",
+        output.status
+    );
+    let out = combined_output(&output);
+    assert!(
+        !out.contains("::error::"),
+        "expected no '::error::' in output, got:\n{out}"
+    );
+    assert!(
+        out.contains("isolation probe verdicts"),
+        "expected header to be printed, got:\n{out}"
+    );
+    assert!(
+        out.contains("issue667/tiocsti: blocked; legacy_tiocsti=1"),
+        "expected verdict contents to be printed, got:\n{out}"
+    );
+}