feat: Performance - load host data in the background and stop reporting failed pre-checks

.github/workflows/verify-split-files.yml

@@ -37,6 +37,7 @@ on:
       - "tests/stale-silent-record.sh"
       - "tests/exit-code-hints.sh"
       - "tests/telemetry-trace.sh"
+      - "tests/host-probes.sh"
       - "tests/forge-release-json.sh"
       - "tests/install-cleanup.sh"
       - "tests/console-hangup.sh"
@@ -73,6 +74,7 @@ on:
       - "tests/stale-silent-record.sh"
       - "tests/exit-code-hints.sh"
       - "tests/telemetry-trace.sh"
+      - "tests/host-probes.sh"
       - "tests/forge-release-json.sh"
       - "tests/install-cleanup.sh"
       - "tests/console-hangup.sh"
@@ -146,6 +148,9 @@ jobs:
       - name: Telemetry trace tests
         run: bash tests/telemetry-trace.sh
 
+      - name: Host probe tests
+        run: bash tests/host-probes.sh
+
       - name: Forge release lookup tests
         run: bash tests/forge-release-json.sh
 

api/API.txt

@@ -14,6 +14,7 @@ _tm_post_progress
 _tm_post_run
 _tm_post_update
 _tm_quiet
+_tm_requirement_code
 _tm_send
 _tm_step_output
 build_error_string

api/sysinfo.func

@@ -235,7 +235,16 @@ telemetry_collect_sysinfo() {
   TM_PLATFORM="${TELEMETRY_PLATFORM:-}"
   TM_PVE_VERSION=""
   if command -v pveversion &>/dev/null; then
-    TM_PVE_VERSION=$(pveversion 2>/dev/null | awk -F'[/ ]' '{print $2}') || true
+    # Every payload is built in $( ), so _TM_SYSINFO_DONE never reaches the
+    # caller and this ran once per send; _pve_version caches in a file, and
+    # the VM engine, which lacks it, keeps what its pve_check read.
+    if declare -f _pve_version >/dev/null 2>&1; then
+      TM_PVE_VERSION="$(_pve_version)"
+    elif [[ "${PVEVERSION:-}" =~ ^[0-9] ]]; then
+      TM_PVE_VERSION="$PVEVERSION"
+    else
+      TM_PVE_VERSION=$(pveversion 2>/dev/null | awk -F'[/ ]' '{print $2}') || true
+    fi
     TM_PLATFORM="${TM_PLATFORM:-pve}"
   elif command -v incus &>/dev/null; then
     TM_PVE_VERSION="incus-$(incus version 2>/dev/null | head -1 | awk '{print $NF}')" || true

api/telemetry.func

@@ -259,16 +259,10 @@ _tm_post_update() {
     ;;
   esac
 
-  # Unset: no install was reported yet. A host run that failed in its checks
-  # (storage, ID, template) reports one now; an update run inside a container
-  # or a cancelled dialog does not. "false" means that send failed.
-  if [[ -z "${POST_TO_API_DONE:-}" ]]; then
-    if [[ "$pb_status" == "failed" && -n "${NSAPP:-}" ]] &&
-      ! { declare -f _is_container_context >/dev/null 2>&1 && _is_container_context; }; then
-      post_to_api
-    fi
-    [[ -z "${POST_TO_API_DONE:-}" ]] && return 0
-  fi
+  # Unset: the run ended before its build reported the install, in the checks
+  # or a dialog. Those report nothing; a host that is not ready is no failed
+  # install. "false" means the install was due and its send failed.
+  [[ -z "${POST_TO_API_DONE:-}" ]] && return 0
 
   if [[ "$pb_status" == "failed" || "$pb_status" == "unknown" ]]; then
     _tm_error_context "$exit_code"
@@ -302,6 +296,17 @@ _tm_post_update() {
   return 0
 }
 
+# Tools and add-ons have no build step that starts their telemetry, so their
+# checks are known by the code they exit with: wrong host or OS, nothing to work
+# on, a prerequisite missing. Like the checks before a container build, those
+# report nothing.
+_tm_requirement_code() {
+  case "$1" in
+  10 | 103 | 104 | 106 | 232 | 233 | 234 | 236 | 238) return 0 ;;
+  esac
+  return 1
+}
+
 # ------------------------------------------------------------------------------
 # _tm_post_run <type> <name> <status> <exit_code> - tool and add-on runs
 #
@@ -318,6 +323,7 @@ _tm_post_run() {
   uuid=$(cat /proc/sys/kernel/random/uuid 2>/dev/null || uuidgen 2>/dev/null || echo "${type}-$(date +%s)")
   [[ "$status" == "done" ]] && status="success"
   [[ "$exit_code" =~ ^[0-9]+$ ]] || exit_code=199
+  [[ "$status" == "failed" ]] && _tm_requirement_code "$exit_code" && return 0
 
   if [[ "$status" == "failed" ]]; then
     _tm_error_context "$exit_code"

core/build.func

@@ -268,7 +268,7 @@ _cs_prefetch_engine() {
     [[ -d "$base" && -w "$base" ]] || continue
     # An EXIT trap would be the obvious cleanup, but core.func and api.func
     # install their own later and the last one wins. Sweep instead.
-    find "$base" -maxdepth 1 \( -name 'cs-engine.*' -o -name 'cs-pveam.*'       -o -name 'cs-pveversion.*' \) -mmin +60 -exec rm -rf {} + 2>/dev/null || true
+    find "$base" -maxdepth 1 \( -name 'cs-engine.*' -o -name 'cs-pveam.*'       -o -name 'cs-pveversion.*' -o -name 'cs-probe.*' \) -mmin +60 -exec rm -rf {} + 2>/dev/null || true
     dir="$(mktemp -d "${base}/cs-engine.XXXXXX" 2>/dev/null)" && break
     dir=""
   done
@@ -305,6 +305,52 @@ _cs_prefetch_engine() {
   return 0
 }
 
+# The checks, the settings and the storage dialogs wait on pveversion, pvesm,
+# pveam and pvesh, each a Perl start of up to a second. Start them now, while the
+# engine downloads and the header loads; _cs_probe_read hands over what they printed.
+# _cs_probe_run <dir> <name> <command...> - run it, keep its output and exit code
+_cs_probe_run() {
+  local dir="$1" name="$2"
+  shift 2
+  "$@" >"${dir}/${name}.out" 2>/dev/null
+  echo "$?" >"${dir}/${name}.tmp" && mv -f "${dir}/${name}.tmp" "${dir}/${name}.rc"
+}
+
+# _cs_probe <dir> <name> <command...> - the same in the background. The empty
+# .out tells readers a result is coming before the subshell has started.
+_cs_probe() {
+  : >"${1}/${2}.out"
+  _cs_probe_run "$@" </dev/null &
+}
+
+_cs_start_host_probes() {
+  command -v pveversion >/dev/null 2>&1 || return 0
+  [[ "$(id -u)" == "0" ]] || return 0
+  local base dir="" name names=(pveversion pvesm vztmpl rootdir pveam nextid)
+  for base in /dev/shm "${TMPDIR:-/tmp}" /tmp; do
+    [[ -d "$base" && -w "$base" ]] || continue
+    dir="$(mktemp -d "${base}/cs-probe.XXXXXX" 2>/dev/null)" && break
+    dir=""
+  done
+  [[ -n "$dir" ]] || return 0
+  [[ -f /etc/pve/corosync.conf ]] && names+=(pvecm)
+  for name in "${names[@]}"; do : >"${dir}/${name}.out"; done
+  # One fork from this shell; the probes branch off inside it.
+  (
+    _cs_probe_run "$dir" pveversion pveversion &
+    _cs_probe_run "$dir" pvesm pvesm status &
+    _cs_probe_run "$dir" vztmpl pvesm status -content vztmpl &
+    _cs_probe_run "$dir" rootdir pvesm status -content rootdir &
+    _cs_probe_run "$dir" pveam pveam available -section system &
+    _cs_probe_run "$dir" nextid pvesh get /cluster/nextid &
+    [[ -f "${dir}/pvecm.out" ]] && _cs_probe_run "$dir" pvecm pvecm status &
+    wait
+  ) </dev/null &
+  CS_PROBE_DIR="$dir"
+  return 0
+}
+
+_cs_start_host_probes
 _cs_prefetch_engine || true
 
 _cs_source_func "lxc/platform.func"

core/core.func

@@ -549,24 +549,60 @@ _cs_runtime_cache_dir() {
   return 1
 }
 
+# _cs_probe_read <name> - output of a host probe that core/build.func started at
+# load, once it has finished; fails when there is none or it failed, and the
+# caller runs the command itself. Polls a file, so it works inside $( ) too.
+_cs_probe_read() {
+  local rc
+  rc="$(_cs_probe_rc "$1")" && [[ "$rc" == "0" ]] || return 1
+  cat "${CS_PROBE_DIR}/$1.out"
+}
+
+# _cs_probe_rc <name> - a finished probe's exit code, for checks that answer
+# with one; fails when there is no such probe
+_cs_probe_rc() {
+  local dir="${CS_PROBE_DIR:-}" n=0
+  [[ -n "$dir" && -f "${dir}/$1.out" ]] || return 1
+  while [[ ! -f "${dir}/$1.rc" ]]; do
+    ((n++ < 1200)) || return 1
+    sleep 0.05
+  done
+  cat "${dir}/$1.rc"
+}
+
+# _cs_probe_drop <name>... - forget probes whose answer a change made stale
+_cs_probe_drop() {
+  local name
+  for name in "$@"; do
+    rm -f "${CS_PROBE_DIR:-/nonexistent}/${name}.out" "${CS_PROBE_DIR:-/nonexistent}/${name}.rc" 2>/dev/null
+  done
+  return 0
+}
+
+_cs_pveversion_out() {
+  _cs_probe_read pveversion || pveversion 2>/dev/null
+}
+
 _pve_version() {
   [[ -n "${PVEVERSION:-}" ]] && { printf '%s' "$PVEVERSION"; return 0; }
   local dir cache
   dir="$(_cs_runtime_cache_dir)" || {
-    pveversion 2>/dev/null | awk -F'/' '{print $2}' | awk -F'-' '{print $1}'
-    return
+    _cs_pveversion_out | awk -F'/' '{print $2}' | awk -F'-' '{print $1}'
+    return 0
   }
   cache="${dir}/cs-pveversion.$$"
   [[ -s "$cache" ]] ||
-    pveversion 2>/dev/null | awk -F'/' '{print $2}' | awk -F'-' '{print $1}' >"$cache"
+    _cs_pveversion_out | awk -F'/' '{print $2}' | awk -F'-' '{print $1}' >"$cache"
   cat "$cache" 2>/dev/null
 }
 
 # PVEVERSION carries the product name on Incus ("Incus 7.3") because telemetry
 # reports it verbatim as pve_version. A fixed "PVE Version" prefix therefore
 # printed "PVE Version Incus 7.3" on Incus hosts.
 _cs_host_version_line() {
-  local v="${PVEVERSION:-unknown}"
+  local v="${PVEVERSION:-}"
+  [[ -z "$v" ]] && command -v pveversion >/dev/null 2>&1 && v="$(_pve_version)"
+  v="${v:-unknown}"
   if [[ "$v" == Incus* ]]; then
     v="${v#Incus}"
     v="${v# }"
@@ -643,11 +679,11 @@ _cs_scripts_ref_line() {
 _pveam_available() {
   local dir cache
   dir="$(_cs_runtime_cache_dir)" || {
-    pveam available -section system 2>/dev/null
-    return
+    _cs_probe_read pveam || pveam available -section system 2>/dev/null
+    return 0
   }
   cache="${dir}/cs-pveam.$$"
-  [[ -s "$cache" ]] || pveam available -section system 2>/dev/null >"$cache"
+  [[ -s "$cache" ]] || { _cs_probe_read pveam || pveam available -section system 2>/dev/null; } >"$cache"
   cat "$cache" 2>/dev/null
 }
 
@@ -656,6 +692,7 @@ _pveam_available() {
 _pveam_update() {
   local dir
   dir="$(_cs_runtime_cache_dir)" && rm -f "${dir}/cs-pveam.$$"
+  _cs_probe_drop pveam
   if command -v timeout &>/dev/null; then
     timeout 30 pveam update >/dev/null 2>&1 || return 1
   else
@@ -735,7 +772,7 @@ pmxcfs_check() {
     exit 228
   fi
 
-  if [[ -f /etc/pve/corosync.conf ]] && ! pvecm status 2>/dev/null | grep -qE '^Quorate:[[:space:]]+Yes'; then
+  if [[ -f /etc/pve/corosync.conf ]] && ! { _cs_probe_read pvecm || pvecm status 2>/dev/null; } | grep -qE '^Quorate:[[:space:]]+Yes'; then
     msg_error "This cluster node has no quorum, so /etc/pve is read-only and nothing can be created."
     msg_custom "🔧" "${YW}" "Bring enough cluster nodes online, then check: pvecm status"
     exit 210
@@ -899,16 +936,17 @@ log_msg() {
   local logfile
   logfile="$(get_active_logfile)"
 
-  [[ -z "$msg" ]] && return
-  [[ -z "$logfile" ]] && return
+  [[ -z "$msg" ]] && return 0
+  [[ -z "$logfile" ]] && return 0
 
   # Ensure log directory exists
   mkdir -p "$(dirname "$logfile")" 2>/dev/null || true
 
   # Strip ANSI codes and write with timestamp
   local clean_msg
   clean_msg=$(strip_ansi "$msg")
-  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $clean_msg" >>"$logfile"
+  # A log that cannot be written (full disk, removed directory) must not end the run.
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $clean_msg" >>"$logfile" 2>/dev/null || true
 }
 
 # ------------------------------------------------------------------------------
@@ -923,15 +961,15 @@ log_section() {
   local logfile
   logfile="$(get_active_logfile)"
 
-  [[ -z "$logfile" ]] && return
+  [[ -z "$logfile" ]] && return 0
   mkdir -p "$(dirname "$logfile")" 2>/dev/null || true
 
   {
     echo ""
     echo "================================================================================"
     echo "[$(date '+%Y-%m-%d %H:%M:%S')] $section"
     echo "================================================================================"
-  } >>"$logfile"
+  } >>"$logfile" 2>/dev/null || true
 }
 
 # ------------------------------------------------------------------------------

docs/api.md

@@ -15,9 +15,21 @@ terminal status: it writes `/root/.install-<SESSION_ID>.failed` and an
 `.errinfo` capture, which the host picks up after `lxc-attach` returns. See
 [`core/error_handler.func`](../core/error_handler.func).
 
-A host run that fails before its install record went out (storage, ID or
-template checks) sends that record first, then the failure. Cancelled dialogs
-and update runs inside a container send nothing.
+Telemetry starts with the install record (`post_to_api`, `post_to_api_vm`),
+which the build sends once the checks and dialogs are behind it. A run that
+ends before that sends nothing: a host that is not ready is not a failed
+install. Tools and add-ons have no such point, so a failure with a
+"requirement not met" code (10, 103, 104, 106, 232, 233, 234, 236, 238) sends
+nothing either. Update runs inside a container send nothing.
+
+The checks, the settings and the storage dialogs wait on `pveversion`,
+`pvesm status` (all, `vztmpl`, `rootdir`), `pveam available` and `pvesh`.
+`core/build.func` starts these in the background before it downloads the
+engine, `variables` adds pve-container's verdict on the release, and the
+readers take them through `_cs_probe_read` / `_cs_probe_rc` (`_pvesm_status`
+for storage). `_offer_storage_content` and `_pveam_update` drop what they
+change with `_cs_probe_drop`. Guest IDs are checked against the configs under
+`/etc/pve/nodes`, which pmxcfs keeps for every node.
 
 ## What a failure carries (payload version 3)