diff --git a/parts/linux/cloud-init/artifacts/cse_config_gpu.sh b/parts/linux/cloud-init/artifacts/cse_config_gpu.sh index 7c288b5854b..77e3b52b2e7 100755 --- a/parts/linux/cloud-init/artifacts/cse_config_gpu.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_gpu.sh @@ -40,13 +40,36 @@ installGPUDriverImage() { # three known codes: 1 (MIG mode on with no MIG instances, which never succeeds), 2 (panic before the # driver userspace is ready), 127 (nvidia-smi missing on pre-reorder aks-gpu images). Anything else # still fails the unit. GPU pods reach the device via containerd's nvidia-container-runtime, not CDI. +# +# Grace-Blackwell also needs a wider START timeout. nvidia-container-toolkit 1.19.1 left this +# Type=oneshot unit at systemd's oneshot default of TimeoutStartSec=infinity; 1.20.1 (aks-gpu #177) +# ships /usr/lib/systemd/system/nvidia-cdi-refresh.service.d/10-container-engines.conf which orders +# the container engines after the unit AND pins TimeoutStartSec=90s. On GB200/GB300 the unit's +# `nvidia-ctk cdi generate` is slow -- ~80s to enumerate the GPUs + driver libs/firmware over the +# arm64 Grace NVLink/nvml path, plus a cold ~30s `nvidia-smi -L` precheck (>90s total) -- so systemd +# SIGTERMs it. A timeout is result=timeout (SIGTERM), NOT one of the tolerated SuccessExitStatus +# codes, so the unit still fails and install.sh's `systemctl restart` returns non-zero (CSE 84). +# x86 generates in <1s (measured: 8x H200 cdi generate <1s), so it keeps the 90s default. +# +# The timeout override lives in a SEPARATE, higher-sorted drop-in. systemd merges drop-ins in lexical +# filename order (last wins) across all dirs, so to beat the toolkit's 10-container-engines.conf our +# file must sort AFTER it -- hence the 90- prefix. (A 10-aks-* name sorts before 10-container-engines +# and loses -- verified: it leaves TimeoutStartUSec at 90s.) Verified live on GB300 that the 90- file +# raises TimeoutStartUSec 90s -> 5min. NVIDIA_CDI_REFRESH_DROP_IN="/etc/systemd/system/nvidia-cdi-refresh.service.d/10-aks-tolerate-generate-failure.conf" +NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN="/etc/systemd/system/nvidia-cdi-refresh.service.d/90-aks-cdi-refresh-timeout.conf" +NVIDIA_CDI_REFRESH_TIMEOUT_START_SEC="${NVIDIA_CDI_REFRESH_TIMEOUT_START_SEC:-300}" configureNvidiaCDIRefresh() { mkdir -p "$(dirname "$NVIDIA_CDI_REFRESH_DROP_IN")" || return 1 printf '[Service]\nSuccessExitStatus=1 2 127\n' > "$NVIDIA_CDI_REFRESH_DROP_IN" || return 1 - # Drop-ins are read when systemd loads a unit, so writing this before the toolkit is installed - # means the unit picks it up on its very first start. + # arm64 Grace-Blackwell (GB200/GB300) only: widen the start timeout so the slow cdi generate can + # finish. x86 generates in <1s, so leave it at the systemd default (no file written there). + if [ "$(getCPUArch)" = "arm64" ]; then + printf '[Service]\nTimeoutStartSec=%s\n' "$NVIDIA_CDI_REFRESH_TIMEOUT_START_SEC" > "$NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN" || return 1 + fi + # Drop-ins are read when systemd loads a unit, so writing these before the toolkit is installed + # means the unit picks them up on its very first start. systemctl daemon-reload || true } diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh index 2f8b394df05..e88fd52b76c 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh @@ -714,6 +714,9 @@ Describe 'cse_config_gpu.sh' setup_cdi_dropin() { CDI_TEST_DIR=$(mktemp -d) NVIDIA_CDI_REFRESH_DROP_IN="${CDI_TEST_DIR}/nvidia-cdi-refresh.service.d/10-aks-tolerate-generate-failure.conf" + # The timeout override is a separate, higher-sorted drop-in so it beats the toolkit's own + # 10-container-engines.conf (TimeoutStartSec=90s) in systemd's lexical drop-in merge. + NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN="${CDI_TEST_DIR}/nvidia-cdi-refresh.service.d/90-aks-cdi-refresh-timeout.conf" } cleanup_cdi_dropin() { rm -rf "${CDI_TEST_DIR}" @@ -722,6 +725,10 @@ Describe 'cse_config_gpu.sh' AfterEach 'cleanup_cdi_dropin' Describe 'configureNvidiaCDIRefresh' + # The TimeoutStartSec widening is arm64 (Grace-Blackwell) only; default the mock to arm64 + # so the shared drop-in assertions below exercise the GB path, and override per-test. + getCPUArch() { echo "${MOCK_CPU_ARCH:-arm64}"; } + It 'treats the known nvidia-ctk failures as success and reloads systemd' systemctl() { echo "systemctl $*"; return 0; } @@ -735,6 +742,33 @@ Describe 'cse_config_gpu.sh' The contents of file "${NVIDIA_CDI_REFRESH_DROP_IN}" should include "SuccessExitStatus=1 2 127" End + It 'writes the timeout override to a 90- drop-in on arm64 (must out-sort toolkit 10-container-engines.conf)' + MOCK_CPU_ARCH="arm64" + systemctl() { return 0; } + + When call configureNvidiaCDIRefresh + + The status should be success + # The 90- prefix is load-bearing: systemd merges drop-ins in lexical order (last wins), + # and the toolkit's own 10-container-engines.conf pins TimeoutStartSec=90s. A 10-aks-* + # name sorts before it and loses, so the override must sort after it. + The path "${NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN}" should be file + The contents of file "${NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN}" should include "TimeoutStartSec=300" + # the tolerate-failure drop-in stays free of the timeout (separation keeps ordering clear) + The contents of file "${NVIDIA_CDI_REFRESH_DROP_IN}" should not include "TimeoutStartSec" + End + + It 'does not write the timeout drop-in on x86 (generates in <1s, default timeout is fine)' + MOCK_CPU_ARCH="amd64" + systemctl() { return 0; } + + When call configureNvidiaCDIRefresh + + The status should be success + The contents of file "${NVIDIA_CDI_REFRESH_DROP_IN}" should include "SuccessExitStatus=1 2 127" + The path "${NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN}" should not be exist + End + It 'keeps the units restartable so the .path trigger can still refresh the spec' systemctl() { return 0; }