diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json b/parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json new file mode 100644 index 00000000000..2c80403afb2 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json @@ -0,0 +1,3 @@ +{ + "scripts_version": "202607.13.1" +} diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 5c659f7c966..e7016f21dba 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -1321,11 +1321,48 @@ logGPUDriverPrebakeReadiness() { echo "AKS_GPU_PREBAKE event=managed_gpu driver_type=${NVIDIA_GPU_DRIVER_TYPE:-} marker_present=${marker_present} driver_kind_match=${driver_kind_match}" } +# cleanUpGridNodeCudaPrebake tears down a cuda-lts driver pre-baked into the shared Ubuntu VHD when +# THIS node installs a GRID driver. The shared VHD bakes only the cuda(-lts) driver + a DKMS marker; +# a GRID/converged (A10, NVv5) node then installs the grid driver on top, and the stale prebaked +# cuda module + its /usr/bin/lib64 userspace libs collide with the grid driver -> nvidia-smi fails +# with "Failed to initialize NVML: Driver/library version mismatch". This is a pure driver-KIND +# mismatch (grid vs cuda), so no version comparison is needed. A legacy marker with no driver_kind= +# line is treated as a cuda prebake (the only kind the VHD bakes today). No-op unless the node is +# GRID and a prebake marker exists. Reuses cleanUpPrebakedGPUDriver (from cse_install_ubuntu.sh) for +# the actual removal. NAP/agentpool cuda nodes are intentionally untouched here. +cleanUpGridNodeCudaPrebake() { + [ "$OS" = "$UBUNTU_OS_NAME" ] || return 0 + local marker="${GPU_DKMS_MARKER_FILE:-/opt/azure/aks-gpu/dkms-marker}" + [ -f "${marker}" ] || return 0 + + local node_kind m_kind + case "${NVIDIA_GPU_DRIVER_TYPE:-}" in + grid*) node_kind=grid ;; + *) return 0 ;; + esac + m_kind="$(sed -n 's/^driver_kind=//p' "${marker}" | head -n1)" + + # Keep only when the prebake is explicitly grid (matches this grid node). An empty marker kind is + # a legacy cuda prebake; a "cuda" marker is a cuda prebake -- both mismatch a grid node, tear down. + if [ "${m_kind}" = "grid" ]; then + return 0 + fi + echo "AKS_GPU_PREBAKE event=grid_cuda_prebake_teardown driver_type=${NVIDIA_GPU_DRIVER_TYPE:-} marker_kind=${m_kind:-none} node_kind=${node_kind} action=teardown" + cleanUpPrebakedGPUDriver +} + ensureGPUDrivers() { if [ "$(isARM64)" -eq 1 ]; then return fi + # Tear down a mismatched cuda-lts VHD prebake before a GRID node installs its own driver, or the + # stale module/libs collide with the grid driver (NVML version mismatch). Runs before the dispatch + # below so it covers both the configGPUDrivers and validateGPUDrivers paths. + if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + logs_to_events "AKS.CSE.ensureGPUDrivers.cleanUpGridNodeCudaPrebake" cleanUpGridNodeCudaPrebake || exit $ERR_GPU_DRIVERS_START_FAIL + fi + if [ "${CONFIG_GPU_DRIVER_IF_NEEDED}" = true ]; then logs_to_events "AKS.CSE.ensureGPUDrivers.configGPUDrivers" configGPUDrivers else diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 4175bdfc8c8..3417c2cc436 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -32,6 +32,16 @@ write_files: content: !!binary | {{GetVariableProperty "cloudInitData" "initAKSCustomCloud"}} + +# ---- hotfix-scripts: auto-generated ---- +- path: {{GetCSEConfigScriptFilepath}} + permissions: "0744" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "provisionConfigs"}} + +# ---- end hotfix-scripts ---- {{- else }} - path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index 5ada517f698..f0651baef53 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -74,6 +74,93 @@ Describe 'cse_config.sh' End End + Describe 'cleanUpGridNodeCudaPrebake' + # Stub the actual removal so tests assert the keep-vs-teardown DECISION without touching the + # real filesystem. OS defaults to Ubuntu (the only path this function acts on). + OS="$UBUNTU_OS_NAME" + # shellcheck disable=SC2329 # invoked dynamically by cleanUpGridNodeCudaPrebake. + cleanUpPrebakedGPUDriver() { echo "STUB_TEARDOWN_CALLED"; } + + It 'tears down a cuda prebake before installing a GRID driver (A10/GRID outage path)' + marker="$(mktemp)" + printf 'driver_kind=cuda\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should include "action=teardown" + The output should include "marker_kind=cuda" + The output should include "node_kind=grid" + The output should include "STUB_TEARDOWN_CALLED" + rm -f "$marker" + End + + It 'tears down for a grid-v20 node whose driver-type maps to grid' + marker="$(mktemp)" + printf 'driver_kind=cuda\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="grid-v20" + When call cleanUpGridNodeCudaPrebake + The output should include "action=teardown" + The output should include "STUB_TEARDOWN_CALLED" + rm -f "$marker" + End + + It 'treats a legacy marker without driver_kind as a cuda prebake and tears down on a GRID node' + marker="$(mktemp)" + printf 'kernel=5.15.0-1114-azure\n' > "$marker" # no driver_kind= line + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should include "action=teardown" + The output should include "marker_kind=none" + The output should include "STUB_TEARDOWN_CALLED" + rm -f "$marker" + End + + It 'is a no-op on a CUDA node (leaves the cuda prebake for the version-match/library-bump path)' + marker="$(mktemp)" + printf 'driver_kind=cuda\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="cuda-lts" + When call cleanUpGridNodeCudaPrebake + The output should not include "STUB_TEARDOWN_CALLED" + The status should be success + rm -f "$marker" + End + + It 'is a no-op when the prebake is already grid (matches the grid node)' + marker="$(mktemp)" + printf 'driver_kind=grid\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should not include "STUB_TEARDOWN_CALLED" + The status should be success + rm -f "$marker" + End + + It 'is a no-op when no prebake marker exists' + GPU_DKMS_MARKER_FILE="$(mktemp)"; rm -f "${GPU_DKMS_MARKER_FILE}" + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should not include "STUB_TEARDOWN_CALLED" + The status should be success + End + + It 'is a no-op on a non-Ubuntu OS even when a mismatched marker is present' + marker="$(mktemp)" + printf 'driver_kind=cuda\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + OS="MARINER" # override the Ubuntu default set at the Describe level + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should not include "STUB_TEARDOWN_CALLED" + The status should be success + OS="$UBUNTU_OS_NAME" # restore for any subsequent examples + rm -f "$marker" + End + End + Describe 'configureAzureJson' AZURE_JSON_PATH="azure.json" AKS_CUSTOM_CLOUD_JSON_PATH="customcloud.json"