From 7ecbb04e6920cb82e16f93741977489735ed16be Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 26 Jun 2026 14:36:31 -0700 Subject: [PATCH 01/78] feat(linux): onboard Ubuntu 26.04 minimal image builds for AMD64 + ARM64 --- .pipelines/.vsts-vhd-builder-release.yaml | 38 +++++++++++------------ parts/linux/cloud-init/artifacts/cis.sh | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index 0813c490cc1..901c9ed72d7 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -117,12 +117,12 @@ parameters: displayName: Build 2204 TL Gen2 Containerd type: boolean default: true - - name: build2204minimalgen1containerd - displayName: Build 2204 Minimal Gen1 Containerd + - name: build2604minimalgen2containerd + displayName: Build 2604 Minimal Gen2 Containerd type: boolean default: false - - name: build2204minimalgen2containerd - displayName: Build 2204 Minimal Gen2 Containerd + - name: build2604minimalarm64gen2containerd + displayName: Build 2604 Minimal ARM64 Gen2 Containerd type: boolean default: false - name: buildflatcargen2 @@ -765,19 +765,19 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2204-tl-gen2-containerd - - job: build2204minimalgen1containerd - condition: eq('${{ parameters.build2204minimalgen1containerd }}', true) + - job: build2604minimalgen2containerd + condition: eq('${{ parameters.build2604minimalgen2containerd }}', true) dependsOn: [ ] timeoutInMinutes: 360 steps: - bash: | echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' - echo '##vso[task.setvariable variable=OS_VERSION]22.04' + echo '##vso[task.setvariable variable=OS_VERSION]26.04' echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' - echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-minimal-jammy-aks-daily-preview' - echo '##vso[task.setvariable variable=IMG_SKU]minimal-aks-22_04-daily-lts' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-26_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]minimal' echo '##vso[task.setvariable variable=IMG_VERSION]latest' - echo '##vso[task.setvariable variable=HYPERV_GENERATION]V1' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' @@ -788,23 +788,23 @@ stages: parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} - artifactName: 2204-minimal-gen1-containerd - - job: build2204minimalgen2containerd - condition: eq('${{ parameters.build2204minimalgen2containerd }}', true) + artifactName: 2604-minimal-gen2-containerd + - job: build2604minimalarm64gen2containerd + condition: eq('${{ parameters.build2604minimalarm64gen2containerd }}', true) dependsOn: [ ] timeoutInMinutes: 360 steps: - bash: | echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' - echo '##vso[task.setvariable variable=OS_VERSION]22.04' + echo '##vso[task.setvariable variable=OS_VERSION]26.04' echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' - echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-minimal-jammy-aks-daily-preview' - echo '##vso[task.setvariable variable=IMG_SKU]minimal-aks-22_04-daily-lts-gen2' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-26_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]minimal-arm64' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' - echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=ARCHITECTURE]ARM64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' displayName: Setup Build Variables @@ -812,7 +812,7 @@ stages: parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} - artifactName: 2204-minimal-gen2-containerd + artifactName: 2604-minimal-arm64-gen2-containerd - job: buildflatcargen2 condition: eq('${{ parameters.buildflatcargen2 }}', true) dependsOn: [ ] diff --git a/parts/linux/cloud-init/artifacts/cis.sh b/parts/linux/cloud-init/artifacts/cis.sh index dcdd110c9f6..6eaa4235da5 100644 --- a/parts/linux/cloud-init/artifacts/cis.sh +++ b/parts/linux/cloud-init/artifacts/cis.sh @@ -9,7 +9,7 @@ assignRootPW() { SALT=$(openssl rand -base64 5) SECRET=$(openssl rand -base64 37) CMD="import crypt, getpass, pwd; print(crypt.crypt('$SECRET', '\$6\$$SALT\$'))" - if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then + if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ] || [ "${VERSION}" = "26.04" ]; then HASH=$(python3 -c "$CMD") else HASH=$(python -c "$CMD") From 14870438195a2db9f8536ae9f9c4c0f1c9a3bbf4 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 26 Jun 2026 14:42:51 -0700 Subject: [PATCH 02/78] chore: update SKU naming logic for mimimal --- .pipelines/templates/.builder-release-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 213fffd8f8e..17b6ca895d5 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -75,12 +75,12 @@ steps: - bash: | SKU_NAME=${OS_VERSION} && \ + if [[ "${IMG_SKU}" == *"minimal"* ]]; then SKU_NAME="${SKU_NAME}minimal"; fi && \ if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ - if [[ "${IMG_SKU}" == *"minimal"* ]]; then SKU_NAME="${SKU_NAME}minimal"; fi && \ if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \ SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ From bc081a34f2a20837d58da6049d091d379ab0f45e Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 26 Jun 2026 15:07:27 -0700 Subject: [PATCH 03/78] chore: add installMinimalBuildDeps --- .../artifacts/ubuntu/cse_install_ubuntu.sh | 40 +++++++++++++++++-- vhdbuilder/packer/pre-install-dependencies.sh | 5 +++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index dfe68bfa8e5..93c2d31048a 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -40,6 +40,40 @@ blobfuseFallbackPackages() { fi } +# Used to install dependencies within pre-install-dependencies.sh on Ubuntu minimal images (currently only 26.04) +installMinimalBuildDeps() { + wait_for_apt_locks + retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT + retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL + + holdWALinuxAgent hold + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + + pkg_list+=(rsyslog) + + # Batch install all packages in a single apt_get_install call instead of + # looping one-by-one. On failure, fall back to individual installs for + # diagnostic clarity. Exit immediately on return code 2 (CSE timeout). + apt_get_install 30 1 600 "${pkg_list[@]}" + local batch_rc=$? + if [ "$batch_rc" -eq 2 ]; then + exit "$batch_rc" + elif [ "$batch_rc" -ne 0 ]; then + echo "Batch install failed, falling back to individual package install" + for apt_package in "${pkg_list[@]}"; do + apt_get_install 30 1 600 "$apt_package" + local pkg_rc=$? + if [ "$pkg_rc" -eq 2 ]; then + exit "$pkg_rc" + elif [ "$pkg_rc" -ne 0 ]; then + tail -n 200 /var/log/apt/term.log || true + tail -n 200 /var/log/dpkg.log || true + exit $ERR_APT_INSTALL_TIMEOUT + fi + done + fi +} + installDeps() { wait_for_apt_locks retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT @@ -55,11 +89,11 @@ installDeps() { [ -n "${fallback_pkg}" ] && pkg_list+=("${fallback_pkg}") done < <(blobfuseFallbackPackages "${OSVERSION}") - if [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then pkg_list+=(irqbalance) fi - if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then pkg_list+=("aznfs=3.0.14") fi @@ -85,7 +119,7 @@ installDeps() { done fi - if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then # disable aznfswatchdog since aznfs install and enable aznfswatchdog and aznfswatchdogv4 services at the same time while we only need aznfswatchdogv4 systemctl disable aznfswatchdog systemctl stop aznfswatchdog diff --git a/vhdbuilder/packer/pre-install-dependencies.sh b/vhdbuilder/packer/pre-install-dependencies.sh index 89ab9f0b885..2bac6347917 100644 --- a/vhdbuilder/packer/pre-install-dependencies.sh +++ b/vhdbuilder/packer/pre-install-dependencies.sh @@ -36,6 +36,11 @@ capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" copyPackerFiles +# Install minimal build dependencies as needed (currently only needed on 26.04) +if [ "${OS_RELEASE}" = "26.04" ]; then + installMinimalBuildDeps +fi + # Update rsyslog configuration RSYSLOG_CONFIG_FILEPATH="/etc/rsyslog.d/60-CIS.conf" if isMarinerOrAzureLinux "$OS"; then From 836c76d64bacf9260d52e31a9ccaf094d8f949c5 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 26 Jun 2026 15:11:31 -0700 Subject: [PATCH 04/78] chore: cleanup --- vhdbuilder/packer/pre-install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/packer/pre-install-dependencies.sh b/vhdbuilder/packer/pre-install-dependencies.sh index 2bac6347917..8eb12ae1d65 100644 --- a/vhdbuilder/packer/pre-install-dependencies.sh +++ b/vhdbuilder/packer/pre-install-dependencies.sh @@ -36,7 +36,7 @@ capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" copyPackerFiles -# Install minimal build dependencies as needed (currently only needed on 26.04) +# Install minimal build dependencies as needed (currently only needed on Ubuntu 26.04) if [ "${OS_RELEASE}" = "26.04" ]; then installMinimalBuildDeps fi From 9252eea6d13d4483f5dfa485ff8e0847605609ce Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 26 Jun 2026 15:12:14 -0700 Subject: [PATCH 05/78] chore: eliminate build retries for debugging --- .pipelines/templates/.builder-release-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 17b6ca895d5..0f581e4a447 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -97,7 +97,7 @@ steps: make -f packer.mk run-packer condition: ne(variables['BUILDER'], 'imagecustomizer') displayName: Build VHD - retryCountOnTaskFailure: 3 + # retryCountOnTaskFailure: 3 env: OS_TYPE: Linux GIT_VERSION: $(Build.SourceVersion) From 29e805f711323ac7247bc6381ee34ad313d47f95 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 26 Jun 2026 15:24:33 -0700 Subject: [PATCH 06/78] chore: typo --- vhdbuilder/packer/pre-install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/packer/pre-install-dependencies.sh b/vhdbuilder/packer/pre-install-dependencies.sh index 8eb12ae1d65..6b494a8114c 100644 --- a/vhdbuilder/packer/pre-install-dependencies.sh +++ b/vhdbuilder/packer/pre-install-dependencies.sh @@ -37,7 +37,7 @@ capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" copyPackerFiles # Install minimal build dependencies as needed (currently only needed on Ubuntu 26.04) -if [ "${OS_RELEASE}" = "26.04" ]; then +if [ "${UBUNTU_RELEASE}" = "26.04" ]; then installMinimalBuildDeps fi From bd236c76259741694ad103170e265b22c2c26a2a Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 26 Jun 2026 15:46:32 -0700 Subject: [PATCH 07/78] chore: remove gawk usage --- parts/linux/cloud-init/artifacts/cse_helpers.sh | 8 ++++---- parts/linux/cloud-init/artifacts/cse_install.sh | 4 ++-- vhdbuilder/packer/install-dependencies.sh | 6 +++--- vhdbuilder/packer/post-install-dependencies.sh | 4 ++-- vhdbuilder/packer/pre-install-dependencies.sh | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index c438e0610d5..78c30a4f442 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -170,11 +170,11 @@ ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network inte # For unit tests, the OS and OS_VERSION will be set in the unit test script. # So whether it's if or else actually doesn't matter to our unit test. if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then - OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') - OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') - OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') + OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') + OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') + OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') else -# This is only for unit test purpose. For example, a Mac OS dev box doesn't have /etc/*-release, then the unit test will continue. + # This is only for unit test purpose. For example, a Mac OS dev box doesn't have /etc/*-release, then the unit test will continue. echo "/etc/*-release not found" fi diff --git a/parts/linux/cloud-init/artifacts/cse_install.sh b/parts/linux/cloud-init/artifacts/cse_install.sh index 0958e3bab86..77f87136032 100755 --- a/parts/linux/cloud-init/artifacts/cse_install.sh +++ b/parts/linux/cloud-init/artifacts/cse_install.sh @@ -15,8 +15,8 @@ K8S_PRIVATE_PACKAGES_CACHE_DIR="/opt/kubernetes/downloads/private-packages" K8S_REGISTRY_REPO="oss/binaries/kubernetes" UBUNTU_RELEASE=$(lsb_release -r -s 2>/dev/null || echo "") # For Mariner 2.0, this returns "MARINER" and for AzureLinux 3.0, this returns "AZURELINUX" -OS=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"'; fi) -OS_VARIANT=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"'; fi) +OS=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]'; fi) +OS_VARIANT=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]'; fi) SECURE_TLS_BOOTSTRAP_CLIENT_DOWNLOAD_DIR="/opt/aks-secure-tls-bootstrap-client/downloads" SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR="/opt/bin" CREDENTIAL_PROVIDER_DOWNLOAD_DIR="/opt/credentialprovider/downloads" diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 2dd451c48f2..e0e2285204f 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -14,13 +14,13 @@ ACL_OS_NAME="AZURECONTAINERLINUX" # Real world examples from the command outputs # For Azure Linux V3: ID=azurelinux VERSION_ID="3.0" # For Azure Linux V2: ID=mariner VERSION_ID="2.0" -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') IS_KATA="false" if grep -q "kata" <<< "$FEATURE_FLAGS"; then IS_KATA="true" fi -OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') +OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') THIS_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)" diff --git a/vhdbuilder/packer/post-install-dependencies.sh b/vhdbuilder/packer/post-install-dependencies.sh index 384ef0289c8..2cf43f7ef6f 100644 --- a/vhdbuilder/packer/post-install-dependencies.sh +++ b/vhdbuilder/packer/post-install-dependencies.sh @@ -1,6 +1,6 @@ #!/bin/bash -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') UBUNTU_OS_NAME="UBUNTU" FLATCAR_OS_NAME="FLATCAR" ACL_OS_NAME="AZURECONTAINERLINUX" diff --git a/vhdbuilder/packer/pre-install-dependencies.sh b/vhdbuilder/packer/pre-install-dependencies.sh index 6b494a8114c..9809b582187 100644 --- a/vhdbuilder/packer/pre-install-dependencies.sh +++ b/vhdbuilder/packer/pre-install-dependencies.sh @@ -1,7 +1,7 @@ #!/bin/bash -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') THIS_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)" #the following sed removes all comments of the format {{/* */}} From a4308bbad1610491f581c2e13955346577a78e18 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 26 Jun 2026 16:02:16 -0700 Subject: [PATCH 08/78] chore: install gpg --- parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 93c2d31048a..9fd835eed25 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -49,7 +49,7 @@ installMinimalBuildDeps() { holdWALinuxAgent hold apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - pkg_list+=(rsyslog) + pkg_list+=(rsyslog gpg) # Batch install all packages in a single apt_get_install call instead of # looping one-by-one. On failure, fall back to individual installs for From 5bb99f64843dfd32d9ef33f873d6f717c066be71 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Mon, 6 Jul 2026 13:55:55 -0700 Subject: [PATCH 09/78] chore: cleanup install-deps.sh and temporarily skip package/binary caching --- vhdbuilder/packer/install-dependencies.sh | 1660 +++++++++++---------- 1 file changed, 837 insertions(+), 823 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index d4192ab0bb5..a5da0684c1e 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -38,207 +38,13 @@ CPU_ARCH=$(getCPUArch) #amd64 or arm64 SYSTEMD_ARCH=$(getSystemdArch) # x86-64 or arm64 VHD_LOGS_FILEPATH=/opt/azure/vhd-install.complete COMPONENTS_FILEPATH=/opt/azure/components.json +LOCALDNS_BINARY_PATH="/opt/azure/containers/localdns/binary" PERFORMANCE_DATA_FILE=/opt/azure/vhd-build-performance-data.json GRID_COMPATIBILITY_DATA_FILE=/opt/azure/vhd-grid-compatibility-data.json -resolve_packages_source_url - -echo "" -echo "Components downloaded in this VHD build (some of the below components might get deleted during cluster provisioning if they are not needed):" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" - -echo "Logging the kernel after purge and reinstall + reboot: $(uname -r)" -# fix grub issue with cvm by reinstalling before other deps -# other VHDs use grub-pc, not grub-efi -if [ "$OS" = "$UBUNTU_OS_NAME" ] && echo "$FEATURE_FLAGS" | grep -q "cvm"; then - apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - wait_for_apt_locks - apt_get_install 10 2 120 grub-efi || exit 1 -fi -capture_benchmark "${SCRIPT_NAME}_reinstall_grub_for_cvm" - -if [ "$OS" = "$UBUNTU_OS_NAME" ]; then - # disable and mask all UU timers/services - # save some background io/latency - systemctl mask apt-daily.service apt-daily-upgrade.service || exit 1 - systemctl disable apt-daily.service apt-daily-upgrade.service || exit 1 - systemctl disable apt-daily.timer apt-daily-upgrade.timer || exit 1 - - tee /etc/apt/apt.conf.d/99periodic > /dev/null <> "${CHRONYD_DIR}"/10-chrony-restarts.conf < /dev/null <<'EOF' -Compress=yes -Storage=persistent -SystemMaxUse=1G -RuntimeMaxUse=1G -ForwardToSyslog=yes -EOF -capture_benchmark "${SCRIPT_NAME}_install_deps_and_set_configs" - -if [ "$(isARM64)" -eq 1 ]; then - # shellcheck disable=SC3010 - if [[ ${HYPERV_GENERATION,,} == "v1" ]]; then - echo "No arm64 support on V1 VM, exiting..." - exit 1 - fi -fi - -# Always override network config and disable NTP + Timesyncd and install Chrony -# Mariner does this differently, so only do it for Ubuntu -if ! isMarinerOrAzureLinux "$OS"; then - overrideNetworkConfig || exit 1 - disableNtpAndTimesyncdInstallChrony || exit 1 -fi - -# ACL inherits Azure Linux behaviors but isMarinerOrAzureLinux returns false, -# so these must be called separately (mirrored in the Mariner/AzureLinux block below). -# Other Mariner functions are safe to skip for ACL: -# setMarinerNetworkdConfig — ACL doesn't ship systemd-bootstrap's 99-dhcp-en.network -# fixCBLMarinerPermissions — product_uuid already 444; no rsyslog on ACL -# addMarinerNvidiaRepo / updateDnfWithNvidiaPkg / disableDNFAutomatic / enableCheckRestart — ACL has no dnf/rpm -# activateNfConntrack — nf_conntrack auto-loads via iptables dependency chain -# disableTimesyncd — ACL handles chrony separately above via disableNtpAndTimesyncdInstallChrony -if isACL "$OS" "$OS_VARIANT"; then - # ACL's iptables.service loads host firewall rules that conflict with Cilium eBPF routing. - disableSystemdIptables || exit 1 - # Repoint /etc/resolv.conf from the stub (127.0.0.53) to the real upstream file - # so DNS queries go directly through localdns. - disableSystemdResolvedCache -fi -capture_benchmark "${SCRIPT_NAME}_validate_container_runtime_and_override_ubuntu_net_config" - -# Configure SSH service during VHD build for Ubuntu 22.10+ -configureSSHService "$OS" "$OS_VERSION" || echo "##vso[task.logissue type=warning]SSH Service configuration failed, but continuing VHD build" - -CONTAINERD_SERVICE_DIR="/etc/systemd/system/containerd.service.d" -mkdir -p "${CONTAINERD_SERVICE_DIR}" -# Explicitly set LimitNOFILE=1048576 (the value that 'infinity' resolves to on Ubuntu 22.04) for both Ubuntu and Mariner/AzureLinux. -# On Ubuntu 24.04 (Containerd 2.0), LimitNOFILE is removed upstream and systemd falls back to an implicit soft:hard limit -# (for example 1024:524288), so containerd inherits a very low soft file descriptor limit (1024) unless we override it here. -# On Mariner/AzureLinux this is redundant with the base containerd.service unit but harmless. -# Not removing LimitNOFILE from parts/linux/cloud-init/artifacts/containerd.service, -# to avoid compatibility issues between new VHDs and old CSE scripts. -tee "${CONTAINERD_SERVICE_DIR}/exec_start.conf" > /dev/null < /dev/null < /etc/udev/rules.d/99-nfs.rules < /etc/udev/rules.d/80-azure-disk.rules <, then skip and move on to next package - if [ "${#PACKAGE_VERSIONS[@]}" -eq 0 ] || [ "${PACKAGE_VERSIONS[0]}" = "" ]; then - echo "INFO: ${name} package versions array is either empty or the first element is . Skipping ${name} installation." - continue +installAndConfigureArtifactStreaming() { + local downloadURL="$1" + local version="$2" + # The arm64 packages have "-arm64" inserted before the file extension, + # e.g. acr-mirror-2204-arm64.deb instead of acr-mirror-2204.deb + if [ "$(isARM64)" -eq 1 ]; then + downloadURL="${downloadURL%.*}-arm64.${downloadURL##*.}" fi - downloadDir=$(echo "${p}" | jq .downloadLocation -r) - #download the package - case $name in - "kubernetes-cri-tools") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadAndInstallCriTools "${downloadDir}" "${evaluatedURL}" "${version}" - done - ;; - "azure-cni") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadAzureCNI "${downloadDir}" "${evaluatedURL}" - unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" - echo " - Azure CNI version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "cni-plugins") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadCNIPlugins "${downloadDir}" "${evaluatedURL}" - unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" - echo " - CNI plugin version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "containernetworking-plugins") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installCNI "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - containernetworking-plugins version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "runc") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - ensureRunc "${version}" "${evaluatedURL}" "${downloadDir}" - echo " - runc version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "containerd") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - if [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then - installContainerd "${downloadDir}" "${evaluatedURL}" "${version}" - elif isMarinerOrAzureLinux "$OS"; then - installStandaloneContainerd "${version}" - fi - echo " - containerd version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "oras") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installOras "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - oras version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "aks-secure-tls-bootstrap-client") - for version in ${PACKAGE_VERSIONS[@]}; do - # removed at provisioning time if secure TLS bootstrapping is disabled - if isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - installPackageFromCache "${name}" "${version}" "/opt/bin/${name}" || exit $? - elif isMarinerOrAzureLinux; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - installRPMPackageFromFile "${name}" "${version}" "/opt/bin/${name}" || exit $? - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? - installSecureTLSBootstrapClientSysext "${version}" || exit $? - fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "azure-acr-credential-provider") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadCredentialProvider "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - azure-acr-credential-provider version ${version}" >> ${VHD_LOGS_FILEPATH} - # ORAS will be used to install other packages for network isolated clusters, it must go first. - done - ;; - "inspektor-gadget") - if isMariner "$OS" || isFlatcar "$OS" || isACL "$OS" "$OS_VARIANT" || isAzureLinuxOSGuard "$OS" "$OS_VARIANT" || [ "${IS_KATA}" = "true" ]; then - echo "Skipping inspektor-gadget installation for ${OS} ${OS_VARIANT:-default} (IS_KATA=${IS_KATA})" - else - ig_version="${PACKAGE_VERSIONS[0]}" - if isUbuntu "$OS"; then - # Ubuntu: download ig deb via apt; ig_install_deb_stack expects it at downloadDir - downloadPkgFromVersion "ig" "${ig_version}" "${downloadDir}" - installIG "${ig_version}" "${downloadDir}" - elif isAzureLinux "$OS"; then - # Azure Linux 3.0: ig_install_rpm_stack handles its own RPM downloads - installIG "${ig_version}" "${downloadDir}" - fi - fi + local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" + retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} + case "$downloadURL" in + *.deb) + apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD ;; - "kubernetes-binaries") - # kubelet and kubectl - # need to cover previously supported version for VMAS scale up scenario - # So keeping as many versions as we can - those unsupported version can be removed when we don't have enough space - # NOTE that we only keep the latest one per k8s patch version as kubelet/kubectl is decided by VHD version - # Please do not use the .1 suffix, because that's only for the base image patches - # regular version >= v1.17.0 or hotfixes >= 20211009 has arm64 binaries. - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - extractKubeBinaries "${version}" "${evaluatedURL}" false "${downloadDir}" - echo " - kubernetes-binaries version ${version}" >> ${VHD_LOGS_FILEPATH} - done + *.rpm) + dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD ;; - azure-acr-credential-provider-pmc) - name=${name%-pmc} - for version in ${PACKAGE_VERSIONS[@]}; do - if isMarinerOrAzureLinux || isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? - fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done + *) + echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 + exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} ;; - kubelet|kubectl) - for version in ${PACKAGE_VERSIONS[@]}; do - if isMarinerOrAzureLinux || isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - cacheVersionedKubernetesPackageBinary "${name}" "${version}" "${downloadDir}" || exit $ERR_K8S_INSTALL_ERR - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + esac + rm "$MIRROR_DOWNLOAD_PATH" + + /opt/acr/tools/overlaybd/install.sh + /opt/acr/tools/overlaybd/config-user-agent.sh azure + /opt/acr/tools/overlaybd/enable-http-auth.sh + /opt/acr/tools/overlaybd/config.sh download.enable false + /opt/acr/tools/overlaybd/config.sh cacheConfig.cacheSizeGB 32 + /opt/acr/tools/overlaybd/config.sh exporterConfig.enable true + /opt/acr/tools/overlaybd/config.sh exporterConfig.port 9863 + systemctl link /opt/overlaybd/overlaybd-tcmu.service /opt/overlaybd/snapshotter/overlaybd-snapshotter.service + # Remove the bundled overlaybd installer packages (~55-58 MB); install.sh already installed them and they're unused at runtime. + rm -f /opt/acr/tools/overlaybd/bin/*.deb /opt/acr/tools/overlaybd/bin/*.rpm + echo " - acr-mirror version ${version}" >> ${VHD_LOGS_FILEPATH} +} + +retagAKSNodeCAWatcher() { + # This function retags the aks-node-ca-watcher image to a static tag + # The static tag is used to bootstrap custom CA trust when MCR egress may be intercepted by an untrusted TLS MITM firewall. + # The image is never pulled, it is only retagged. + + watcher=$(jq '.ContainerImages[] | select(.downloadURL | contains("aks-node-ca-watcher"))' $COMPONENTS_FILEPATH) + watcherBaseImg=$(echo $watcher | jq -r .downloadURL) + watcherVersion=$(echo $watcher | jq -r .multiArchVersionsV2[0].latestVersion) + watcherFullImg=${watcherBaseImg//\*/$watcherVersion} + + # this image will never get pulled, the tag must be the same across different SHAs. + # it will only ever be upgraded via node image changes. + # we do this because the image is used to bootstrap custom CA trust when MCR egress + # may be intercepted by an untrusted TLS MITM firewall. + watcherStaticImg=${watcherBaseImg//\*/static} + + # can't use $cliTool variable because crictl doesn't support retagging. + retagContainerImage "ctr" ${watcherFullImg} ${watcherStaticImg} +} +retagAKSNodeCAWatcher +capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" + +pinPodSandboxImages() { + # This function pins the pod sandbox image(s) to avoid Kubelet's Garbage Collector (GC) from removing them. + # This is achieved by setting the "io.cri-containerd.pinned" label on the image with a value of "pinned". + # These images are critical for pod startup and aren't supported with private ACR since containerd won't be using azure-acr-credential to fetch them. + + # Get all pause images as individual JSON objects + local pause_images + pause_images=$(jq -c '.ContainerImages[] | select(.downloadURL | contains("pause"))' $COMPONENTS_FILEPATH) + + if [ -z "$pause_images" ]; then + echo "Warning: No pause images found in components.json" + return 0 + fi + + # Process each pause image separately + while IFS= read -r podSandbox; do + if [ -z "$podSandbox" ]; then + continue + fi + + local podSandboxBaseImg + local podSandboxVersion + local podSandboxFullImg + + podSandboxBaseImg=$(echo "$podSandbox" | jq -r '.downloadURL') + podSandboxVersion=$(echo "$podSandbox" | jq -r '.multiArchVersionsV2[0].latestVersion') + + # Skip if we couldn't extract the required information + if [ "$podSandboxBaseImg" = "null" ] || [ "$podSandboxVersion" = "null" ]; then + echo "Warning: Could not extract downloadURL or latestVersion from pause image: $podSandbox" + continue + fi + + podSandboxFullImg=${podSandboxBaseImg//\*/$podSandboxVersion} + + echo "Pinning pause image: $podSandboxFullImg" + labelContainerImage "${podSandboxFullImg}" "io.cri-containerd.pinned" "pinned" + + done <<< "$pause_images" +} + +# download kubernetes package from the given URL using azcopy +# if it is a kube-proxy package, extract image from the downloaded package +cacheKubePackageFromPrivateUrl() { + local kube_private_binary_url="$1" + + echo "process private package url: $kube_private_binary_url" + + mkdir -p ${K8S_PRIVATE_PACKAGES_CACHE_DIR} # /opt/kubernetes/downloads/private-packages + + # save kube pkg with version number from the url path, this convention is used to find the cached package at run-time + local k8s_tgz_name + k8s_tgz_name=$(echo "$kube_private_binary_url" | grep -o -P '(?<=\/kubernetes\/).*(?=\/binaries\/)').tar.gz + + # use azcopy instead of curl to download packages + getAzCopyCurrentPath + + export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" + export AZCOPY_CONCURRENCY_VALUE="AUTO" + export AZCOPY_LOG_LOCATION="$(pwd)/azcopy-log-files/" + export AZCOPY_JOB_PLAN_LOCATION="$(pwd)/azcopy-job-plan-files/" + mkdir -p "${AZCOPY_LOG_LOCATION}" + mkdir -p "${AZCOPY_JOB_PLAN_LOCATION}" + + cached_pkg="${K8S_PRIVATE_PACKAGES_CACHE_DIR}/${k8s_tgz_name}" + echo "download private package ${kube_private_binary_url} and store as ${cached_pkg}" + + if ! ./azcopy copy "${kube_private_binary_url}" "${cached_pkg}"; then + azExitCode=$? + # loop through azcopy log files + shopt -s nullglob + for f in "${AZCOPY_LOG_LOCATION}"/*.log; do + echo "Azcopy log file: $f" + # upload the log file as an attachment to vso + echo "##vso[build.uploadlog]$f" + # check if the log file contains any errors + if grep -q '"level":"Error"' "$f"; then + echo "log file $f contains errors" + echo "##vso[task.logissue type=error]Azcopy log file $f contains errors" + # print the log file + cat "$f" + fi + done + shopt -u nullglob + exit $ERR_PRIVATE_K8S_PKG_ERR + fi +} + +# Configure LSM modules to include BPF +configureLsmWithBpf() { + echo "Configuring LSM modules to include BPF..." + + # Read current LSM modules + if [ ! -f /sys/kernel/security/lsm ]; then + echo "Warning: /sys/kernel/security/lsm not found, skipping LSM configuration" + return 0 + fi + + local current_lsm + current_lsm=$(cat /sys/kernel/security/lsm) + echo "Current LSM modules: $current_lsm" + + # Prepend bpf to the LSM list if not already present + if ! echo "$current_lsm" | grep -q bpf; then + if [ "$IS_KATA" = "true" ] || echo "$FEATURE_FLAGS" | grep -q "cvm"; then + echo "Warning: this is a Kata/CVM SKU - will not add BPF to LSM configuration" + return 0 + fi + + if isAzureLinuxOSGuard "$OS" "$OS_VARIANT"; then + echo "Warning: Azure Linux OS Guard built with signed UKI, not enabling BPF LSM" + return 0 + fi + + local new_lsm="bpf,$current_lsm" + echo "New LSM configuration: $new_lsm" + + if [ "$OS" = "$UBUNTU_OS_NAME" ] && [ "$OS_VERSION" = "24.04" ]; then + local grub_cfg="/etc/default/grub.d/50-cloudimg-settings.cfg" + if [ -f "$grub_cfg" ]; then + if grep -q "lsm=" "$grub_cfg"; then + sed -i "s/lsm=[^[:space:]]*/lsm=$new_lsm/g" "$grub_cfg" + else + sed -i "s/GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"lsm=$new_lsm /" "$grub_cfg" fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "${K8S_DEVICE_PLUGIN_PKG}") - for version in ${PACKAGE_VERSIONS[@]}; do - if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isMarinerOrAzureLinux "$OS"; then - downloadPkgFromVersion "${K8S_DEVICE_PLUGIN_PKG}" "${version}" "${downloadDir}" + echo "Updating GRUB configuration for Ubuntu 24.04..." + update-grub2 /boot/grub/grub.cfg || echo "Warning: Failed to update GRUB configuration" + else + echo "Warning: $grub_cfg not found, skipping LSM configuration" + fi + elif isMarinerOrAzureLinux "$OS" && [ "$OS_VERSION" = "3.0" ]; then + if [ -f /etc/default/grub ]; then + if grep -q "lsm=" /etc/default/grub; then + sed -i "s/lsm=[^[:space:]]*/lsm=$new_lsm/g" /etc/default/grub + else + sed -i "s/GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"lsm=$new_lsm /" /etc/default/grub fi - echo " - ${K8S_DEVICE_PLUGIN_PKG} version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "dra-driver-nvidia-gpu") - for version in ${PACKAGE_VERSIONS[@]}; do - if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isAzureLinux "$OS"; then - downloadPkgFromVersion "dra-driver-nvidia-gpu" "${version}" "${downloadDir}" + echo "Updating GRUB configuration for Azure Linux 3.0..." + grub2-mkconfig -o /boot/grub2/grub.cfg || echo "Warning: Failed to update GRUB configuration" + else + echo "Warning: /etc/default/grub not found, skipping LSM configuration" + fi + else + echo "LSM BPF configuration is only enabled for Ubuntu 24.04 and Azure Linux 3.0, skipping" + fi + + echo "LSM configuration update completed" + else + echo "BPF LSM already configured, skipping" + fi +} + +cachePackageAndBinaryComponents() { + # Download/cache all declared packages and binaries within components.json that apply to the respective OS SKU + packages=$(jq ".Packages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) + # Iterate over each element in the packages array + while IFS= read -r p; do + #getting metadata for each package + name=$(echo "${p}" | jq .name -r) + os=${OS} + # TODO(mheberling): Remove this once kata uses standard containerd. This OS is referenced + # in file `parts/common/component.json` with the same ${MARINER_KATA_OS_NAME}. + if isMariner "${OS}" && [ "${IS_KATA}" = "true" ]; then + # This is temporary for kata-cc because it uses a modified version of containerd and + # name is referenced in parts/common.json marinerkata. + os=${MARINER_KATA_OS_NAME} + fi + if isAzureLinux "${OS}" && [ "${IS_KATA}" = "true" ]; then + # This is temporary for kata-cc because it uses a modified version of containerd and + # name is referenced in parts/common.json azurelinuxkata. + os=${AZURELINUX_KATA_OS_NAME} + fi + updatePackageVersions "${p}" "${os}" "${OS_VERSION}" "${OS_VARIANT}" + updatePackageDownloadURL "${p}" "${os}" "${OS_VERSION}" "${OS_VARIANT}" + echo "In components.json, processing components.packages \"${name}\" \"${PACKAGE_VERSIONS[@]}\" \"${PACKAGE_DOWNLOAD_URL}\"" + + # if ${PACKAGE_VERSIONS[@]} count is 0 or if the first element of the array is , then skip and move on to next package + if [ "${#PACKAGE_VERSIONS[@]}" -eq 0 ] || [ "${PACKAGE_VERSIONS[0]}" = "" ]; then + echo "INFO: ${name} package versions array is either empty or the first element is . Skipping ${name} installation." + continue + fi + downloadDir=$(echo "${p}" | jq .downloadLocation -r) + #download the package + case $name in + "kubernetes-cri-tools") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadAndInstallCriTools "${downloadDir}" "${evaluatedURL}" "${version}" + done + ;; + "azure-cni") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadAzureCNI "${downloadDir}" "${evaluatedURL}" + unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" + echo " - Azure CNI version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "cni-plugins") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadCNIPlugins "${downloadDir}" "${evaluatedURL}" + unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" + echo " - CNI plugin version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "containernetworking-plugins") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installCNI "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - containernetworking-plugins version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "runc") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + ensureRunc "${version}" "${evaluatedURL}" "${downloadDir}" + echo " - runc version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "containerd") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + if [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then + installContainerd "${downloadDir}" "${evaluatedURL}" "${version}" + elif isMarinerOrAzureLinux "$OS"; then + installStandaloneContainerd "${version}" + fi + echo " - containerd version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "oras") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installOras "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - oras version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "aks-secure-tls-bootstrap-client") + for version in ${PACKAGE_VERSIONS[@]}; do + # removed at provisioning time if secure TLS bootstrapping is disabled + if isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + installPackageFromCache "${name}" "${version}" "/opt/bin/${name}" || exit $? + elif isMarinerOrAzureLinux; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + installRPMPackageFromFile "${name}" "${version}" "/opt/bin/${name}" || exit $? + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + installSecureTLSBootstrapClientSysext "${version}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "azure-acr-credential-provider") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadCredentialProvider "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - azure-acr-credential-provider version ${version}" >> ${VHD_LOGS_FILEPATH} + # ORAS will be used to install other packages for network isolated clusters, it must go first. + done + ;; + "inspektor-gadget") + if isMariner "$OS" || isFlatcar "$OS" || isACL "$OS" "$OS_VARIANT" || isAzureLinuxOSGuard "$OS" "$OS_VARIANT" || [ "${IS_KATA}" = "true" ]; then + echo "Skipping inspektor-gadget installation for ${OS} ${OS_VARIANT:-default} (IS_KATA=${IS_KATA})" + else + ig_version="${PACKAGE_VERSIONS[0]}" + if isUbuntu "$OS"; then + # Ubuntu: download ig deb via apt; ig_install_deb_stack expects it at downloadDir + downloadPkgFromVersion "ig" "${ig_version}" "${downloadDir}" + installIG "${ig_version}" "${downloadDir}" + elif isAzureLinux "$OS"; then + # Azure Linux 3.0: ig_install_rpm_stack handles its own RPM downloads + installIG "${ig_version}" "${downloadDir}" + fi fi - echo " - dra-driver-nvidia-gpu version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "datacenter-gpu-manager-4-core") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "datacenter-gpu-manager-4-core" "${version}" "${downloadDir}" - echo " - datacenter-gpu-manager-4-core version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "datacenter-gpu-manager-4-proprietary") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "datacenter-gpu-manager-4-proprietary" "${version}" "${downloadDir}" - echo " - datacenter-gpu-manager-4-proprietary version ${version}" >> ${VHD_LOGS_FILEPATH} + ;; + "kubernetes-binaries") + # kubelet and kubectl + # need to cover previously supported version for VMAS scale up scenario + # So keeping as many versions as we can - those unsupported version can be removed when we don't have enough space + # NOTE that we only keep the latest one per k8s patch version as kubelet/kubectl is decided by VHD version + # Please do not use the .1 suffix, because that's only for the base image patches + # regular version >= v1.17.0 or hotfixes >= 20211009 has arm64 binaries. + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + extractKubeBinaries "${version}" "${evaluatedURL}" false "${downloadDir}" + echo " - kubernetes-binaries version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + azure-acr-credential-provider-pmc) + name=${name%-pmc} + for version in ${PACKAGE_VERSIONS[@]}; do + if isMarinerOrAzureLinux || isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + kubelet|kubectl) + for version in ${PACKAGE_VERSIONS[@]}; do + if isMarinerOrAzureLinux || isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + cacheVersionedKubernetesPackageBinary "${name}" "${version}" "${downloadDir}" || exit $ERR_K8S_INSTALL_ERR + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "${K8S_DEVICE_PLUGIN_PKG}") + for version in ${PACKAGE_VERSIONS[@]}; do + if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isMarinerOrAzureLinux "$OS"; then + downloadPkgFromVersion "${K8S_DEVICE_PLUGIN_PKG}" "${version}" "${downloadDir}" + fi + echo " - ${K8S_DEVICE_PLUGIN_PKG} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "dra-driver-nvidia-gpu") + for version in ${PACKAGE_VERSIONS[@]}; do + if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isAzureLinux "$OS"; then + downloadPkgFromVersion "dra-driver-nvidia-gpu" "${version}" "${downloadDir}" + fi + echo " - dra-driver-nvidia-gpu version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "datacenter-gpu-manager-4-core") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "datacenter-gpu-manager-4-core" "${version}" "${downloadDir}" + echo " - datacenter-gpu-manager-4-core version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "datacenter-gpu-manager-4-proprietary") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "datacenter-gpu-manager-4-proprietary" "${version}" "${downloadDir}" + echo " - datacenter-gpu-manager-4-proprietary version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "dcgm-exporter") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" + echo " - dcgm-exporter version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "node-exporter") + # Skipping is handled by empty versionsV2 arrays in components.json + # for mariner, flatcar, acl, and osguard. Kata is skipped explicitly here. + if [ "${IS_KATA}" = "true" ]; then + echo "Skipping node-exporter installation for kata (IS_KATA=${IS_KATA})" + else + # Download and install node-exporter-kubernetes at VHD build time. + # node-exporter is installed on the VHD so CSE only needs to enable+start it. + installNodeExporter "${PACKAGE_VERSIONS[0]}" + fi + ;; + "acr-mirror") + # acr-mirror is handled separately below via installAndConfigureArtifactStreaming. + ;; + "aznfs") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL "${PACKAGE_DOWNLOAD_URL}") + mkdir -p "${downloadDir}" + aznfsFilename=$(basename "${evaluatedURL}") + echo "Downloading aznfs RPM from ${evaluatedURL} to ${downloadDir}/${aznfsFilename}" + retrycmd_curl_file 120 5 25 "${downloadDir}/${aznfsFilename}" "${evaluatedURL}" || exit $ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT + echo " - aznfs version ${version}" >> ${VHD_LOGS_FILEPATH} + done + installAznfsPackage || exit $ERR_AZNFS_INSTALL_FAIL + ;; + "blobfuse"|"blobfuse2") + for version in "${PACKAGE_VERSIONS[@]}"; do + if isUbuntu "$OS"; then + if ! apt_get_install 10 2 120 "${name}=${version}"; then + journalctl --no-pager -u "${name}" || true + tail -n 200 /var/log/apt/term.log || true + tail -n 200 /var/log/dpkg.log || true + exit $ERR_APT_INSTALL_TIMEOUT + fi + echo " - ${name} version ${version}" >> "${VHD_LOGS_FILEPATH}" + else + echo " - ${name} installation skipped for ${OS}" >> "${VHD_LOGS_FILEPATH}" + fi + done + ;; + *) + echo "Package name: ${name} not supported for download. Please implement the download logic in the script." + # We can add a common function to download a generic package here. + # However, installation could be different for different packages. + ;; + esac + capture_benchmark "${SCRIPT_NAME}_download_${name}" + done <<< "$packages" +} + +cacheContainerImageComponents() { + # Download/cache all declared container images within components.json that apply to the respective OS SKU + ContainerImages=$(jq ".ContainerImages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq .downloadURL -r) + amd64OnlyVersionsStr=$(echo "${imageToBePulled}" | jq .amd64OnlyVersions -r) + updateMultiArchVersions "${imageToBePulled}" + amd64OnlyVersions="" + if [ "${amd64OnlyVersionsStr}" != "null" ]; then + amd64OnlyVersions=$(echo "${amd64OnlyVersionsStr}" | jq -r ".[]") + fi + + if [ "$(isARM64)" -eq 1 ]; then + versions="${MULTI_ARCH_VERSIONS[*]}" + else + versions="${amd64OnlyVersions} ${MULTI_ARCH_VERSIONS[*]}" + fi + + for version in ${versions}; do + CONTAINER_IMAGE=$(string_replace $downloadURL $version) + pullContainerImage "${cliTool}" "${CONTAINER_IMAGE}" & + image_pids+=($!) + echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} + while [ "$(jobs -p | wc -l)" -ge "$parallel_container_image_pull_limit" ]; do + wait -n || { + ret=$? + echo "A background job pullContainerImage failed: ${ret}, ${CONTAINER_IMAGE}. Exiting..." >&2 + for pid in "${image_pids[@]}"; do + kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" + done + exit "${ret}" + } done - ;; - "dcgm-exporter") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" - echo " - dcgm-exporter version ${version}" >> ${VHD_LOGS_FILEPATH} + done + done <<< "$ContainerImages" + echo "Waiting for container image pulls to finish. PID: ${image_pids[@]}" + while [ "$(jobs -p | wc -l)" -gt 0 ]; do + wait -n || { + ret=$? + echo "A background job pullContainerImage failed: ${ret}. Exiting..." >&2 + for pid in "${image_pids[@]}"; do + kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" done - ;; - "node-exporter") - # Skipping is handled by empty versionsV2 arrays in components.json - # for mariner, flatcar, acl, and osguard. Kata is skipped explicitly here. - if [ "${IS_KATA}" = "true" ]; then - echo "Skipping node-exporter installation for kata (IS_KATA=${IS_KATA})" - else - # Download and install node-exporter-kubernetes at VHD build time. - # node-exporter is installed on the VHD so CSE only needs to enable+start it. - installNodeExporter "${PACKAGE_VERSIONS[0]}" + exit "${ret}" + } + done +} + +# This function extracts CoreDNS binary from cached coredns images (latest version) +# and copies it to - /opt/azure/containers/localdns/binary/coredns. +# The binary is later used by localdns systemd unit. +# The function also handles the cleanup of temporary directories and unmounting of images. +extractAndCacheCoreDnsBinary() { + local coredns_image_list=($(ctr -n k8s.io images list -q | grep coredns)) + if [ "${#coredns_image_list[@]}" -eq 0 ]; then + echo "Error: No coredns images found." + exit 1 + fi + + rm -rf "${LOCALDNS_BINARY_PATH}" || exit 1 + mkdir -p "${LOCALDNS_BINARY_PATH}" || exit 1 + + cleanup_coredns_imports() { + set +e + if [ -n "${ctr_temp}" ]; then + ctr -n k8s.io images unmount "${ctr_temp}" >/dev/null + rm -rf "${ctr_temp}" + fi + } + trap cleanup_coredns_imports EXIT ABRT ERR INT PIPE QUIT TERM + + # Extract available coredns image tags (v1.12.0-1 format) and sort them in descending order. + local sorted_coredns_tags=($(for image in "${coredns_image_list[@]}"; do echo "${image##*:}"; done | sort -V -r)) + + # Determine latest version. + local latest_coredns_tag="${sorted_coredns_tags[0]}" + + # Extract the CoreDNS binary for the latest version. + for coredns_image_url in "${coredns_image_list[@]}"; do + if [ "${coredns_image_url##*:}" != "${latest_coredns_tag}" ]; then + continue + fi + + ctr_temp="$(mktemp -d)" + local max_retries=3 + local retry_count=0 + while [ $retry_count -lt $max_retries ]; do + if ctr -n k8s.io images mount "${coredns_image_url}" "${ctr_temp}" >/dev/null; then + break + fi + echo "Warning: Failed to mount ${coredns_image_url}, retrying..." >> "${VHD_LOGS_FILEPATH}" + sleep 2 + ((retry_count++)) + done + + if [ "$retry_count" -eq "$max_retries" ]; then + echo "Error: Failed to mount ${coredns_image_url} after ${max_retries} attempts." >> "${VHD_LOGS_FILEPATH}" + exit 1 + fi + + local coredns_binary="${ctr_temp}/usr/bin/coredns" + if [ -f "${coredns_binary}" ]; then + cp "${coredns_binary}" "${LOCALDNS_BINARY_PATH}/coredns" || { + echo "Error: Failed to copy coredns binary of ${latest_coredns_tag}" >> "${VHD_LOGS_FILEPATH}" + exit 1 + } + echo "Successfully copied coredns binary of ${latest_coredns_tag}" >> "${VHD_LOGS_FILEPATH}" + else + echo "Coredns binary not found for ${coredns_image_url}" >> "${VHD_LOGS_FILEPATH}" + fi + + ctr -n k8s.io images unmount "${ctr_temp}" >/dev/null + rm -rf "${ctr_temp}" + done + + # Clear the trap. + trap - EXIT ABRT ERR INT PIPE QUIT TERM +} + +# Collect grid compatibility data (placeholder for now - will be extended later) +collect_grid_compatibility_data() { + if [ -z "${GRID_COMPATIBILITY_DATA_FILE}" ] ; then + return + fi + + # Create basic grid compatibility data structure + # This is scaffolding - the actual Kusto query and analysis will be added later + local compatibility_data=$(jq -n \ + --arg os "${OS}" \ + --arg os_version "${OS_VERSION}" \ + --arg cpu_arch "${CPU_ARCH}" \ + --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --arg feature_flags "${FEATURE_FLAGS:-}" \ + '{ + "grid_compatibility_check": { + "timestamp": $timestamp, + "os": $os, + "os_version": $os_version, + "cpu_architecture": $cpu_arch, + "feature_flags": $feature_flags, + "compatibility_status": "data_collected", + "kusto_query_placeholder": "SELECT * FROM GridCompatibility WHERE timestamp > ago(1d)" + } + }') + + echo "${compatibility_data}" > "${GRID_COMPATIBILITY_DATA_FILE}" + chmod 755 "${GRID_COMPATIBILITY_DATA_FILE}" +} + +resolve_packages_source_url + +echo "" +echo "Components downloaded in this VHD build (some of the below components might get deleted during cluster provisioning if they are not needed):" >> ${VHD_LOGS_FILEPATH} +capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" + +echo "Logging the kernel after purge and reinstall + reboot: $(uname -r)" +# fix grub issue with cvm by reinstalling before other deps +# other VHDs use grub-pc, not grub-efi +if [ "$OS" = "$UBUNTU_OS_NAME" ] && echo "$FEATURE_FLAGS" | grep -q "cvm"; then + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + wait_for_apt_locks + apt_get_install 10 2 120 grub-efi || exit 1 +fi +capture_benchmark "${SCRIPT_NAME}_reinstall_grub_for_cvm" + +if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + # disable and mask all UU timers/services + # save some background io/latency + systemctl mask apt-daily.service apt-daily-upgrade.service || exit 1 + systemctl disable apt-daily.service apt-daily-upgrade.service || exit 1 + systemctl disable apt-daily.timer apt-daily-upgrade.timer || exit 1 + + tee /etc/apt/apt.conf.d/99periodic > /dev/null <> "${CHRONYD_DIR}"/10-chrony-restarts.conf < /dev/null <<'EOF' +Compress=yes +Storage=persistent +SystemMaxUse=1G +RuntimeMaxUse=1G +ForwardToSyslog=yes +EOF +capture_benchmark "${SCRIPT_NAME}_install_deps_and_set_configs" + +if [ "$(isARM64)" -eq 1 ]; then + # shellcheck disable=SC3010 + if [[ ${HYPERV_GENERATION,,} == "v1" ]]; then + echo "No arm64 support on V1 VM, exiting..." + exit 1 + fi +fi + +# Always override network config and disable NTP + Timesyncd and install Chrony +# Mariner does this differently, so only do it for Ubuntu +if ! isMarinerOrAzureLinux "$OS"; then + overrideNetworkConfig || exit 1 + disableNtpAndTimesyncdInstallChrony || exit 1 +fi + +# ACL inherits Azure Linux behaviors but isMarinerOrAzureLinux returns false, +# so these must be called separately (mirrored in the Mariner/AzureLinux block below). +# Other Mariner functions are safe to skip for ACL: +# setMarinerNetworkdConfig — ACL doesn't ship systemd-bootstrap's 99-dhcp-en.network +# fixCBLMarinerPermissions — product_uuid already 444; no rsyslog on ACL +# addMarinerNvidiaRepo / updateDnfWithNvidiaPkg / disableDNFAutomatic / enableCheckRestart — ACL has no dnf/rpm +# activateNfConntrack — nf_conntrack auto-loads via iptables dependency chain +# disableTimesyncd — ACL handles chrony separately above via disableNtpAndTimesyncdInstallChrony +if isACL "$OS" "$OS_VARIANT"; then + # ACL's iptables.service loads host firewall rules that conflict with Cilium eBPF routing. + disableSystemdIptables || exit 1 + # Repoint /etc/resolv.conf from the stub (127.0.0.53) to the real upstream file + # so DNS queries go directly through localdns. + disableSystemdResolvedCache +fi +capture_benchmark "${SCRIPT_NAME}_validate_container_runtime_and_override_ubuntu_net_config" + +# Configure SSH service during VHD build for Ubuntu 22.10+ +configureSSHService "$OS" "$OS_VERSION" || echo "##vso[task.logissue type=warning]SSH Service configuration failed, but continuing VHD build" + +CONTAINERD_SERVICE_DIR="/etc/systemd/system/containerd.service.d" +mkdir -p "${CONTAINERD_SERVICE_DIR}" +# Explicitly set LimitNOFILE=1048576 (the value that 'infinity' resolves to on Ubuntu 22.04) for both Ubuntu and Mariner/AzureLinux. +# On Ubuntu 24.04 (Containerd 2.0), LimitNOFILE is removed upstream and systemd falls back to an implicit soft:hard limit +# (for example 1024:524288), so containerd inherits a very low soft file descriptor limit (1024) unless we override it here. +# On Mariner/AzureLinux this is redundant with the base containerd.service unit but harmless. +# Not removing LimitNOFILE from parts/linux/cloud-init/artifacts/containerd.service, +# to avoid compatibility issues between new VHDs and old CSE scripts. +tee "${CONTAINERD_SERVICE_DIR}/exec_start.conf" > /dev/null < /dev/null < /etc/udev/rules.d/99-nfs.rules < /etc/udev/rules.d/80-azure-disk.rules <> ${VHD_LOGS_FILEPATH} - done - installAznfsPackage || exit $ERR_AZNFS_INSTALL_FAIL - ;; - "blobfuse"|"blobfuse2") - for version in "${PACKAGE_VERSIONS[@]}"; do - if isUbuntu "$OS"; then - if ! apt_get_install 10 2 120 "${name}=${version}"; then - journalctl --no-pager -u "${name}" || true - tail -n 200 /var/log/apt/term.log || true - tail -n 200 /var/log/dpkg.log || true - exit $ERR_APT_INSTALL_TIMEOUT - fi - echo " - ${name} version ${version}" >> "${VHD_LOGS_FILEPATH}" - else - echo " - ${name} installation skipped for ${OS}" >> "${VHD_LOGS_FILEPATH}" - fi - done - ;; - *) - echo "Package name: ${name} not supported for download. Please implement the download logic in the script." - # We can add a common function to download a generic package here. - # However, installation could be different for different packages. - ;; - esac - capture_benchmark "${SCRIPT_NAME}_download_${name}" -done <<< "$packages" + fi + disableTimesyncd + disableDNFAutomatic + enableCheckRestart + activateNfConntrack +elif [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then + updateAptWithMicrosoftPkg + updateAptWithNvidiaPkg +fi +capture_benchmark "${SCRIPT_NAME}_handle_os_specific_configurations" -installAndConfigureArtifactStreaming() { - local downloadURL="$1" - local version="$2" - # The arm64 packages have "-arm64" inserted before the file extension, - # e.g. acr-mirror-2204-arm64.deb instead of acr-mirror-2204.deb - if [ "$(isARM64)" -eq 1 ]; then - downloadURL="${downloadURL%.*}-arm64.${downloadURL##*.}" - fi - local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" - retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} - case "$downloadURL" in - *.deb) - apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD - ;; - *.rpm) - dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD - ;; - *) - echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 - exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} - ;; - esac - rm "$MIRROR_DOWNLOAD_PATH" +echo "VHD will be built with containerd as the container runtime" +# check if COMPONENTS_FILEPATH exists +if [ ! -f "$COMPONENTS_FILEPATH" ]; then + echo "Components file not found at $COMPONENTS_FILEPATH. Exiting..." + exit 1 +fi - /opt/acr/tools/overlaybd/install.sh - /opt/acr/tools/overlaybd/config-user-agent.sh azure - /opt/acr/tools/overlaybd/enable-http-auth.sh - /opt/acr/tools/overlaybd/config.sh download.enable false - /opt/acr/tools/overlaybd/config.sh cacheConfig.cacheSizeGB 32 - /opt/acr/tools/overlaybd/config.sh exporterConfig.enable true - /opt/acr/tools/overlaybd/config.sh exporterConfig.port 9863 - systemctl link /opt/overlaybd/overlaybd-tcmu.service /opt/overlaybd/snapshotter/overlaybd-snapshotter.service - # Remove the bundled overlaybd installer packages (~55-58 MB); install.sh already installed them and they're unused at runtime. - rm -f /opt/acr/tools/overlaybd/bin/*.deb /opt/acr/tools/overlaybd/bin/*.rpm - echo " - acr-mirror version ${version}" >> ${VHD_LOGS_FILEPATH} -} +# Cache packages and binaries declared within components.json +# cachePackageAndBinaryComponents # Artifact streaming (acr-mirror) - version and URLs resolved from components.json, # OS filtering handled declaratively by components.json entries ( for unsupported OSes). @@ -839,148 +1174,40 @@ PRESENT_DIR=$(pwd) ( cd $PRESENT_DIR || { echo "Subshell in the wrong directory" >&2; exit 1; } installBcc - exit $? -) > /var/log/bcc_installation.log 2>&1 & - -BCC_PID=$! - -# Add a separate section for runtime-installed components -# This clearly distinguishes components installed during CSE from VHD build-time components -# Only add for Ubuntu -if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then - cat << EOF >> ${VHD_LOGS_FILEPATH} -Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): - - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} -EOF -fi - -echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_pull_nvidia_driver_and_start_ebpf_downloads" - -string_replace() { - echo ${1//\*/$2} -} - -# Limit number of parallel pulls to 2 less than number of processor cores in order to prevent issues with network, CPU, and disk resources -# Account for possibility that number of cores is 3 or less -num_proc=$(nproc) -if [ "$num_proc" -gt 3 ]; then - parallel_container_image_pull_limit=$(nproc --ignore=2) -else - parallel_container_image_pull_limit=1 -fi -echo "Limit for parallel container image pulls set to $parallel_container_image_pull_limit" - -declare -a image_pids=() - -ContainerImages=$(jq ".ContainerImages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) -while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq .downloadURL -r) - amd64OnlyVersionsStr=$(echo "${imageToBePulled}" | jq .amd64OnlyVersions -r) - updateMultiArchVersions "${imageToBePulled}" - amd64OnlyVersions="" - if [ "${amd64OnlyVersionsStr}" != "null" ]; then - amd64OnlyVersions=$(echo "${amd64OnlyVersionsStr}" | jq -r ".[]") - fi - - if [ "$(isARM64)" -eq 1 ]; then - versions="${MULTI_ARCH_VERSIONS[*]}" - else - versions="${amd64OnlyVersions} ${MULTI_ARCH_VERSIONS[*]}" - fi - - for version in ${versions}; do - CONTAINER_IMAGE=$(string_replace $downloadURL $version) - pullContainerImage "${cliTool}" "${CONTAINER_IMAGE}" & - image_pids+=($!) - echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} - while [ "$(jobs -p | wc -l)" -ge "$parallel_container_image_pull_limit" ]; do - wait -n || { - ret=$? - echo "A background job pullContainerImage failed: ${ret}, ${CONTAINER_IMAGE}. Exiting..." >&2 - for pid in "${image_pids[@]}"; do - kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" - done - exit "${ret}" - } - done - done -done <<< "$ContainerImages" -echo "Waiting for container image pulls to finish. PID: ${image_pids[@]}" -while [ "$(jobs -p | wc -l)" -gt 0 ]; do - wait -n || { - ret=$? - echo "A background job pullContainerImage failed: ${ret}. Exiting..." >&2 - for pid in "${image_pids[@]}"; do - kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" - done - exit "${ret}" - } -done -capture_benchmark "${SCRIPT_NAME}_caching_container_images" - -retagAKSNodeCAWatcher() { - # This function retags the aks-node-ca-watcher image to a static tag - # The static tag is used to bootstrap custom CA trust when MCR egress may be intercepted by an untrusted TLS MITM firewall. - # The image is never pulled, it is only retagged. - - watcher=$(jq '.ContainerImages[] | select(.downloadURL | contains("aks-node-ca-watcher"))' $COMPONENTS_FILEPATH) - watcherBaseImg=$(echo $watcher | jq -r .downloadURL) - watcherVersion=$(echo $watcher | jq -r .multiArchVersionsV2[0].latestVersion) - watcherFullImg=${watcherBaseImg//\*/$watcherVersion} - - # this image will never get pulled, the tag must be the same across different SHAs. - # it will only ever be upgraded via node image changes. - # we do this because the image is used to bootstrap custom CA trust when MCR egress - # may be intercepted by an untrusted TLS MITM firewall. - watcherStaticImg=${watcherBaseImg//\*/static} - - # can't use $cliTool variable because crictl doesn't support retagging. - retagContainerImage "ctr" ${watcherFullImg} ${watcherStaticImg} -} -retagAKSNodeCAWatcher -capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" - -pinPodSandboxImages() { - # This function pins the pod sandbox image(s) to avoid Kubelet's Garbage Collector (GC) from removing them. - # This is achieved by setting the "io.cri-containerd.pinned" label on the image with a value of "pinned". - # These images are critical for pod startup and aren't supported with private ACR since containerd won't be using azure-acr-credential to fetch them. - - # Get all pause images as individual JSON objects - local pause_images - pause_images=$(jq -c '.ContainerImages[] | select(.downloadURL | contains("pause"))' $COMPONENTS_FILEPATH) - - if [ -z "$pause_images" ]; then - echo "Warning: No pause images found in components.json" - return 0 - fi - - # Process each pause image separately - while IFS= read -r podSandbox; do - if [ -z "$podSandbox" ]; then - continue - fi + exit $? +) > /var/log/bcc_installation.log 2>&1 & - local podSandboxBaseImg - local podSandboxVersion - local podSandboxFullImg +BCC_PID=$! - podSandboxBaseImg=$(echo "$podSandbox" | jq -r '.downloadURL') - podSandboxVersion=$(echo "$podSandbox" | jq -r '.multiArchVersionsV2[0].latestVersion') +# Add a separate section for runtime-installed components +# This clearly distinguishes components installed during CSE from VHD build-time components +# Only add for Ubuntu +if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then + cat << EOF >> ${VHD_LOGS_FILEPATH} +Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): + - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} +EOF +fi - # Skip if we couldn't extract the required information - if [ "$podSandboxBaseImg" = "null" ] || [ "$podSandboxVersion" = "null" ]; then - echo "Warning: Could not extract downloadURL or latestVersion from pause image: $podSandbox" - continue - fi +echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} +capture_benchmark "${SCRIPT_NAME}_pull_nvidia_driver_and_start_ebpf_downloads" - podSandboxFullImg=${podSandboxBaseImg//\*/$podSandboxVersion} +# Limit number of parallel pulls to 2 less than number of processor cores in order to prevent issues with network, CPU, and disk resources +# Account for possibility that number of cores is 3 or less +num_proc=$(nproc) +if [ "$num_proc" -gt 3 ]; then + parallel_container_image_pull_limit=$(nproc --ignore=2) +else + parallel_container_image_pull_limit=1 +fi +echo "Limit for parallel container image pulls set to $parallel_container_image_pull_limit" - echo "Pinning pause image: $podSandboxFullImg" - labelContainerImage "${podSandboxFullImg}" "io.cri-containerd.pinned" "pinned" +declare -a image_pids=() + +# Cache container images declared within components.json +cacheContainerImageComponents +capture_benchmark "${SCRIPT_NAME}_caching_container_images" - done <<< "$pause_images" -} pinPodSandboxImages capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" @@ -1011,53 +1238,6 @@ if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ fi capture_benchmark "${SCRIPT_NAME}_configure_telemetry" -# download kubernetes package from the given URL using azcopy -# if it is a kube-proxy package, extract image from the downloaded package -cacheKubePackageFromPrivateUrl() { - local kube_private_binary_url="$1" - - echo "process private package url: $kube_private_binary_url" - - mkdir -p ${K8S_PRIVATE_PACKAGES_CACHE_DIR} # /opt/kubernetes/downloads/private-packages - - # save kube pkg with version number from the url path, this convention is used to find the cached package at run-time - local k8s_tgz_name - k8s_tgz_name=$(echo "$kube_private_binary_url" | grep -o -P '(?<=\/kubernetes\/).*(?=\/binaries\/)').tar.gz - - # use azcopy instead of curl to download packages - getAzCopyCurrentPath - - export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" - export AZCOPY_CONCURRENCY_VALUE="AUTO" - export AZCOPY_LOG_LOCATION="$(pwd)/azcopy-log-files/" - export AZCOPY_JOB_PLAN_LOCATION="$(pwd)/azcopy-job-plan-files/" - mkdir -p "${AZCOPY_LOG_LOCATION}" - mkdir -p "${AZCOPY_JOB_PLAN_LOCATION}" - - cached_pkg="${K8S_PRIVATE_PACKAGES_CACHE_DIR}/${k8s_tgz_name}" - echo "download private package ${kube_private_binary_url} and store as ${cached_pkg}" - - if ! ./azcopy copy "${kube_private_binary_url}" "${cached_pkg}"; then - azExitCode=$? - # loop through azcopy log files - shopt -s nullglob - for f in "${AZCOPY_LOG_LOCATION}"/*.log; do - echo "Azcopy log file: $f" - # upload the log file as an attachment to vso - echo "##vso[build.uploadlog]$f" - # check if the log file contains any errors - if grep -q '"level":"Error"' "$f"; then - echo "log file $f contains errors" - echo "##vso[task.logissue type=error]Azcopy log file $f contains errors" - # print the log file - cat "$f" - fi - done - shopt -u nullglob - exit $ERR_PRIVATE_K8S_PKG_ERR - fi -} - if [ "$OS" = "$UBUNTU_OS_NAME" ]; then # remove snapd, which is not used by container stack apt_get_purge 20 30 120 snapd || exit 1 @@ -1091,70 +1271,6 @@ else fi capture_benchmark "${SCRIPT_NAME}_finish_installing_bcc_tools" -# Configure LSM modules to include BPF -configureLsmWithBpf() { - echo "Configuring LSM modules to include BPF..." - - # Read current LSM modules - if [ ! -f /sys/kernel/security/lsm ]; then - echo "Warning: /sys/kernel/security/lsm not found, skipping LSM configuration" - return 0 - fi - - local current_lsm - current_lsm=$(cat /sys/kernel/security/lsm) - echo "Current LSM modules: $current_lsm" - - # Prepend bpf to the LSM list if not already present - if ! echo "$current_lsm" | grep -q bpf; then - if [ "$IS_KATA" = "true" ] || echo "$FEATURE_FLAGS" | grep -q "cvm"; then - echo "Warning: this is a Kata/CVM SKU - will not add BPF to LSM configuration" - return 0 - fi - - if isAzureLinuxOSGuard "$OS" "$OS_VARIANT"; then - echo "Warning: Azure Linux OS Guard built with signed UKI, not enabling BPF LSM" - return 0 - fi - - local new_lsm="bpf,$current_lsm" - echo "New LSM configuration: $new_lsm" - - if [ "$OS" = "$UBUNTU_OS_NAME" ] && [ "$OS_VERSION" = "24.04" ]; then - local grub_cfg="/etc/default/grub.d/50-cloudimg-settings.cfg" - if [ -f "$grub_cfg" ]; then - if grep -q "lsm=" "$grub_cfg"; then - sed -i "s/lsm=[^[:space:]]*/lsm=$new_lsm/g" "$grub_cfg" - else - sed -i "s/GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"lsm=$new_lsm /" "$grub_cfg" - fi - echo "Updating GRUB configuration for Ubuntu 24.04..." - update-grub2 /boot/grub/grub.cfg || echo "Warning: Failed to update GRUB configuration" - else - echo "Warning: $grub_cfg not found, skipping LSM configuration" - fi - elif isMarinerOrAzureLinux "$OS" && [ "$OS_VERSION" = "3.0" ]; then - if [ -f /etc/default/grub ]; then - if grep -q "lsm=" /etc/default/grub; then - sed -i "s/lsm=[^[:space:]]*/lsm=$new_lsm/g" /etc/default/grub - else - sed -i "s/GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"lsm=$new_lsm /" /etc/default/grub - fi - echo "Updating GRUB configuration for Azure Linux 3.0..." - grub2-mkconfig -o /boot/grub2/grub.cfg || echo "Warning: Failed to update GRUB configuration" - else - echo "Warning: /etc/default/grub not found, skipping LSM configuration" - fi - else - echo "LSM BPF configuration is only enabled for Ubuntu 24.04 and Azure Linux 3.0, skipping" - fi - - echo "LSM configuration update completed" - else - echo "BPF LSM already configured, skipping" - fi -} - configureLsmWithBpf capture_benchmark "${SCRIPT_NAME}_configure_lsm_with_bpf" @@ -1167,114 +1283,12 @@ if [ -n "${PRIVATE_PACKAGES_URL:-}" ]; then cacheKubePackageFromPrivateUrl "$private_url" done fi - -LOCALDNS_BINARY_PATH="/opt/azure/containers/localdns/binary" -# This function extracts CoreDNS binary from cached coredns images (latest version) -# and copies it to - /opt/azure/containers/localdns/binary/coredns. -# The binary is later used by localdns systemd unit. -# The function also handles the cleanup of temporary directories and unmounting of images. -extractAndCacheCoreDnsBinary() { - local coredns_image_list=($(ctr -n k8s.io images list -q | grep coredns)) - if [ "${#coredns_image_list[@]}" -eq 0 ]; then - echo "Error: No coredns images found." - exit 1 - fi - - rm -rf "${LOCALDNS_BINARY_PATH}" || exit 1 - mkdir -p "${LOCALDNS_BINARY_PATH}" || exit 1 - - cleanup_coredns_imports() { - set +e - if [ -n "${ctr_temp}" ]; then - ctr -n k8s.io images unmount "${ctr_temp}" >/dev/null - rm -rf "${ctr_temp}" - fi - } - trap cleanup_coredns_imports EXIT ABRT ERR INT PIPE QUIT TERM - - # Extract available coredns image tags (v1.12.0-1 format) and sort them in descending order. - local sorted_coredns_tags=($(for image in "${coredns_image_list[@]}"; do echo "${image##*:}"; done | sort -V -r)) - - # Determine latest version. - local latest_coredns_tag="${sorted_coredns_tags[0]}" - - # Extract the CoreDNS binary for the latest version. - for coredns_image_url in "${coredns_image_list[@]}"; do - if [ "${coredns_image_url##*:}" != "${latest_coredns_tag}" ]; then - continue - fi - - ctr_temp="$(mktemp -d)" - local max_retries=3 - local retry_count=0 - while [ $retry_count -lt $max_retries ]; do - if ctr -n k8s.io images mount "${coredns_image_url}" "${ctr_temp}" >/dev/null; then - break - fi - echo "Warning: Failed to mount ${coredns_image_url}, retrying..." >> "${VHD_LOGS_FILEPATH}" - sleep 2 - ((retry_count++)) - done - - if [ "$retry_count" -eq "$max_retries" ]; then - echo "Error: Failed to mount ${coredns_image_url} after ${max_retries} attempts." >> "${VHD_LOGS_FILEPATH}" - exit 1 - fi - - local coredns_binary="${ctr_temp}/usr/bin/coredns" - if [ -f "${coredns_binary}" ]; then - cp "${coredns_binary}" "${LOCALDNS_BINARY_PATH}/coredns" || { - echo "Error: Failed to copy coredns binary of ${latest_coredns_tag}" >> "${VHD_LOGS_FILEPATH}" - exit 1 - } - echo "Successfully copied coredns binary of ${latest_coredns_tag}" >> "${VHD_LOGS_FILEPATH}" - else - echo "Coredns binary not found for ${coredns_image_url}" >> "${VHD_LOGS_FILEPATH}" - fi - - ctr -n k8s.io images unmount "${ctr_temp}" >/dev/null - rm -rf "${ctr_temp}" - done - - # Clear the trap. - trap - EXIT ABRT ERR INT PIPE QUIT TERM -} +rm -f ./azcopy # cleanup immediately after usage will return in two downloads extractAndCacheCoreDnsBinary -rm -f ./azcopy # cleanup immediately after usage will return in two downloads echo "install-dependencies step completed successfully" -# Collect grid compatibility data (placeholder for now - will be extended later) -collect_grid_compatibility_data() { - if [ -z "${GRID_COMPATIBILITY_DATA_FILE}" ] ; then - return - fi - - # Create basic grid compatibility data structure - # This is scaffolding - the actual Kusto query and analysis will be added later - local compatibility_data=$(jq -n \ - --arg os "${OS}" \ - --arg os_version "${OS_VERSION}" \ - --arg cpu_arch "${CPU_ARCH}" \ - --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --arg feature_flags "${FEATURE_FLAGS:-}" \ - '{ - "grid_compatibility_check": { - "timestamp": $timestamp, - "os": $os, - "os_version": $os_version, - "cpu_architecture": $cpu_arch, - "feature_flags": $feature_flags, - "compatibility_status": "data_collected", - "kusto_query_placeholder": "SELECT * FROM GridCompatibility WHERE timestamp > ago(1d)" - } - }') - - echo "${compatibility_data}" > "${GRID_COMPATIBILITY_DATA_FILE}" - chmod 755 "${GRID_COMPATIBILITY_DATA_FILE}" -} - collect_grid_compatibility_data # nvidia repos are non msft public endpoints and should not be present on VHDs. From 404dcbcdc64c24d50e46c16e15e3951d63edb678 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Mon, 6 Jul 2026 14:16:45 -0700 Subject: [PATCH 10/78] chore: always updateAptWithMicrosoftPkg + installDeps --- vhdbuilder/packer/install-dependencies.sh | 36 ++++++++++++++--------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index a5da0684c1e..c52525e67fc 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -793,6 +793,14 @@ collect_grid_compatibility_data() { chmod 755 "${GRID_COMPATIBILITY_DATA_FILE}" } +# --------------------------------------------------------- INSTALL DEPENDENCIES START --------------------------------------------------------- + +# check if COMPONENTS_FILEPATH exists +if [ ! -f "$COMPONENTS_FILEPATH" ]; then + echo "Components file not found at $COMPONENTS_FILEPATH. Exiting..." + exit 1 +fi + resolve_packages_source_url echo "" @@ -824,13 +832,16 @@ APT::Periodic::Unattended-Upgrade "0"; EOF fi -# If the IMG_SKU does not contain "minimal", installDeps normally -# shellcheck disable=SC3010 -if [[ "$IMG_SKU" != *"minimal"* ]]; then - installDeps -else - updateAptWithMicrosoftPkg -fi +# # If the IMG_SKU does not contain "minimal", installDeps normally +# # shellcheck disable=SC3010 +# if [[ "$IMG_SKU" != *"minimal"* ]]; then +# installDeps +# else +# updateAptWithMicrosoftPkg +# fi + +updateAptWithMicrosoftPkg +installDeps CHRONYD_DIR=/etc/systemd/system/chronyd.service.d @@ -994,11 +1005,6 @@ fi capture_benchmark "${SCRIPT_NAME}_handle_os_specific_configurations" echo "VHD will be built with containerd as the container runtime" -# check if COMPONENTS_FILEPATH exists -if [ ! -f "$COMPONENTS_FILEPATH" ]; then - echo "Components file not found at $COMPONENTS_FILEPATH. Exiting..." - exit 1 -fi # Cache packages and binaries declared within components.json # cachePackageAndBinaryComponents @@ -1287,8 +1293,6 @@ rm -f ./azcopy # cleanup immediately after usage will return in two downloads extractAndCacheCoreDnsBinary -echo "install-dependencies step completed successfully" - collect_grid_compatibility_data # nvidia repos are non msft public endpoints and should not be present on VHDs. @@ -1297,5 +1301,9 @@ collect_grid_compatibility_data removeNvidiaRepos capture_benchmark "${SCRIPT_NAME}_remove_nvidia_repos" +echo "install-dependencies step completed successfully" + +# --------------------------------------------------------- INSTALL DEPENDENCIES END --------------------------------------------------------- + capture_benchmark "${SCRIPT_NAME}_overall" true process_benchmarks From 8fe04f06b3d8406519a0932f8efb6d9b1eb7c358 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Mon, 6 Jul 2026 15:16:53 -0700 Subject: [PATCH 11/78] chore: more changes --- vhdbuilder/packer/install-dependencies.sh | 564 +++++++++++----------- 1 file changed, 287 insertions(+), 277 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index c52525e67fc..56c0230f717 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -239,11 +239,8 @@ retagAKSNodeCAWatcher() { # may be intercepted by an untrusted TLS MITM firewall. watcherStaticImg=${watcherBaseImg//\*/static} - # can't use $cliTool variable because crictl doesn't support retagging. retagContainerImage "ctr" ${watcherFullImg} ${watcherStaticImg} } -retagAKSNodeCAWatcher -capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" pinPodSandboxImages() { # This function pins the pod sandbox image(s) to avoid Kubelet's Garbage Collector (GC) from removing them. @@ -313,7 +310,6 @@ cacheKubePackageFromPrivateUrl() { echo "download private package ${kube_private_binary_url} and store as ${cached_pkg}" if ! ./azcopy copy "${kube_private_binary_url}" "${cached_pkg}"; then - azExitCode=$? # loop through azcopy log files shopt -s nullglob for f in "${AZCOPY_LOG_LOCATION}"/*.log; do @@ -333,6 +329,38 @@ cacheKubePackageFromPrivateUrl() { fi } +starteBPFToolsInstallation() { + installBpftrace + echo " - $(bpftrace --version)" >> ${VHD_LOGS_FILEPATH} + + PRESENT_DIR=$(pwd) + # run installBcc in a subshell and continue on with container image pull in order to decrease total build time + ( + cd $PRESENT_DIR || { echo "Subshell in the wrong directory" >&2; exit 1; } + installBcc + exit $? + ) > /var/log/bcc_installation.log 2>&1 & + + BCC_PID=$! +} + +finisheBPFToolsInstallation() { + wait $BCC_PID + BCC_EXIT_CODE=$? + chmod 644 /var/log/bcc_installation.log + + if [ "$BCC_EXIT_CODE" -eq 0 ]; then + echo "Bcc tools successfully installed." + cat << EOF >> ${VHD_LOGS_FILEPATH} + - bcc-tools + - libbcc-examples +EOF + else + echo "Error: installBcc subshell failed with exit code $BCC_EXIT_CODE" >&2 + exit $BCC_EXIT_CODE + fi +} + # Configure LSM modules to include BPF configureLsmWithBpf() { echo "Configuring LSM modules to include BPF..." @@ -397,6 +425,70 @@ configureLsmWithBpf() { fi } +setUdevRules() { + # set udev rules + echo "set read ahead size to 15380 KB" + AWK_PATH=$(command -v awk) + cat > /etc/udev/rules.d/99-nfs.rules < /etc/udev/rules.d/80-azure-disk.rules < for unsupported OSes). + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installAndConfigureArtifactStreaming "${evaluatedURL}" "${version}" + done ;; "aznfs") for version in ${PACKAGE_VERSIONS[@]}; do @@ -646,6 +743,19 @@ cachePackageAndBinaryComponents() { cacheContainerImageComponents() { # Download/cache all declared container images within components.json that apply to the respective OS SKU + + # Limit number of parallel pulls to 2 less than number of processor cores in order to prevent issues with network, CPU, and disk resources + # Account for possibility that number of cores is 3 or less + num_proc=$(nproc) + if [ "$num_proc" -gt 3 ]; then + parallel_container_image_pull_limit=$(nproc --ignore=2) + else + parallel_container_image_pull_limit=1 + fi + echo "Limit for parallel container image pulls set to $parallel_container_image_pull_limit" + + declare -a image_pids=() + ContainerImages=$(jq ".ContainerImages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) while IFS= read -r imageToBePulled; do downloadURL=$(echo "${imageToBePulled}" | jq .downloadURL -r) @@ -664,7 +774,7 @@ cacheContainerImageComponents() { for version in ${versions}; do CONTAINER_IMAGE=$(string_replace $downloadURL $version) - pullContainerImage "${cliTool}" "${CONTAINER_IMAGE}" & + pullContainerImage "ctr" "${CONTAINER_IMAGE}" & image_pids+=($!) echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} while [ "$(jobs -p | wc -l)" -ge "$parallel_container_image_pull_limit" ]; do @@ -692,6 +802,159 @@ cacheContainerImageComponents() { done } +cacheGPUContainerImageComponents() { + # Download/install GPU container image components + GPUContainerImages=$(jq -c '.GPUContainerImages[]' $COMPONENTS_FILEPATH) + + NVIDIA_DRIVER_IMAGE="" + NVIDIA_DRIVER_IMAGE_TAG="" + NVIDIA_GRID_DRIVER_VERSION="" + + # Extract GRID driver version for release notes (applicable to all Linux distributions) + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') + # shellcheck disable=SC2001 + imageName=$(echo "$downloadURL" | sed 's/:.*$//') + + if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-grid" ]; then + NVIDIA_GRID_DRIVER_VERSION=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') + # Continue to extract CUDA driver info as well + fi + done <<< "$GPUContainerImages" + + # For Ubuntu, pre-pull the CUDA driver image + if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now + gpu_action="copy" + + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') + # shellcheck disable=SC2001 + imageName=$(echo "$downloadURL" | sed 's/:.*$//') + + if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-cuda-lts" ]; then + latestVersion=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') + NVIDIA_DRIVER_IMAGE="$imageName" + NVIDIA_DRIVER_IMAGE_TAG="$latestVersion" + break # Exit the loop once we find the image + fi + done <<< "$GPUContainerImages" + + # Check if the NVIDIA_DRIVER_IMAGE and NVIDIA_DRIVER_IMAGE_TAG were found + if [ -z "$NVIDIA_DRIVER_IMAGE" ] || [ -z "$NVIDIA_DRIVER_IMAGE_TAG" ]; then + echo "Error: Unable to find aks-gpu-cuda-lts image in components.json" + exit 1 + fi + + mkdir -p /opt/{actions,gpu} + + /opt/azure/containers/image-fetcher "$NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG" + + cat << EOF >> ${VHD_LOGS_FILEPATH} + - nvidia-cuda-driver=${NVIDIA_DRIVER_IMAGE_TAG} +EOF + + # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the + # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles + # and DKMS-registers the kernel module + stages userspace libs against THIS VHD's kernel, + # performs NO device access (safe on the GPU-less Packer builder), and writes the marker + # /opt/azure/aks-gpu/dkms-marker. At node boot, configGPUDrivers passes "install-skip-build" + # when that marker matches, running only the device-dependent steps. + # The driver image is intentionally LEFT in the VHD: boot-time device init still sources the + # container toolkit debs, fabric manager, containerd runtime config and udev rules from it. + # Dropping the image is a separate, deferred size optimization. + if grep -q "NVIDIA_CUDA_PREBAKE" <<< "$FEATURE_FLAGS"; then + echo "Pre-building NVIDIA CUDA kernel module into the VHD (build-only) for kernel $(uname -r)" + # nvidia-installer needs gcc/make + libc6-dev to compile; the builder lacks them here, so install + # them. A boot-time fallback recompile (marker mismatch) still has them: the VHD ships + # build-essential -> libc6-dev (release-notes manifests) -- the toolchain baseline GPU nodes compile + # with at boot (installDeps runs apt --no-install-recommends, so gcc alone doesn't pull libc6-dev). + apt_get_install 10 2 300 gcc make libc6-dev || exit 1 + CTR_GPU_PREBUILD_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind" + retrycmd_if_failure 3 10 600 bash -c "$CTR_GPU_PREBUILD_CMD $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG gpuprebuild /entrypoint.sh build-only" || exit 1 + if [ ! -f /opt/azure/aks-gpu/dkms-marker ]; then + echo "Error: NVIDIA CUDA prebake did not produce /opt/azure/aks-gpu/dkms-marker" + exit 1 + fi + cat << EOF >> ${VHD_LOGS_FILEPATH} + - nvidia-cuda-driver-prebaked=${NVIDIA_DRIVER_IMAGE_TAG} (kernel $(uname -r)) +EOF + fi + fi + + # Add a separate section for runtime-installed components + # This clearly distinguishes components installed during CSE from VHD build-time components + # Only add for Ubuntu + if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then + cat << EOF >> ${VHD_LOGS_FILEPATH} +Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): + - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} +EOF + fi +} + +configureGraceBlackwell() { + if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then + # NVIDIA GB setup is only supported on arm64 Ubuntu 24.04. + if [ "${CPU_ARCH}" = "arm64" ] && [ "${UBUNTU_RELEASE}" = "24.04" ]; then + # Replicate all functionality from github.com/azure/aks-gpu/install.sh. + # aks-gpu is designed to run at node boot/join time, whereas the NVIDIA GB VHD is set up + # to have all drivers installed at VHD build time. + + # 1. Blacklist nouveau driver + cat << EOF >> /etc/modprobe.d/blacklist-nouveau.conf +blacklist nouveau +options nouveau modeset=0 +EOF + update-initramfs -u + + # 2. install drivers + BOM_PATH="gb-mai-bom.json" + + # Install a custom repository if a doca-custom-repo is specified + DOCA_CUSTOM_REPO=$(jq -r '.["doca-custom-repo"]' $BOM_PATH) + if [ -n "$DOCA_CUSTOM_REPO" ]; then + mv /etc/apt/sources.list.d/doca-net.list /etc/apt/sources.list.d/doca-net.list.backup + echo "deb [arch=arm64 signed-by=/etc/apt/keyrings/doca-net.pub] $DOCA_CUSTOM_REPO ./" > /etc/apt/sources.list.d/doca-net.list + apt-get update + fi + + # Install DOCA/OFED before the GPU driver so nvidia-peermem can build against the RDMA APIs provided by OFED. + # Farcically, nvidia-dkms-580-open cannot be installed together with the CUDA toolkit. Something about that package changes the build environment in an incompatible way. I've seen people mention CUDA including an old version of gcc that somehow makes its way onto the PATH... + # Therefore we install DOCA/OFED first, the GPU driver and its dependencies second, then all downstream reverse-dependencies (CUDA, DCGM, and so forth) third. + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave1"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave2"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave3"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + + # 3. Add char device symlinks for NVIDIA devices + mkdir -p "$(dirname /lib/udev/rules.d/71-nvidia-dev-char.rules)" + cat << EOF >> /lib/udev/rules.d/71-nvidia-dev-char.rules +ACTION=="add", DEVPATH=="/bus/pci/drivers/nvidia", RUN+="/usr/bin/nvidia-ctk system create-dev-char-symlinks --create-all" +EOF + + # Create systemd drop-in to override nvidia-device-plugin dependencies + mkdir -p /etc/systemd/system/nvidia-device-plugin.service.d + cat << EOF > /etc/systemd/system/nvidia-device-plugin.service.d/override.conf +[Unit] +After=kubelet.service + +[Service] +ExecStartPre=-/usr/bin/mkdir -p /var/lib/kubelet/device-plugins +EOF + + # Now we are off-piste: enable DCGM, DCGM exporter, container device plugin, and the NVIDIA containerd config. + systemctl enable nvidia-dcgm + systemctl enable nvidia-dcgm-exporter + systemctl enable nvidia-device-plugin + systemctl enable openibd + + # One additional request from MAI: signal that NPD is pre-installed on the VHD. + # When this file is present, the Azure AKS VM Extension skips installing NPD at provision time. + mkdir -p /etc/node-problem-detector.d/ + touch /etc/node-problem-detector.d/skip_vhd_npd + fi + fi +} + # This function extracts CoreDNS binary from cached coredns images (latest version) # and copies it to - /opt/azure/containers/localdns/binary/coredns. # The binary is later used by localdns systemd unit. @@ -840,6 +1103,7 @@ fi # updateAptWithMicrosoftPkg # fi +# TODO: confirm if this is right updateAptWithMicrosoftPkg installDeps @@ -918,66 +1182,7 @@ net.bridge.bridge-nf-call-iptables = 1 EOF capture_benchmark "${SCRIPT_NAME}_set_ip_forwarding" -echo "set read ahead size to 15380 KB" -AWK_PATH=$(command -v awk) -cat > /etc/udev/rules.d/99-nfs.rules < /etc/udev/rules.d/80-azure-disk.rules < for unsupported OSes). -acrMirrorPackage=$(echo "${packages}" | jq -c 'select(.name == "acr-mirror")') -updatePackageVersions "${acrMirrorPackage}" "${OS}" "${OS_VERSION}" "${OS_VARIANT}" -updatePackageDownloadURL "${acrMirrorPackage}" "${OS}" "${OS_VERSION}" "${OS_VARIANT}" -if [ "${#PACKAGE_VERSIONS[@]}" -gt 0 ] && [ "${PACKAGE_VERSIONS[0]}" != "" ]; then - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installAndConfigureArtifactStreaming "${evaluatedURL}" "${version}" - done -fi -capture_benchmark "${SCRIPT_NAME}_install_artifact_streaming" +starteBPFToolsInstallation +capture_benchmark "${SCRIPT_NAME}_start_install_ebpf_tools" +# Cache container images declared within components.json # k8s will use images in the k8s.io namespaces - create it ctr namespace create k8s.io -cliTool="ctr" - -INSTALLED_RUNC_VERSION=$(runc --version | head -n1 | sed 's/runc version //') -echo " - runc version ${INSTALLED_RUNC_VERSION}" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_install_crictl" - -GPUContainerImages=$(jq -c '.GPUContainerImages[]' $COMPONENTS_FILEPATH) - -NVIDIA_DRIVER_IMAGE="" -NVIDIA_DRIVER_IMAGE_TAG="" -NVIDIA_GRID_DRIVER_VERSION="" - -# Extract GRID driver version for release notes (applicable to all Linux distributions) -while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') - # shellcheck disable=SC2001 - imageName=$(echo "$downloadURL" | sed 's/:.*$//') - - if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-grid" ]; then - NVIDIA_GRID_DRIVER_VERSION=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') - # Continue to extract CUDA driver info as well - fi -done <<< "$GPUContainerImages" - -# For Ubuntu, pre-pull the CUDA driver image -if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now - gpu_action="copy" - - while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') - # shellcheck disable=SC2001 - imageName=$(echo "$downloadURL" | sed 's/:.*$//') - - if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-cuda-lts" ]; then - latestVersion=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') - NVIDIA_DRIVER_IMAGE="$imageName" - NVIDIA_DRIVER_IMAGE_TAG="$latestVersion" - break # Exit the loop once we find the image - fi - done <<< "$GPUContainerImages" - - # Check if the NVIDIA_DRIVER_IMAGE and NVIDIA_DRIVER_IMAGE_TAG were found - if [ -z "$NVIDIA_DRIVER_IMAGE" ] || [ -z "$NVIDIA_DRIVER_IMAGE_TAG" ]; then - echo "Error: Unable to find aks-gpu-cuda-lts image in components.json" - exit 1 - fi - - mkdir -p /opt/{actions,gpu} - - /opt/azure/containers/image-fetcher "$NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG" - - cat << EOF >> ${VHD_LOGS_FILEPATH} - - nvidia-cuda-driver=${NVIDIA_DRIVER_IMAGE_TAG} -EOF - - # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the - # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles - # and DKMS-registers the kernel module + stages userspace libs against THIS VHD's kernel, - # performs NO device access (safe on the GPU-less Packer builder), and writes the marker - # /opt/azure/aks-gpu/dkms-marker. At node boot, configGPUDrivers passes "install-skip-build" - # when that marker matches, running only the device-dependent steps. - # The driver image is intentionally LEFT in the VHD: boot-time device init still sources the - # container toolkit debs, fabric manager, containerd runtime config and udev rules from it. - # Dropping the image is a separate, deferred size optimization. - if grep -q "NVIDIA_CUDA_PREBAKE" <<< "$FEATURE_FLAGS"; then - echo "Pre-building NVIDIA CUDA kernel module into the VHD (build-only) for kernel $(uname -r)" - # nvidia-installer needs gcc/make + libc6-dev to compile; the builder lacks them here, so install - # them. A boot-time fallback recompile (marker mismatch) still has them: the VHD ships - # build-essential -> libc6-dev (release-notes manifests) -- the toolchain baseline GPU nodes compile - # with at boot (installDeps runs apt --no-install-recommends, so gcc alone doesn't pull libc6-dev). - apt_get_install 10 2 300 gcc make libc6-dev || exit 1 - CTR_GPU_PREBUILD_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind" - retrycmd_if_failure 3 10 600 bash -c "$CTR_GPU_PREBUILD_CMD $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG gpuprebuild /entrypoint.sh build-only" || exit 1 - if [ ! -f /opt/azure/aks-gpu/dkms-marker ]; then - echo "Error: NVIDIA CUDA prebake did not produce /opt/azure/aks-gpu/dkms-marker" - exit 1 - fi - cat << EOF >> ${VHD_LOGS_FILEPATH} - - nvidia-cuda-driver-prebaked=${NVIDIA_DRIVER_IMAGE_TAG} (kernel $(uname -r)) -EOF - fi -fi - -if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then - # NVIDIA GB setup is only supported on arm64 Ubuntu 24.04. - if [ "${CPU_ARCH}" = "arm64" ] && [ "${UBUNTU_RELEASE}" = "24.04" ]; then - # Replicate all functionality from github.com/azure/aks-gpu/install.sh. - # aks-gpu is designed to run at node boot/join time, whereas the NVIDIA GB VHD is set up - # to have all drivers installed at VHD build time. - - # 1. Blacklist nouveau driver - cat << EOF >> /etc/modprobe.d/blacklist-nouveau.conf -blacklist nouveau -options nouveau modeset=0 -EOF - update-initramfs -u - - # 2. install drivers - BOM_PATH="gb-mai-bom.json" - - # Install a custom repository if a doca-custom-repo is specified - DOCA_CUSTOM_REPO=$(jq -r '.["doca-custom-repo"]' $BOM_PATH) - if [ -n "$DOCA_CUSTOM_REPO" ]; then - mv /etc/apt/sources.list.d/doca-net.list /etc/apt/sources.list.d/doca-net.list.backup - echo "deb [arch=arm64 signed-by=/etc/apt/keyrings/doca-net.pub] $DOCA_CUSTOM_REPO ./" > /etc/apt/sources.list.d/doca-net.list - apt-get update - fi - - # Install DOCA/OFED before the GPU driver so nvidia-peermem can build against the RDMA APIs provided by OFED. - # Farcically, nvidia-dkms-580-open cannot be installed together with the CUDA toolkit. Something about that package changes the build environment in an incompatible way. I've seen people mention CUDA including an old version of gcc that somehow makes its way onto the PATH... - # Therefore we install DOCA/OFED first, the GPU driver and its dependencies second, then all downstream reverse-dependencies (CUDA, DCGM, and so forth) third. - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave1"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave2"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave3"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - - # 3. Add char device symlinks for NVIDIA devices - mkdir -p "$(dirname /lib/udev/rules.d/71-nvidia-dev-char.rules)" - cat << EOF >> /lib/udev/rules.d/71-nvidia-dev-char.rules -ACTION=="add", DEVPATH=="/bus/pci/drivers/nvidia", RUN+="/usr/bin/nvidia-ctk system create-dev-char-symlinks --create-all" -EOF - - # Create systemd drop-in to override nvidia-device-plugin dependencies - mkdir -p /etc/systemd/system/nvidia-device-plugin.service.d - cat << EOF > /etc/systemd/system/nvidia-device-plugin.service.d/override.conf -[Unit] -After=kubelet.service - -[Service] -ExecStartPre=-/usr/bin/mkdir -p /var/lib/kubelet/device-plugins -EOF - - # Now we are off-piste: enable DCGM, DCGM exporter, container device plugin, and the NVIDIA containerd config. - systemctl enable nvidia-dcgm - systemctl enable nvidia-dcgm-exporter - systemctl enable nvidia-device-plugin - systemctl enable openibd - - # One additional request from MAI: signal that NPD is pre-installed on the VHD. - # When this file is present, the Azure AKS VM Extension skips installing NPD at provision time. - mkdir -p /etc/node-problem-detector.d/ - touch /etc/node-problem-detector.d/skip_vhd_npd - fi -fi +echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} +# TODO: uncomment once able +# cacheContainerImageComponents +# cacheGPUContainerImageComponents +capture_benchmark "${SCRIPT_NAME}_caching_container_images" +configureGraceBlackwell if [ -d "/opt/gpu" ] && [ "$(ls -A /opt/gpu)" ]; then ls -ltr /opt/gpu/* >> ${VHD_LOGS_FILEPATH} fi -installBpftrace -echo " - $(bpftrace --version)" >> ${VHD_LOGS_FILEPATH} - -PRESENT_DIR=$(pwd) -# run installBcc in a subshell and continue on with container image pull in order to decrease total build time -( - cd $PRESENT_DIR || { echo "Subshell in the wrong directory" >&2; exit 1; } - installBcc - exit $? -) > /var/log/bcc_installation.log 2>&1 & - -BCC_PID=$! - -# Add a separate section for runtime-installed components -# This clearly distinguishes components installed during CSE from VHD build-time components -# Only add for Ubuntu -if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then - cat << EOF >> ${VHD_LOGS_FILEPATH} -Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): - - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} -EOF -fi - -echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_pull_nvidia_driver_and_start_ebpf_downloads" - -# Limit number of parallel pulls to 2 less than number of processor cores in order to prevent issues with network, CPU, and disk resources -# Account for possibility that number of cores is 3 or less -num_proc=$(nproc) -if [ "$num_proc" -gt 3 ]; then - parallel_container_image_pull_limit=$(nproc --ignore=2) -else - parallel_container_image_pull_limit=1 -fi -echo "Limit for parallel container image pulls set to $parallel_container_image_pull_limit" - -declare -a image_pids=() - -# Cache container images declared within components.json -cacheContainerImageComponents -capture_benchmark "${SCRIPT_NAME}_caching_container_images" +# TODO: uncomment once able +# retagAKSNodeCAWatcher +capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" pinPodSandboxImages capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" @@ -1261,20 +1283,7 @@ if [ "$OS" = "$UBUNTU_OS_NAME" ]; then fi capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu" -wait $BCC_PID -BCC_EXIT_CODE=$? -chmod 644 /var/log/bcc_installation.log - -if [ "$BCC_EXIT_CODE" -eq 0 ]; then - echo "Bcc tools successfully installed." - cat << EOF >> ${VHD_LOGS_FILEPATH} - - bcc-tools - - libbcc-examples -EOF -else - echo "Error: installBcc subshell failed with exit code $BCC_EXIT_CODE" >&2 - exit $BCC_EXIT_CODE -fi +finisheBPFToolsInstallation capture_benchmark "${SCRIPT_NAME}_finish_installing_bcc_tools" configureLsmWithBpf @@ -1291,7 +1300,8 @@ if [ -n "${PRIVATE_PACKAGES_URL:-}" ]; then fi rm -f ./azcopy # cleanup immediately after usage will return in two downloads -extractAndCacheCoreDnsBinary +# TODO: uncomment once able +# extractAndCacheCoreDnsBinary collect_grid_compatibility_data From 15fb65caa16c099e569039cf1d089f6dd58ce08c Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Mon, 6 Jul 2026 15:23:27 -0700 Subject: [PATCH 12/78] chore: fix diff --- .pipelines/.vsts-vhd-builder-release.yaml | 26 ++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index c666513d17f..b90fd302c3c 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -786,7 +786,31 @@ stages: parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} - artifactName: 2204-minimal-gen2-containerd + artifactName: 2604-minimal-gen2-containerd + - job: build2604minimalarm64gen2containerd + condition: eq('${{ parameters.build2604minimalarm64gen2containerd }}', true) + dependsOn: [ ] + timeoutInMinutes: 360 + steps: + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' + echo '##vso[task.setvariable variable=OS_VERSION]26.04' + echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-26_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]minimal-arm64' + echo '##vso[task.setvariable variable=IMG_VERSION]latest' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' + echo '##vso[task.setvariable variable=ARCHITECTURE]ARM64' + echo '##vso[task.setvariable variable=ENABLE_FIPS]False' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' + displayName: Setup Build Variables + - template: ./templates/.builder-release-template.yaml + parameters: + useOverrides: ${{ parameters.useOverrides }} + overrideBranch: ${{ parameters.overrideBranch }} + artifactName: 2604-minimal-arm64-gen2-containerd - job: buildacltlgen2 condition: eq('${{ parameters.buildacltlgen2 }}', true) dependsOn: [ ] From 6cc28a63ca62f3065bd324c02220a6efaa14cb24 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Mon, 6 Jul 2026 15:29:13 -0700 Subject: [PATCH 13/78] chore: address feedback --- parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index b4e285d8757..ddb5f7417b0 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -49,7 +49,7 @@ installMinimalBuildDeps() { holdWALinuxAgent hold apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - pkg_list+=(rsyslog gpg) + local -a pkg_list+=(rsyslog gpg) # Batch install all packages in a single apt_get_install call instead of # looping one-by-one. On failure, fall back to individual installs for From 2f4236f97dd260f835d59e9e609c83b08722eb1d Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Mon, 6 Jul 2026 15:50:46 -0700 Subject: [PATCH 14/78] chore: debug --- vhdbuilder/packer/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 56c0230f717..20280f5436a 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1104,7 +1104,7 @@ fi # fi # TODO: confirm if this is right -updateAptWithMicrosoftPkg +# updateAptWithMicrosoftPkg installDeps CHRONYD_DIR=/etc/systemd/system/chronyd.service.d From cd1005d57688314b4a2a88e495aeaba72dbd06b9 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Mon, 6 Jul 2026 16:15:16 -0700 Subject: [PATCH 15/78] chore: use correct aznfs version --- parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index ddb5f7417b0..caae927bb58 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -93,8 +93,10 @@ installDeps() { pkg_list+=(irqbalance) fi - if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then + if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then pkg_list+=("aznfs=3.0.14") + elif [ "${OSVERSION}" = "26.04" ]; then + pkg_list+=("aznfs=3.0.18") fi # Batch install all packages in a single apt_get_install call instead of From 623faa719a028dc745bdbe80d29e2e71b54260db Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 09:32:57 -0700 Subject: [PATCH 16/78] chore: skip linux-modules-extra- --- .../cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index caae927bb58..fb649c1cbd7 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -82,7 +82,12 @@ installDeps() { holdWALinuxAgent hold apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - pkg_list=(apparmor-utils bind9-dnsutils ca-certificates ceph-common cgroup-lite cifs-utils conntrack cracklib-runtime ebtables ethtool glusterfs-client htop init-system-helpers inotify-tools iotop iproute2 ipset iptables nftables jq libpam-pwquality libpwquality-tools mount nfs-common pigz socat sysfsutils sysstat util-linux xz-utils netcat-openbsd zip rng-tools kmod gcc make dkms initramfs-tools linux-headers-$(uname -r) linux-modules-extra-$(uname -r)) + pkg_list=(apparmor-utils bind9-dnsutils ca-certificates ceph-common cgroup-lite cifs-utils conntrack cracklib-runtime ebtables ethtool glusterfs-client htop init-system-helpers inotify-tools iotop iproute2 ipset iptables nftables jq libpam-pwquality libpwquality-tools mount nfs-common pigz socat sysfsutils sysstat util-linux xz-utils netcat-openbsd zip rng-tools kmod gcc make dkms initramfs-tools linux-headers-$(uname -r)) + + if [ "${OSVERSION}" != "26.04" ]; then + # linux-modules-extra-* content is bundled into linux-modules-* on 26.04 (resolute) + pkg_list+=(linux-modules-extra-$(uname -r)) + fi local OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") while IFS= read -r fallback_pkg; do @@ -96,6 +101,7 @@ installDeps() { if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then pkg_list+=("aznfs=3.0.14") elif [ "${OSVERSION}" = "26.04" ]; then + # TODO(2604): confirm aznfs=3.0.18 is okay to use with Andy pkg_list+=("aznfs=3.0.18") fi From d35b7765dde15d592299d641b276e322c7abdb75 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 09:52:00 -0700 Subject: [PATCH 17/78] chore: fix --- parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index fb649c1cbd7..e234802b0b1 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -82,6 +82,9 @@ installDeps() { holdWALinuxAgent hold apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + pkg_list=(apparmor-utils bind9-dnsutils ca-certificates ceph-common cgroup-lite cifs-utils conntrack cracklib-runtime ebtables ethtool glusterfs-client htop init-system-helpers inotify-tools iotop iproute2 ipset iptables nftables jq libpam-pwquality libpwquality-tools mount nfs-common pigz socat sysfsutils sysstat util-linux xz-utils netcat-openbsd zip rng-tools kmod gcc make dkms initramfs-tools linux-headers-$(uname -r)) if [ "${OSVERSION}" != "26.04" ]; then @@ -89,7 +92,6 @@ installDeps() { pkg_list+=(linux-modules-extra-$(uname -r)) fi - local OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") while IFS= read -r fallback_pkg; do [ -n "${fallback_pkg}" ] && pkg_list+=("${fallback_pkg}") done < <(blobfuseFallbackPackages "${OSVERSION}") From ecf21af0fe174201c48bb31c0bb4b164534ae9b2 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 10:15:32 -0700 Subject: [PATCH 18/78] chore: add nvidia repo support --- .../artifacts/ubuntu/cse_install_ubuntu.sh | 8 +++++++- vhdbuilder/packer/install-dependencies.sh | 12 ++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index e234802b0b1..005e5e04721 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -197,6 +197,8 @@ updateAptWithNvidiaPkg() { nvidia_ubuntu_release="ubuntu2204" elif [ "${UBUNTU_RELEASE}" = "24.04" ]; then nvidia_ubuntu_release="ubuntu2404" + elif [ "${UBUNTU_RELEASE}" = "26.04" ]; then + nvidia_ubuntu_release="ubuntu2604" else echo "NVIDIA repo setup is not supported on Ubuntu ${UBUNTU_RELEASE}" return @@ -206,7 +208,11 @@ updateAptWithNvidiaPkg() { echo "deb [arch=${cpu_arch} signed-by=${nvidia_gpg_keyring_path}] https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch} /" > ${nvidia_sources_list_path} # Add NVIDIA repository - local nvidia_gpg_key_url="https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch}/3bf863cc.pub" + local nvidia_gpg_key_name="3bf863cc.pub" + if [ "${UBUNTU_RELEASE}" = "26.04" ]; then + nvidia_gpg_key_name="60DF8A40.pub" + fi + local nvidia_gpg_key_url="https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch}/${nvidia_gpg_key_name}" # Download and add the GPG key for the NVIDIA repository retrycmd_curl_file 120 5 25 ${nvidia_gpg_keyring_path} ${nvidia_gpg_key_url} 300 || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 20280f5436a..6c78d2651bc 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1103,7 +1103,7 @@ fi # updateAptWithMicrosoftPkg # fi -# TODO: confirm if this is right +# TODO(2604): confirm if this is right # updateAptWithMicrosoftPkg installDeps @@ -1212,7 +1212,7 @@ capture_benchmark "${SCRIPT_NAME}_handle_os_specific_configurations" echo "VHD will be built with containerd as the container runtime" # Cache packages and binaries declared within components.json -# TODO: uncomment once able +# TODO(2604): uncomment once able # cachePackageAndBinaryComponents starteBPFToolsInstallation @@ -1220,9 +1220,9 @@ capture_benchmark "${SCRIPT_NAME}_start_install_ebpf_tools" # Cache container images declared within components.json # k8s will use images in the k8s.io namespaces - create it -ctr namespace create k8s.io +# TODO(2604): uncomment all below once able +# ctr namespace create k8s.io echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} -# TODO: uncomment once able # cacheContainerImageComponents # cacheGPUContainerImageComponents capture_benchmark "${SCRIPT_NAME}_caching_container_images" @@ -1232,7 +1232,7 @@ if [ -d "/opt/gpu" ] && [ "$(ls -A /opt/gpu)" ]; then ls -ltr /opt/gpu/* >> ${VHD_LOGS_FILEPATH} fi -# TODO: uncomment once able +# TODO(2604): uncomment once able # retagAKSNodeCAWatcher capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" @@ -1300,7 +1300,7 @@ if [ -n "${PRIVATE_PACKAGES_URL:-}" ]; then fi rm -f ./azcopy # cleanup immediately after usage will return in two downloads -# TODO: uncomment once able +# TODO(2604): uncomment once able # extractAndCacheCoreDnsBinary collect_grid_compatibility_data From 6d285ff97138f36272b7647d157de0cac746b6c9 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 10:43:40 -0700 Subject: [PATCH 19/78] chore: debug --- parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 4 ++-- vhdbuilder/packer/install-dependencies.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 005e5e04721..b960ba94e68 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -197,8 +197,8 @@ updateAptWithNvidiaPkg() { nvidia_ubuntu_release="ubuntu2204" elif [ "${UBUNTU_RELEASE}" = "24.04" ]; then nvidia_ubuntu_release="ubuntu2404" - elif [ "${UBUNTU_RELEASE}" = "26.04" ]; then - nvidia_ubuntu_release="ubuntu2604" + # elif [ "${UBUNTU_RELEASE}" = "26.04" ]; then + # nvidia_ubuntu_release="ubuntu2604" else echo "NVIDIA repo setup is not supported on Ubuntu ${UBUNTU_RELEASE}" return diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 6c78d2651bc..597e1605f2e 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1236,7 +1236,8 @@ fi # retagAKSNodeCAWatcher capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" -pinPodSandboxImages +# TODO(2604): uncomment once able +# pinPodSandboxImages capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" # IPv6 nftables rules are only available on Ubuntu or Mariner/AzureLinux From 8dd80e46a99c79af3757002f9dce3ebf336f5b65 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 11:41:14 -0700 Subject: [PATCH 20/78] chore: bcc fixes --- .../cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 4 ++-- vhdbuilder/packer/install-dependencies.sh | 11 +++++++---- .../scripts/linux/ubuntu/tool_installs_ubuntu.sh | 2 ++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index b960ba94e68..005e5e04721 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -197,8 +197,8 @@ updateAptWithNvidiaPkg() { nvidia_ubuntu_release="ubuntu2204" elif [ "${UBUNTU_RELEASE}" = "24.04" ]; then nvidia_ubuntu_release="ubuntu2404" - # elif [ "${UBUNTU_RELEASE}" = "26.04" ]; then - # nvidia_ubuntu_release="ubuntu2604" + elif [ "${UBUNTU_RELEASE}" = "26.04" ]; then + nvidia_ubuntu_release="ubuntu2604" else echo "NVIDIA repo setup is not supported on Ubuntu ${UBUNTU_RELEASE}" return diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 597e1605f2e..7098b78f2e3 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -345,9 +345,9 @@ starteBPFToolsInstallation() { } finisheBPFToolsInstallation() { - wait $BCC_PID - BCC_EXIT_CODE=$? - chmod 644 /var/log/bcc_installation.log + local BCC_EXIT_CODE=0 + wait "$BCC_PID" || BCC_EXIT_CODE=$? + chmod 644 /var/log/bcc_installation.log || true if [ "$BCC_EXIT_CODE" -eq 0 ]; then echo "Bcc tools successfully installed." @@ -357,7 +357,10 @@ finisheBPFToolsInstallation() { EOF else echo "Error: installBcc subshell failed with exit code $BCC_EXIT_CODE" >&2 - exit $BCC_EXIT_CODE + echo "===== BEGIN /var/log/bcc_installation.log =====" >&2 + cat /var/log/bcc_installation.log >&2 2>/dev/null || echo "(bcc_installation.log missing)" >&2 + echo "===== END /var/log/bcc_installation.log =====" >&2 + exit "$BCC_EXIT_CODE" fi } diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 807d67bae41..05eb39be31f 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -29,6 +29,8 @@ installBcc() { VERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT + elif [ "${VERSION}" = "26.04" ]; then + apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev python zlib1g-dev libelf-dev python3-distutils libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi From 2444539ba240806d19efe519348370628df4eaf8 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 12:32:48 -0700 Subject: [PATCH 21/78] chore: add missing packages for bcc install --- vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 05eb39be31f..88ef607de47 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -30,7 +30,7 @@ installBcc() { if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT elif [ "${VERSION}" = "26.04" ]; then - apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm22 llvm-22-dev libclang-22-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev python zlib1g-dev libelf-dev python3-distutils libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi From 23f68b4e2b4415bc0996beb99ee4b7018c1666e2 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 12:46:37 -0700 Subject: [PATCH 22/78] chore: purge after bcc install --- vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 88ef607de47..35f439c7376 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -66,6 +66,8 @@ installBcc() { # only ensuring they are installed above. if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_purge 120 5 300 bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT + elif [ "${VERSION}" = "26.04" ]; then + apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm22 llvm-22-dev libclang-22-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi From 2ad1ba493754633ab288238d992d77871ab90457 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 13:12:40 -0700 Subject: [PATCH 23/78] chore: handle nvidia repo gpg --- .../artifacts/ubuntu/cse_install_ubuntu.sh | 19 +++++++++++++++---- .../linux/ubuntu/tool_installs_ubuntu.sh | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 005e5e04721..01cc1385b9a 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -176,7 +176,7 @@ updatePMCRepository() { } updateAptWithNvidiaPkg() { - readonly nvidia_gpg_keyring_path="/etc/apt/keyrings/nvidia.pub" + readonly nvidia_gpg_keyring_path="/etc/apt/keyrings/nvidia.gpg" mkdir -p "$(dirname "${nvidia_gpg_keyring_path}")" readonly nvidia_sources_list_path="/etc/apt/sources.list.d/nvidia.list" @@ -214,8 +214,15 @@ updateAptWithNvidiaPkg() { fi local nvidia_gpg_key_url="https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch}/${nvidia_gpg_key_name}" - # Download and add the GPG key for the NVIDIA repository - retrycmd_curl_file 120 5 25 ${nvidia_gpg_keyring_path} ${nvidia_gpg_key_url} 300 || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + # Download the armored NVIDIA repo key and dearmor it into a binary keyring. + # apt only accepts a signed-by keyring with a .gpg (binary) or .asc (armored) extension; + # NVIDIA publishes an ASCII-armored *.pub, so a raw .pub file is rejected as an "unsupported + # filetype" and the key is ignored (the repo then fails to verify with NO_PUBKEY). Newer apt + # (e.g. 3.x on Ubuntu 26.04) enforces this strictly, so dearmor to nvidia.gpg. + local nvidia_gpg_key_tmp="/tmp/${nvidia_gpg_key_name}" + retrycmd_curl_file 120 5 25 "${nvidia_gpg_key_tmp}" "${nvidia_gpg_key_url}" 300 || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + gpg --dearmor < "${nvidia_gpg_key_tmp}" > "${nvidia_gpg_keyring_path}" || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + rm -f "${nvidia_gpg_key_tmp}" apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT } @@ -267,9 +274,13 @@ removeNvidiaRepos() { rm -f /etc/apt/sources.list.d/nvidia.list echo "Removed NVIDIA apt repository" fi + if [ -f /etc/apt/keyrings/nvidia.gpg ]; then + rm -f /etc/apt/keyrings/nvidia.gpg + echo "Removed NVIDIA GPG key (nvidia.gpg)" + fi if [ -f /etc/apt/keyrings/nvidia.pub ]; then rm -f /etc/apt/keyrings/nvidia.pub - echo "Removed NVIDIA GPG key" + echo "Removed NVIDIA GPG key (nvidia.pub)" fi } diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 35f439c7376..77cb047b083 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -61,6 +61,7 @@ installBcc() { sudo make install || exit 1 popd popd + # we explicitly do not remove build-essential or python # these are standard packages we want to keep, they should usually be in the final build anyway. # only ensuring they are installed above. From 7bb7b430eb73cbbe648ea548b08759d17e5ae726 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 13:31:54 -0700 Subject: [PATCH 24/78] chore: enable LSM configuration with BPF --- vhdbuilder/packer/install-dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 7098b78f2e3..473b70b8a24 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -334,7 +334,7 @@ starteBPFToolsInstallation() { echo " - $(bpftrace --version)" >> ${VHD_LOGS_FILEPATH} PRESENT_DIR=$(pwd) - # run installBcc in a subshell and continue on with container image pull in order to decrease total build time + # run installBcc in a subshell asynchronously so we can do other things in the meantime ( cd $PRESENT_DIR || { echo "Subshell in the wrong directory" >&2; exit 1; } installBcc @@ -393,7 +393,7 @@ configureLsmWithBpf() { local new_lsm="bpf,$current_lsm" echo "New LSM configuration: $new_lsm" - if [ "$OS" = "$UBUNTU_OS_NAME" ] && [ "$OS_VERSION" = "24.04" ]; then + if [ "$OS" = "$UBUNTU_OS_NAME" ] && { [ "$OS_VERSION" = "24.04" ] || [ "$OS_VERSION" = "26.04" ]; }; then local grub_cfg="/etc/default/grub.d/50-cloudimg-settings.cfg" if [ -f "$grub_cfg" ]; then if grep -q "lsm=" "$grub_cfg"; then From ce500a3535408aac1b9ec754d4518f31f3643d23 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 13:47:58 -0700 Subject: [PATCH 25/78] chore: cleanup trivy scanning --- .../linux/cloud-init/artifacts/cse_config.sh | 2 +- vhdbuilder/packer/packer_source.sh | 2 +- vhdbuilder/packer/trivy-scan.sh | 63 +++++++++++-------- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 5c659f7c966..517fead4b93 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -321,7 +321,7 @@ disableSystemdResolved() { ls -ltr /etc/resolv.conf cat /etc/resolv.conf UBUNTU_RELEASE=$(lsb_release -r -s 2>/dev/null || echo "") - if [ "${UBUNTU_RELEASE}" = "20.04" ] || [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ]; then + if [ "${UBUNTU_RELEASE}" = "20.04" ] || [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then echo "Ingoring systemd-resolved query service but using its resolv.conf file" echo "This is the simplest approach to workaround resolved issues without completely uninstall it" [ -f /run/systemd/resolve/resolv.conf ] && ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index 77eec3c2277..2325e24bb9c 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -409,7 +409,7 @@ copyPackerFiles() { cpAndMode $AKS_CHECK_NETWORK_SCRIPT_SRC $AKS_CHECK_NETWORK_SCRIPT_DEST 755 cpAndMode $AKS_CHECK_NETWORK_SERVICE_SRC $AKS_CHECK_NETWORK_SERVICE_DEST 644 - if [ ${UBUNTU_RELEASE} = "22.04" ] || [ ${UBUNTU_RELEASE} = "24.04" ]; then + if [ ${UBUNTU_RELEASE} = "22.04" ] || [ ${UBUNTU_RELEASE} = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then PAM_D_COMMON_AUTH_SRC=/home/packer/pam-d-common-auth-2204 fi diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index d6d148ba4a9..3ee38935056 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -83,33 +83,19 @@ install_azure_cli() { ARCHITECTURE=${3} TEST_VM_ADMIN_USERNAME=${4} - if [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "22.04" ] && [ "${ARCHITECTURE,,}" = "arm64" ]; then - apt_get_update - apt_get_install 5 1 60 python3-pip - pip install azure-cli - export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" - CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z $CHECKAZ ]; then - echo "Azure CLI is not installed properly." - exit 1 - fi - elif [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "24.04" ] && [ "${ARCHITECTURE,,}" = "arm64" ]; then - apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - apt_get_update - apt_get_install 5 1 60 azure-cli - elif [ "$OS_SKU" = "Ubuntu" ] && { [ "$OS_VERSION" = "20.04" ] || [ "$OS_VERSION" = "22.04" ] || [ "$OS_VERSION" = "24.04" ]; } && [ "${ARCHITECTURE,,}" != "arm64" ]; then - apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - apt_get_update - apt_get_install 5 1 60 azure-cli - elif [ "$OS_SKU" = "CBLMariner" ] || [ "$OS_SKU" = "AzureLinux" ]; then + if [ "$OS_SKU" != "Ubuntu" ] && [ "$OS_SKU" != "CBLMariner" ] && [ "$OS_SKU" != "AzureLinux" ] && [ "$OS_SKU" != "Flatcar" ] && [ "$OS_SKU" != "AzureContainerLinux" ] && [ "$OS_SKU" != "AzureLinuxOSGuard" ]; then + echo "Unrecognized SKU, Version, and Architecture combination for downloading az: $OS_SKU $OS_VERSION $ARCHITECTURE" + exit 1 + fi + + if [ "$OS_SKU" = "CBLMariner" ] || [ "$OS_SKU" = "AzureLinux" ]; then sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc sudo sh -c 'echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo' sudo dnf install -y azure-cli - elif [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ]; then + return 0 + fi + + if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ]; then python3 -m venv "/home/$TEST_VM_ADMIN_USERNAME/venv" export PATH="/home/$TEST_VM_ADMIN_USERNAME/venv/bin:$PATH" pip install azure-cli @@ -118,9 +104,34 @@ install_azure_cli() { echo "Azure CLI is not installed properly." exit 1 fi + return 0 + fi + + if [ "${ARCHITECTURE,,}" = "arm64" ]; then + if [ "$OS_VERSION" = "22.04" ]; then + apt_get_update + apt_get_install 5 1 60 python3-pip + pip install azure-cli + export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" + CHECKAZ=$(pip freeze | grep "azure-cli==") + if [ -z $CHECKAZ ]; then + echo "Azure CLI is not installed properly." + exit 1 + fi + fi + if [ "$OS_VERSION" = "24.04" ] || [ "$OS_VERSION" = "26.04" ]; then + apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg + curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + apt_get_update + apt_get_install 5 1 60 azure-cli + fi else - echo "Unrecognized SKU, Version, and Architecture combination for downloading az: $OS_SKU $OS_VERSION $ARCHITECTURE" - exit 1 + apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg + curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + apt_get_update + apt_get_install 5 1 60 azure-cli fi } From f390a4fb0f400dbdc1870037e919b86c9bcb5459 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 13:49:05 -0700 Subject: [PATCH 26/78] chore: typo --- vhdbuilder/packer/trivy-scan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 3ee38935056..6714ee69cb1 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -84,7 +84,7 @@ install_azure_cli() { TEST_VM_ADMIN_USERNAME=${4} if [ "$OS_SKU" != "Ubuntu" ] && [ "$OS_SKU" != "CBLMariner" ] && [ "$OS_SKU" != "AzureLinux" ] && [ "$OS_SKU" != "Flatcar" ] && [ "$OS_SKU" != "AzureContainerLinux" ] && [ "$OS_SKU" != "AzureLinuxOSGuard" ]; then - echo "Unrecognized SKU, Version, and Architecture combination for downloading az: $OS_SKU $OS_VERSION $ARCHITECTURE" + echo "Unrecognized OS SKU for downloading az: $OS_SKU $OS_VERSION $ARCHITECTURE" exit 1 fi From b90651ba9b01ee8705ea05a98a23f99d83e91613 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 15:28:50 -0700 Subject: [PATCH 27/78] chore: bump bcc --- vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 77cb047b083..80c4ab43b4f 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -45,12 +45,18 @@ installBcc() { apt_get_install 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi + local bcc_version="v0.29.0" + if [ "${VERSION}" = "26.04" ]; then + # 26.04 requires a newer bcc version due to it baking CMake v4, otherwise CMake will fail with: "Compatibility with CMake < 3.5 has been removed from CMake" + bcc_version="v0.36.1" + fi + mkdir -p /tmp/bcc pushd /tmp/bcc git clone https://github.com/iovisor/bcc.git mkdir bcc/build; cd bcc/build - git checkout v0.29.0 + git checkout ${bcc_version} cmake -DENABLE_EXAMPLES=off .. || exit 1 make From 3acc692d88ff1542e1006d4f875481a20ddf58cf Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 15:30:51 -0700 Subject: [PATCH 28/78] chore: bcc v0.37.0 --- vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 80c4ab43b4f..fa8de99cb10 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -47,8 +47,8 @@ installBcc() { local bcc_version="v0.29.0" if [ "${VERSION}" = "26.04" ]; then - # 26.04 requires a newer bcc version due to it baking CMake v4, otherwise CMake will fail with: "Compatibility with CMake < 3.5 has been removed from CMake" - bcc_version="v0.36.1" + # 26.04 bakes CMake v4 and thus requires a newer bcc version, otherwise CMake will fail with: "Compatibility with CMake < 3.5 has been removed from CMake" + bcc_version="v0.37.0" fi mkdir -p /tmp/bcc From 26eb4883b3aff389193c60918cd8d0e6e15aaee7 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 15:56:46 -0700 Subject: [PATCH 29/78] chore: add libpolly-22-dev for bcc install --- .../scripts/linux/ubuntu/tool_installs_ubuntu.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index fa8de99cb10..3301fc5e0d9 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -30,7 +30,7 @@ installBcc() { if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT elif [ "${VERSION}" = "26.04" ]; then - apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm22 llvm-22-dev libclang-22-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm22 llvm-22-dev libclang-22-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev python zlib1g-dev libelf-dev python3-distutils libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi @@ -45,6 +45,11 @@ installBcc() { apt_get_install 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi + # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-22 that is downloaded for 26.04 + if [ "${VERSION}" = "26.04" ]; then + apt_get_install 120 5 300 libpolly-22-dev || exit $ERR_BCC_INSTALL_TIMEOUT + fi + local bcc_version="v0.29.0" if [ "${VERSION}" = "26.04" ]; then # 26.04 bakes CMake v4 and thus requires a newer bcc version, otherwise CMake will fail with: "Compatibility with CMake < 3.5 has been removed from CMake" @@ -74,6 +79,7 @@ installBcc() { if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_purge 120 5 300 bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT elif [ "${VERSION}" = "26.04" ]; then + # we remove git to keep the image as minimal as possible apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm22 llvm-22-dev libclang-22-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT @@ -84,6 +90,11 @@ installBcc() { apt_get_purge 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi + # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-22 that is downloaded for 26.04 + if [ "${VERSION}" = "26.04" ]; then + apt_get_install 120 5 300 libpolly-22-dev || exit $ERR_BCC_INSTALL_TIMEOUT + fi + rm -rf /tmp/bcc } From 34ac5aadb6482bc52855469e9af97198506d6768 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 15:59:23 -0700 Subject: [PATCH 30/78] chore: typo --- vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 3301fc5e0d9..bddd21536c0 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -92,7 +92,7 @@ installBcc() { # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-22 that is downloaded for 26.04 if [ "${VERSION}" = "26.04" ]; then - apt_get_install 120 5 300 libpolly-22-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_purge 120 5 300 libpolly-22-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi rm -rf /tmp/bcc From 39912576b194b14238449daaf36c095c98256722 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 16:22:25 -0700 Subject: [PATCH 31/78] chore: downgrade to llvm-18 --- .../scripts/linux/ubuntu/tool_installs_ubuntu.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index bddd21536c0..3d5ab7c57e2 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -30,7 +30,11 @@ installBcc() { if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT elif [ "${VERSION}" = "26.04" ]; then - apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm22 llvm-22-dev libclang-22-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT + # LLVM is pinned to 18 (not 26.04's default 22): BCC v0.37.0 links the individual clang + # static libs, and LLVM 21/22 reorganized clang's driver/options libraries, breaking that + # static link (undefined refs to clang::getDriverOptTable / GetResourcesPath / getLastArgIntValue). + # 18 is the newest major with an explicit clang-lib guard in BCC's CMakeLists and is in the 26.04 archive. + apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm18 llvm-18-dev libclang-18-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev python zlib1g-dev libelf-dev python3-distutils libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi @@ -45,9 +49,9 @@ installBcc() { apt_get_install 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi - # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-22 that is downloaded for 26.04 + # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-18 that is downloaded for 26.04 if [ "${VERSION}" = "26.04" ]; then - apt_get_install 120 5 300 libpolly-22-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_install 120 5 300 libpolly-18-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi local bcc_version="v0.29.0" @@ -80,7 +84,7 @@ installBcc() { apt_get_purge 120 5 300 bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT elif [ "${VERSION}" = "26.04" ]; then # we remove git to keep the image as minimal as possible - apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm22 llvm-22-dev libclang-22-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm18 llvm-18-dev libclang-18-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi @@ -90,9 +94,9 @@ installBcc() { apt_get_purge 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi - # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-22 that is downloaded for 26.04 + # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-18 that is downloaded for 26.04 if [ "${VERSION}" = "26.04" ]; then - apt_get_purge 120 5 300 libpolly-22-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_purge 120 5 300 libpolly-18-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi rm -rf /tmp/bcc From 1fa7fd284b175ffad1131579cece4623e465b07b Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 7 Jul 2026 16:43:22 -0700 Subject: [PATCH 32/78] chore: cleanup --- vhdbuilder/packer/post-install-dependencies.sh | 3 ++- .../scripts/linux/ubuntu/tool_installs_ubuntu.sh | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/vhdbuilder/packer/post-install-dependencies.sh b/vhdbuilder/packer/post-install-dependencies.sh index 2cf43f7ef6f..bdaff8dba78 100644 --- a/vhdbuilder/packer/post-install-dependencies.sh +++ b/vhdbuilder/packer/post-install-dependencies.sh @@ -71,7 +71,8 @@ fi # shellcheck disable=SC2129 echo "kubelet/kubectl downloaded:" >> ${VHD_LOGS_FILEPATH} -ls -ltr /opt/bin/kube* >> ${VHD_LOGS_FILEPATH} +# TODO(2604): uncomment when able +# ls -ltr /opt/bin/kube* >> ${VHD_LOGS_FILEPATH} # shellcheck disable=SC2010 ls -ltr /dev/* | grep sgx >> ${VHD_LOGS_FILEPATH} || true diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 3d5ab7c57e2..f8ea8124de9 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -30,11 +30,11 @@ installBcc() { if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT elif [ "${VERSION}" = "26.04" ]; then - # LLVM is pinned to 18 (not 26.04's default 22): BCC v0.37.0 links the individual clang + # LLVM is pinned to 20 (not 26.04's default 22): BCC v0.37.0 links the individual clang # static libs, and LLVM 21/22 reorganized clang's driver/options libraries, breaking that # static link (undefined refs to clang::getDriverOptTable / GetResourcesPath / getLastArgIntValue). - # 18 is the newest major with an explicit clang-lib guard in BCC's CMakeLists and is in the 26.04 archive. - apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm18 llvm-18-dev libclang-18-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT + # 20 is the newest major before that reorg and is in the 26.04 archive. + apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm20 llvm-20-dev libclang-20-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev python zlib1g-dev libelf-dev python3-distutils libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi @@ -49,9 +49,9 @@ installBcc() { apt_get_install 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi - # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-18 that is downloaded for 26.04 + # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-20 that is downloaded for 26.04 if [ "${VERSION}" = "26.04" ]; then - apt_get_install 120 5 300 libpolly-18-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_install 120 5 300 libpolly-20-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi local bcc_version="v0.29.0" @@ -84,7 +84,7 @@ installBcc() { apt_get_purge 120 5 300 bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT elif [ "${VERSION}" = "26.04" ]; then # we remove git to keep the image as minimal as possible - apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm18 llvm-18-dev libclang-18-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm20 llvm-20-dev libclang-20-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi @@ -94,9 +94,9 @@ installBcc() { apt_get_purge 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi - # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-18 that is downloaded for 26.04 + # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-20 that is downloaded for 26.04 if [ "${VERSION}" = "26.04" ]; then - apt_get_purge 120 5 300 libpolly-18-dev || exit $ERR_BCC_INSTALL_TIMEOUT + apt_get_purge 120 5 300 libpolly-20-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi rm -rf /tmp/bcc From 3da8f5c89d96774f27a6dcfb0af6627b3b594bc5 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 8 Jul 2026 09:09:19 -0700 Subject: [PATCH 33/78] chore: tweaks --- .pipelines/.vsts-vhd-builder-release.yaml | 4 +- .../linux/cloud-init/artifacts/cse_helpers.sh | 7 ++ .../artifacts/ubuntu/cse_install_ubuntu.sh | 89 +++++++++---------- vhdbuilder/packer/pre-install-dependencies.sh | 4 +- .../linux/ubuntu/tool_installs_ubuntu.sh | 46 +++++----- 5 files changed, 77 insertions(+), 73 deletions(-) diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index b90fd302c3c..40229daf23f 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -777,7 +777,7 @@ stages: echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]minimal' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' @@ -801,7 +801,7 @@ stages: echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v5' - echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]minimal' echo '##vso[task.setvariable variable=ARCHITECTURE]ARM64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 78c30a4f442..66431293133 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -969,6 +969,13 @@ isUbuntu() { return 1 } +isMinimalImage() { + if grep -q "minimal" <<< "$FEATURE_FLAGS"; then + return 0 + fi + return 1 +} + evalPackageDownloadURL() { local url=${1:-} if [ -n "$url" ]; then diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 01cc1385b9a..11d87df8668 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -4,6 +4,33 @@ removeContainerd() { apt_get_purge 10 5 300 moby-containerd } +# Batch install all packages in a single apt_get_install call instead of looping one-by-one. +# On failure, fall back to individual installs for diagnostic clarity. A return code of 2 from +# apt_get_install signals a CSE timeout and is propagated immediately by exiting the script. +aptGetBatchInstallPackagesWithFallback() { + local -a pkg_list=("$@") + + apt_get_install 30 1 600 "${pkg_list[@]}" + local batch_rc=$? + if [ "$batch_rc" -eq 2 ]; then + exit "$batch_rc" + elif [ "$batch_rc" -ne 0 ]; then + echo "Batch install failed, falling back to individual package install" + local apt_package + for apt_package in "${pkg_list[@]}"; do + apt_get_install 30 1 600 "$apt_package" + local pkg_rc=$? + if [ "$pkg_rc" -eq 2 ]; then + exit "$pkg_rc" + elif [ "$pkg_rc" -ne 0 ]; then + tail -n 200 /var/log/apt/term.log || true + tail -n 200 /var/log/dpkg.log || true + exit $ERR_APT_INSTALL_TIMEOUT + fi + done + fi +} + blobfuseFallbackPackages() { local OSVERSION="${1}" # blobfuse/blobfuse2 started to be centralized in components.json around April 2026. @@ -40,8 +67,21 @@ blobfuseFallbackPackages() { fi } -# Used to install dependencies within pre-install-dependencies.sh on Ubuntu minimal images (currently only 26.04) +# Installs any required dependencies needed to build the particular Ubuntu minimal image (currently only 26.04) installMinimalBuildDeps() { + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + + if [ "${OSVERSION}" = "26.04" ]; then + installUbuntu2604MinimalBuildDeps + return 0 + fi + + echo "Unrecognized Ubuntu minimal version ${OSVERSION} - cannot install minimal build dependencies" + exit 1 +} + +installUbuntu2604MinimalBuildDeps() { wait_for_apt_locks retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL @@ -49,29 +89,8 @@ installMinimalBuildDeps() { holdWALinuxAgent hold apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - local -a pkg_list+=(rsyslog gpg) - - # Batch install all packages in a single apt_get_install call instead of - # looping one-by-one. On failure, fall back to individual installs for - # diagnostic clarity. Exit immediately on return code 2 (CSE timeout). - apt_get_install 30 1 600 "${pkg_list[@]}" - local batch_rc=$? - if [ "$batch_rc" -eq 2 ]; then - exit "$batch_rc" - elif [ "$batch_rc" -ne 0 ]; then - echo "Batch install failed, falling back to individual package install" - for apt_package in "${pkg_list[@]}"; do - apt_get_install 30 1 600 "$apt_package" - local pkg_rc=$? - if [ "$pkg_rc" -eq 2 ]; then - exit "$pkg_rc" - elif [ "$pkg_rc" -ne 0 ]; then - tail -n 200 /var/log/apt/term.log || true - tail -n 200 /var/log/dpkg.log || true - exit $ERR_APT_INSTALL_TIMEOUT - fi - done - fi + local -a pkg_list=(rsyslog gpg) + aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" } installDeps() { @@ -107,27 +126,7 @@ installDeps() { pkg_list+=("aznfs=3.0.18") fi - # Batch install all packages in a single apt_get_install call instead of - # looping one-by-one. On failure, fall back to individual installs for - # diagnostic clarity. Exit immediately on return code 2 (CSE timeout). - apt_get_install 30 1 600 "${pkg_list[@]}" - local batch_rc=$? - if [ "$batch_rc" -eq 2 ]; then - exit "$batch_rc" - elif [ "$batch_rc" -ne 0 ]; then - echo "Batch install failed, falling back to individual package install" - for apt_package in "${pkg_list[@]}"; do - apt_get_install 30 1 600 "$apt_package" - local pkg_rc=$? - if [ "$pkg_rc" -eq 2 ]; then - exit "$pkg_rc" - elif [ "$pkg_rc" -ne 0 ]; then - tail -n 200 /var/log/apt/term.log || true - tail -n 200 /var/log/dpkg.log || true - exit $ERR_APT_INSTALL_TIMEOUT - fi - done - fi + aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then # disable aznfswatchdog since aznfs install and enable aznfswatchdog and aznfswatchdogv4 services at the same time while we only need aznfswatchdogv4 diff --git a/vhdbuilder/packer/pre-install-dependencies.sh b/vhdbuilder/packer/pre-install-dependencies.sh index 9809b582187..12b2eef6a44 100644 --- a/vhdbuilder/packer/pre-install-dependencies.sh +++ b/vhdbuilder/packer/pre-install-dependencies.sh @@ -36,8 +36,8 @@ capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" copyPackerFiles -# Install minimal build dependencies as needed (currently only needed on Ubuntu 26.04) -if [ "${UBUNTU_RELEASE}" = "26.04" ]; then +# Install required dependencies needed to build minimal images if needed (currently only Ubuntu 26.04) +if isMinimalImage && isUbuntu "$OS"; then installMinimalBuildDeps fi diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index f8ea8124de9..20f1723a152 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -1,23 +1,21 @@ #!/bin/bash -{{/* FIPS-related error codes */}} -ERR_UA_TOOLS_INSTALL_TIMEOUT=180 {{/* Timeout waiting for ubuntu-advantage-tools install */}} -ERR_ADD_UA_APT_REPO=181 {{/* Error to add UA apt repository */}} -ERR_UA_ATTACH=182 {{/* Error attaching UA */}} -ERR_UA_DISABLE_LIVEPATCH=183 {{/* Error to disable UA livepatch */}} -ERR_UA_ENABLE_FIPS=184 {{/* Error to enable UA FIPS */}} -ERR_UA_DETACH=185 {{/* Error to detach UA */}} -ERR_LINUX_HEADER_INSTALL_TIMEOUT=186 {{/* Timeout to install linux header */}} -ERR_STRONGSWAN_INSTALL_TIMEOUT=187 {{/* Timeout to install strongswan */}} -ERR_UA_ESM_HOOK_CLEANUP=188 {{/* Error removing the apt ESM hook for Ubuntu Pro */}} -ERR_UA_MASK_UNIT=189 {{/* Error stopping/disabling/masking an Ubuntu Pro background unit */}} -ERR_UA_TOKEN_CLEANUP=190 {{/* Error removing the baked-in Ubuntu Pro machine token state */}} - -ERR_NTP_INSTALL_TIMEOUT=10 {{/*Unable to install NTP */}} -ERR_NTP_START_TIMEOUT=11 {{/* Unable to start NTP */}} -ERR_STOP_OR_DISABLE_SYSTEMD_TIMESYNCD_TIMEOUT=12 {{/* Timeout waiting for systemd-timesyncd stop */}} -ERR_STOP_OR_DISABLE_NTP_TIMEOUT=13 {{/* Timeout waiting for ntp stop */}} -ERR_CHRONY_INSTALL_TIMEOUT=14 {{/*Unable to install CHRONY */}} -ERR_CHRONY_START_TIMEOUT=15 {{/* Unable to start CHRONY */}} +ERR_UA_TOOLS_INSTALL_TIMEOUT=180 # Timeout waiting for ubuntu-advantage-tools install +ERR_ADD_UA_APT_REPO=181 # Error to add UA apt repository +ERR_UA_ATTACH=182 # Error attaching UA +ERR_UA_DISABLE_LIVEPATCH=183 # Error to disable UA livepatch +ERR_UA_ENABLE_FIPS=184 # Error to enable UA FIPS +ERR_UA_DETACH=185 # Error to detach UA +ERR_LINUX_HEADER_INSTALL_TIMEOUT=186 # Timeout to install linux header +ERR_STRONGSWAN_INSTALL_TIMEOUT=187 # Timeout to install strongswan +ERR_UA_ESM_HOOK_CLEANUP=188 # Error removing the apt ESM hook for Ubuntu Pro +ERR_UA_MASK_UNIT=189 # Error stopping/disabling/masking an Ubuntu Pro background unit +ERR_UA_TOKEN_CLEANUP=190 # Error removing the baked-in Ubuntu Pro machine token state +ERR_NTP_INSTALL_TIMEOUT=10 # Unable to install NTP +ERR_NTP_START_TIMEOUT=11 # Unable to start NTP +ERR_STOP_OR_DISABLE_SYSTEMD_TIMESYNCD_TIMEOUT=12 # Timeout waiting for systemd-timesyncd stop +ERR_STOP_OR_DISABLE_NTP_TIMEOUT=13 # Timeout waiting for ntp stop +ERR_CHRONY_INSTALL_TIMEOUT=14 # Unable to install CHRONY +ERR_CHRONY_START_TIMEOUT=15 # Unable to start CHRONY echo "Sourcing tool_installs_ubuntu.sh" @@ -61,9 +59,9 @@ installBcc() { fi mkdir -p /tmp/bcc - pushd /tmp/bcc + pushd /tmp/bcc || exit 1 git clone https://github.com/iovisor/bcc.git - mkdir bcc/build; cd bcc/build + mkdir bcc/build; cd bcc/build || exit 1 git checkout ${bcc_version} @@ -71,11 +69,11 @@ installBcc() { make sudo make install || exit 1 cmake -DPYTHON_CMD=python3 .. || exit 1 # build python3 binding - pushd src/python/ + pushd src/python/ || exit 1 make sudo make install || exit 1 - popd - popd + popd || exit 1 + popd || exit 1 # we explicitly do not remove build-essential or python # these are standard packages we want to keep, they should usually be in the final build anyway. From d69de077b256e01990dd031536b36e4b623c542a Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 8 Jul 2026 09:16:02 -0700 Subject: [PATCH 34/78] chore: more tweaks --- .pipelines/templates/.builder-release-template.yaml | 2 +- parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh | 4 ++++ .../cloud-init/artifacts/azlosguard/cse_install_osguard.sh | 4 ++++ .../linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh | 4 ++++ .../linux/cloud-init/artifacts/mariner/cse_install_mariner.sh | 4 ++++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 5c8729c1a74..2afa5bf6608 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -75,7 +75,7 @@ steps: - bash: | SKU_NAME=${OS_VERSION} && \ - if [[ "${IMG_SKU}" == *"minimal"* ]]; then SKU_NAME="${SKU_NAME}minimal"; fi && \ + if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \ if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index 07d128156c3..c09c023d9a5 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -88,6 +88,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh b/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh index 9193afdfde3..0874dde7d3e 100644 --- a/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh +++ b/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh @@ -89,6 +89,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh b/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh index cf45aba65d3..fbdea58bf58 100755 --- a/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh +++ b/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh @@ -73,6 +73,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh index 5f316f93324..a58afef0bcb 100755 --- a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh +++ b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh @@ -666,6 +666,10 @@ cleanUpGPUDrivers() { done } +installMinimalBuildDeps() { + echo "installMinimalBuildDeps not implemented for mariner" +} + downloadContainerdFromVersion() { echo "downloadContainerdFromVersion not implemented for mariner" } From c8ea241b44e0d430acfd26065fee7807e093be42 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 8 Jul 2026 10:10:02 -0700 Subject: [PATCH 35/78] chore: fix logs --- vhdbuilder/packer/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 473b70b8a24..5d78c383eca 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -401,7 +401,7 @@ configureLsmWithBpf() { else sed -i "s/GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"lsm=$new_lsm /" "$grub_cfg" fi - echo "Updating GRUB configuration for Ubuntu 24.04..." + echo "Updating GRUB configuration for Ubuntu..." update-grub2 /boot/grub/grub.cfg || echo "Warning: Failed to update GRUB configuration" else echo "Warning: $grub_cfg not found, skipping LSM configuration" From 93afcf1f2e98825266e4a3d4e036c553d5cf984c Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 8 Jul 2026 10:25:17 -0700 Subject: [PATCH 36/78] chore: upload release notes --- .pipelines/templates/.builder-release-template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 2afa5bf6608..fb073410175 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -364,6 +364,7 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: PublishPipelineArtifact@0 + condition: always() displayName: Publish Release Notes inputs: artifactName: 'vhd-release-notes-${{ parameters.artifactName }}' From 3441fa736446ba665da7bfc8a2c68147328dd3e0 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 8 Jul 2026 13:07:23 -0700 Subject: [PATCH 37/78] chore: add components entries for 2604 --- parts/common/components.json | 134 +++++++++++++++++++++++++++++++++++ schemas/components.cue | 1 + 2 files changed, 135 insertions(+) diff --git a/parts/common/components.json b/parts/common/components.json index 1f1f28a7fab..ed70f208c8b 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -953,6 +953,14 @@ "windowsDownloadLocation": "c:\\akse-cache\\aks-secure-tls-bootstrap-client\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.1.4-ubuntu24.04u1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1055,6 +1063,14 @@ "latestVersion": "1.4.3-ubuntu24.04u1" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=moby-runc, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.5.0-ubuntu26.04u1" + } + ] } } } @@ -1065,6 +1081,14 @@ "windowsDownloadLocation": "c:\\akse-cache\\containerd\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=moby-containerd, repository=production, os=ubuntu, release=26.04", + "latestVersion": "2.3.2-ubuntu26.04u1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1254,6 +1278,14 @@ "latestVersion": "1.9.0-ubuntu24.04u1" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=containernetworking-plugins, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.9.1-ubuntu26.04u1" + } + ] } }, "azurelinux": { @@ -1357,6 +1389,14 @@ "latestVersion": "1.34.0-ubuntu24.04u2" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.0-ubuntu26.04u1" + } + ] } }, "mariner": { @@ -1415,6 +1455,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\kubelet\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.2-ubuntu26.04u1", + "previousLatestVersion": "1.36.1-ubuntu26.04u1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1513,6 +1563,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\kubectl\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.2-ubuntu26.04u1", + "previousLatestVersion": "1.36.1-ubuntu26.04u1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1715,6 +1775,14 @@ "downloadLocation": "/opt/azure/ig", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=ig, repository=production, os=ubuntu, release=26.04", + "latestVersion": "0.54.0-ubuntu26.04u1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1758,6 +1826,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\azure-acr-credential-provider\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.2-ubuntu26.04u1", + "previousLatestVersion": "1.36.1-ubuntu26.04u1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1962,6 +2040,14 @@ "downloadLocation": "/opt/nvidia-device-plugin/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=nvidia-device-plugin, repository=production, os=ubuntu, release=26.04", + "latestVersion": "0.19.3-ubuntu26.04u1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1996,6 +2082,14 @@ "downloadLocation": "/opt/dra-driver-nvidia-gpu/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=dra-driver-nvidia-gpu, repository=production, os=ubuntu, release=26.04", + "latestVersion": "0.4.1-ubuntu26.04u2" + } + ] + }, "r2404": { "versionsV2": [ { @@ -2030,6 +2124,14 @@ "downloadLocation": "/opt/datacenter-gpu-manager-4-core/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=26.04", + "latestVersion": "1:4.6.0-1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -2064,6 +2166,14 @@ "downloadLocation": "/opt/datacenter-gpu-manager-4-proprietary/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=26.04", + "latestVersion": "1:4.6.0-1" + } + ] + }, "r2404": { "versionsV2": [ { @@ -2098,6 +2208,14 @@ "downloadLocation": "/opt/dcgm-exporter/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=dcgm-exporter, repository=production, os=ubuntu, release=26.04", + "latestVersion": "4.8.2-ubuntu26.04u9" + } + ] + }, "r2404": { "versionsV2": [ { @@ -2132,6 +2250,14 @@ "downloadLocation": "/opt/node-exporter", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.9.1-ubuntu26.04u20" + } + ] + }, "r2404": { "versionsV2": [ { @@ -2358,6 +2484,14 @@ "latestVersion": "2.5.3" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=blobfuse2, repository=production, os=ubuntu, release=26.04", + "latestVersion": "2.5.3" + } + ] } } } diff --git a/schemas/components.cue b/schemas/components.cue index 442f736d96e..88cd1a8902a 100644 --- a/schemas/components.cue +++ b/schemas/components.cue @@ -54,6 +54,7 @@ package components r2004?: #ReleaseDownloadURI r2204?: #ReleaseDownloadURI r2404?: #ReleaseDownloadURI + r2604?: #ReleaseDownloadURI } #DefaultOSDistro: { From a6a3b2f47192a1124e0e4d85be0a52d2821f84b4 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 8 Jul 2026 13:08:55 -0700 Subject: [PATCH 38/78] chore: typo --- parts/common/components.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/common/components.json b/parts/common/components.json index ed70f208c8b..08e2ac3fc49 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -957,7 +957,7 @@ "versionsV2": [ { "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.1.4-ubuntu24.04u1" + "latestVersion": "1.1.4-ubuntu26.04u1" } ] }, From 53fd79562d307c871b4525fd037149c4e1b0b551 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 8 Jul 2026 13:28:38 -0700 Subject: [PATCH 39/78] chore: finalized pin to aznfs=3.0.18 --- parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 11d87df8668..91d39af67c9 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -122,7 +122,6 @@ installDeps() { if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then pkg_list+=("aznfs=3.0.14") elif [ "${OSVERSION}" = "26.04" ]; then - # TODO(2604): confirm aznfs=3.0.18 is okay to use with Andy pkg_list+=("aznfs=3.0.18") fi From 501107931b05b020d2fa5210d386da3412891517 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 8 Jul 2026 13:31:17 -0700 Subject: [PATCH 40/78] chore: download release notes after post-install-deps --- vhdbuilder/packer/vhd-image-builder-base.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index 1bb630a346e..f0796bef2c7 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -811,6 +811,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} /bin/bash -eux /home/packer/post-install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/opt/azure/vhd-install.complete", + "destination": "release-notes.txt" + }, { "type": "file", "source": "vhdbuilder/packer/list-images.sh", @@ -828,12 +834,6 @@ "source": "/opt/azure/containers/image-bom.json", "destination": "image-bom.json" }, - { - "type": "file", - "direction": "download", - "source": "/opt/azure/vhd-install.complete", - "destination": "release-notes.txt" - }, { "type": "file", "direction": "download", From 39935b54c221939dbd22eebe27676f608d38336d Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 9 Jul 2026 10:37:46 -0700 Subject: [PATCH 41/78] chore: uncomment unsupported func calls to test other VHDs --- vhdbuilder/packer/install-dependencies.sh | 14 +++++++------- vhdbuilder/packer/post-install-dependencies.sh | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 5d78c383eca..9377851d9a1 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1216,7 +1216,7 @@ echo "VHD will be built with containerd as the container runtime" # Cache packages and binaries declared within components.json # TODO(2604): uncomment once able -# cachePackageAndBinaryComponents +cachePackageAndBinaryComponents starteBPFToolsInstallation capture_benchmark "${SCRIPT_NAME}_start_install_ebpf_tools" @@ -1224,10 +1224,10 @@ capture_benchmark "${SCRIPT_NAME}_start_install_ebpf_tools" # Cache container images declared within components.json # k8s will use images in the k8s.io namespaces - create it # TODO(2604): uncomment all below once able -# ctr namespace create k8s.io +ctr namespace create k8s.io echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} -# cacheContainerImageComponents -# cacheGPUContainerImageComponents +cacheContainerImageComponents +cacheGPUContainerImageComponents capture_benchmark "${SCRIPT_NAME}_caching_container_images" configureGraceBlackwell @@ -1236,11 +1236,11 @@ if [ -d "/opt/gpu" ] && [ "$(ls -A /opt/gpu)" ]; then fi # TODO(2604): uncomment once able -# retagAKSNodeCAWatcher +retagAKSNodeCAWatcher capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" # TODO(2604): uncomment once able -# pinPodSandboxImages +pinPodSandboxImages capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" # IPv6 nftables rules are only available on Ubuntu or Mariner/AzureLinux @@ -1305,7 +1305,7 @@ fi rm -f ./azcopy # cleanup immediately after usage will return in two downloads # TODO(2604): uncomment once able -# extractAndCacheCoreDnsBinary +extractAndCacheCoreDnsBinary collect_grid_compatibility_data diff --git a/vhdbuilder/packer/post-install-dependencies.sh b/vhdbuilder/packer/post-install-dependencies.sh index bdaff8dba78..56903b6bf7d 100644 --- a/vhdbuilder/packer/post-install-dependencies.sh +++ b/vhdbuilder/packer/post-install-dependencies.sh @@ -72,7 +72,7 @@ fi # shellcheck disable=SC2129 echo "kubelet/kubectl downloaded:" >> ${VHD_LOGS_FILEPATH} # TODO(2604): uncomment when able -# ls -ltr /opt/bin/kube* >> ${VHD_LOGS_FILEPATH} +ls -ltr /opt/bin/kube* >> ${VHD_LOGS_FILEPATH} # shellcheck disable=SC2010 ls -ltr /dev/* | grep sgx >> ${VHD_LOGS_FILEPATH} || true From 2b5eb8e4541300f78b950c679b197e9926011046 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 9 Jul 2026 12:19:25 -0700 Subject: [PATCH 42/78] chore: reorder BCC + nvidia kernel module prebake to avoid running out of disk space --- vhdbuilder/packer/install-dependencies.sh | 33 +++++++++++++++-------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 9377851d9a1..ec3bec1b793 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -855,6 +855,22 @@ cacheGPUContainerImageComponents() { cat << EOF >> ${VHD_LOGS_FILEPATH} - nvidia-cuda-driver=${NVIDIA_DRIVER_IMAGE_TAG} EOF + fi + + # Add a separate section for runtime-installed components + # This clearly distinguishes components installed during CSE from VHD build-time components + # Only add for Ubuntu + if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then + cat << EOF >> ${VHD_LOGS_FILEPATH} +Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): + - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} +EOF + fi +} + +buildNVIDIAKernelModule() { + if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now + gpu_action="copy" # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles @@ -883,16 +899,6 @@ EOF EOF fi fi - - # Add a separate section for runtime-installed components - # This clearly distinguishes components installed during CSE from VHD build-time components - # Only add for Ubuntu - if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then - cat << EOF >> ${VHD_LOGS_FILEPATH} -Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): - - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} -EOF - fi } configureGraceBlackwell() { @@ -1288,7 +1294,12 @@ fi capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu" finisheBPFToolsInstallation -capture_benchmark "${SCRIPT_NAME}_finish_installing_bcc_tools" +capture_benchmark "${SCRIPT_NAME}_finish_installing_ebpf_tools" + +# Note: it seems that calling buildNVIDIAKernelModule while BCC is building/compiling in the background (starteBPFToolsInstallation) will cause +# the packer VM's disk to run out of space while building the kernel module - explicitly build the kernel module AFTER BCC build/compilation is finished. +buildNVIDIAKernelModule +capture_benchmark "${SCRIPT_NAME}_build_nvidia_kernel_module" configureLsmWithBpf capture_benchmark "${SCRIPT_NAME}_configure_lsm_with_bpf" From c66a2332fe4f1e1d76e6aee4c0c6b8cff6a8067f Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 9 Jul 2026 14:26:53 -0700 Subject: [PATCH 43/78] chore: more reordering --- vhdbuilder/packer/install-dependencies.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index ec3bec1b793..ae356135c6b 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1224,16 +1224,27 @@ echo "VHD will be built with containerd as the container runtime" # TODO(2604): uncomment once able cachePackageAndBinaryComponents +# k8s will use images in the k8s.io namespaces - create it +# TODO(2604): uncomment all below once able +ctr namespace create k8s.io + +# Fetch and pre-build the NVIDIA CUDA driver BEFORE starting the BCC background build and BEFORE caching +# the bulk container images. The driver's kernel-module compile and userspace lib install are disk-heavy; +# running them here (near-empty disk, BCC not yet started) avoids exhausting the 30GB packer build disk. +# Running them after the container-image cache and/or concurrently with the BCC build fills the disk +# (worse on 24.04), failing at the nvidia.ko link or the driver lib copy with "No space left on device". +cacheGPUContainerImageComponents +buildNVIDIAKernelModule +capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" + +# Start eBPF tool installation in the background while we pull container images in the foreground starteBPFToolsInstallation capture_benchmark "${SCRIPT_NAME}_start_install_ebpf_tools" # Cache container images declared within components.json -# k8s will use images in the k8s.io namespaces - create it # TODO(2604): uncomment all below once able -ctr namespace create k8s.io echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} cacheContainerImageComponents -cacheGPUContainerImageComponents capture_benchmark "${SCRIPT_NAME}_caching_container_images" configureGraceBlackwell @@ -1293,14 +1304,10 @@ if [ "$OS" = "$UBUNTU_OS_NAME" ]; then fi capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu" +# Ensure eBPF tools installed successfully finisheBPFToolsInstallation capture_benchmark "${SCRIPT_NAME}_finish_installing_ebpf_tools" -# Note: it seems that calling buildNVIDIAKernelModule while BCC is building/compiling in the background (starteBPFToolsInstallation) will cause -# the packer VM's disk to run out of space while building the kernel module - explicitly build the kernel module AFTER BCC build/compilation is finished. -buildNVIDIAKernelModule -capture_benchmark "${SCRIPT_NAME}_build_nvidia_kernel_module" - configureLsmWithBpf capture_benchmark "${SCRIPT_NAME}_configure_lsm_with_bpf" From b55a4e199bc71bb825a6e0caaf0673843c9c3362 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 10 Jul 2026 16:09:34 -0700 Subject: [PATCH 44/78] chore: add correct gpg key for 26.04 PMC repos --- .../cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 91d39af67c9..f380088f7ec 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -135,6 +135,9 @@ installDeps() { } updateAptWithMicrosoftPkg() { + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + retrycmd_silent 120 5 25 curl https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/prod.list > /tmp/microsoft-prod.list || exit $ERR_MOBY_APT_LIST_TIMEOUT retrycmd_if_failure 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit $ERR_MOBY_APT_LIST_TIMEOUT @@ -142,6 +145,13 @@ updateAptWithMicrosoftPkg() { retrycmd_silent 120 5 25 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT retrycmd_if_failure 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + + if [ "${OSVERSION}" = "26.04" ]; then + # Ubuntu 26.04 (Resolute) PMC repo is signed with Microsoft's newer 2025 gpg key + retrycmd_silent 120 5 25 curl https://packages.microsoft.com/keys/microsoft-2025.asc | gpg --dearmor > /tmp/microsoft-2025.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft-2025.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + fi + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT } From 2a082c30c4d55d916966cd4214f03d3d1fe9a6fa Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Tue, 14 Jul 2026 15:10:21 -0700 Subject: [PATCH 45/78] chore: skip ebpf installs --- vhdbuilder/packer/install-dependencies.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index ae356135c6b..bece7680b64 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -330,6 +330,11 @@ cacheKubePackageFromPrivateUrl() { } starteBPFToolsInstallation() { + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - skipping eBPF tools installation" + return 0 + fi + installBpftrace echo " - $(bpftrace --version)" >> ${VHD_LOGS_FILEPATH} @@ -345,6 +350,11 @@ starteBPFToolsInstallation() { } finisheBPFToolsInstallation() { + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - skipping eBPF tools installation" + return 0 + fi + local BCC_EXIT_CODE=0 wait "$BCC_PID" || BCC_EXIT_CODE=$? chmod 644 /var/log/bcc_installation.log || true From 7c0836cb9576aa3ee7a3906fe15fd18297c8e9e2 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 09:07:53 -0700 Subject: [PATCH 46/78] chore: skip unpublished packages temporarily --- vhdbuilder/packer/install-dependencies.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index bece7680b64..cb68d5ba414 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -589,6 +589,10 @@ cachePackageAndBinaryComponents() { done ;; "aks-secure-tls-bootstrap-client") + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - temporarily skipping aks-secure-tls-bootstrap-client installation" + return 0 + fi for version in ${PACKAGE_VERSIONS[@]}; do # removed at provisioning time if secure TLS bootstrapping is disabled if isUbuntu; then @@ -694,6 +698,10 @@ cachePackageAndBinaryComponents() { done ;; "dcgm-exporter") + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - temporarily skipping dcgm-exporter installation" + return 0 + fi for version in ${PACKAGE_VERSIONS[@]}; do downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" echo " - dcgm-exporter version ${version}" >> ${VHD_LOGS_FILEPATH} @@ -711,6 +719,10 @@ cachePackageAndBinaryComponents() { fi ;; "acr-mirror") + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - temporarily skipping acr-mirror installation" + return 0 + fi # Artifact streaming (acr-mirror) - version and URLs resolved from components.json, # OS filtering handled declaratively by components.json entries ( for unsupported OSes). for version in ${PACKAGE_VERSIONS[@]}; do @@ -730,6 +742,10 @@ cachePackageAndBinaryComponents() { installAznfsPackage || exit $ERR_AZNFS_INSTALL_FAIL ;; "blobfuse"|"blobfuse2") + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - temporarily skipping blobfuse2 installation" + return 0 + fi for version in "${PACKAGE_VERSIONS[@]}"; do if isUbuntu "$OS"; then if ! apt_get_install 10 2 120 "${name}=${version}"; then From 9052aed04551951c3c0fc23510a2458d5df7754b Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 09:25:27 -0700 Subject: [PATCH 47/78] chore: correct package versions --- parts/common/components.json | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index 8d5e5c57b25..c6dccfed21d 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1068,7 +1068,7 @@ "versionsV2": [ { "renovateTag": "name=moby-runc, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.5.0-ubuntu26.04u1" + "latestVersion": "1.4.3-ubuntu26.04u2" } ] } @@ -1085,7 +1085,7 @@ "versionsV2": [ { "renovateTag": "name=moby-containerd, repository=production, os=ubuntu, release=26.04", - "latestVersion": "2.3.2-ubuntu26.04u1" + "latestVersion": "2.3.2-ubuntu26.04u2" } ] }, @@ -1283,7 +1283,7 @@ "versionsV2": [ { "renovateTag": "name=containernetworking-plugins, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.9.1-ubuntu26.04u1" + "latestVersion": "1.9.0-ubuntu26.04u3" } ] } @@ -1394,7 +1394,7 @@ "versionsV2": [ { "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.0-ubuntu26.04u1" + "latestVersion": "1.34.0-ubuntu26.04u5" } ] } @@ -1460,8 +1460,7 @@ { "k8sVersion": "1.36", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.2-ubuntu26.04u1", - "previousLatestVersion": "1.36.1-ubuntu26.04u1" + "latestVersion": "1.36.2-ubuntu26.04u2" } ] }, @@ -1568,8 +1567,8 @@ { "k8sVersion": "1.36", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.2-ubuntu26.04u1", - "previousLatestVersion": "1.36.1-ubuntu26.04u1" + "latestVersion": "1.36.2-ubuntu26.04u5", + "previousLatestVersion": "1.36.1-ubuntu26.04u9" } ] }, @@ -1779,7 +1778,7 @@ "versionsV2": [ { "renovateTag": "name=ig, repository=production, os=ubuntu, release=26.04", - "latestVersion": "0.54.0-ubuntu26.04u1" + "latestVersion": "0.53.2-ubuntu26.04u4" } ] }, @@ -1831,8 +1830,8 @@ { "k8sVersion": "1.36", "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.2-ubuntu26.04u1", - "previousLatestVersion": "1.36.1-ubuntu26.04u1" + "latestVersion": "1.36.3-ubuntu26.04u2", + "previousLatestVersion": "1.36.2-ubuntu26.04u2" } ] }, @@ -2044,7 +2043,7 @@ "versionsV2": [ { "renovateTag": "name=nvidia-device-plugin, repository=production, os=ubuntu, release=26.04", - "latestVersion": "0.19.3-ubuntu26.04u1" + "latestVersion": "0.19.3-ubuntu26.04u6" } ] }, @@ -2254,7 +2253,7 @@ "versionsV2": [ { "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.9.1-ubuntu26.04u20" + "latestVersion": "1.9.1-ubuntu26.04u21" } ] }, From 417863be99b4e584d93e617ec9d1c64ca7fd9ca1 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 09:31:36 -0700 Subject: [PATCH 48/78] chore: correct package version for dra-driver-nvidia-gpu --- parts/common/components.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/common/components.json b/parts/common/components.json index 8248c6c61be..e3c24f6c8d2 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2085,7 +2085,7 @@ "versionsV2": [ { "renovateTag": "name=dra-driver-nvidia-gpu, repository=production, os=ubuntu, release=26.04", - "latestVersion": "0.4.1-ubuntu26.04u2" + "latestVersion": "0.4.1-ubuntu26.04u5" } ] }, From ad02eb3c74804e3f2f821ed653f54c44f56adccd Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 09:40:41 -0700 Subject: [PATCH 49/78] chore: fix --- vhdbuilder/packer/install-dependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index cb68d5ba414..646c3df50cf 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -591,7 +591,7 @@ cachePackageAndBinaryComponents() { "aks-secure-tls-bootstrap-client") if [ "$OS_VERSION" = "26.04" ]; then echo "Ubuntu 26.04 - temporarily skipping aks-secure-tls-bootstrap-client installation" - return 0 + continue fi for version in ${PACKAGE_VERSIONS[@]}; do # removed at provisioning time if secure TLS bootstrapping is disabled @@ -700,7 +700,7 @@ cachePackageAndBinaryComponents() { "dcgm-exporter") if [ "$OS_VERSION" = "26.04" ]; then echo "Ubuntu 26.04 - temporarily skipping dcgm-exporter installation" - return 0 + continue fi for version in ${PACKAGE_VERSIONS[@]}; do downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" @@ -721,7 +721,7 @@ cachePackageAndBinaryComponents() { "acr-mirror") if [ "$OS_VERSION" = "26.04" ]; then echo "Ubuntu 26.04 - temporarily skipping acr-mirror installation" - return 0 + continue fi # Artifact streaming (acr-mirror) - version and URLs resolved from components.json, # OS filtering handled declaratively by components.json entries ( for unsupported OSes). @@ -744,7 +744,7 @@ cachePackageAndBinaryComponents() { "blobfuse"|"blobfuse2") if [ "$OS_VERSION" = "26.04" ]; then echo "Ubuntu 26.04 - temporarily skipping blobfuse2 installation" - return 0 + continue fi for version in "${PACKAGE_VERSIONS[@]}"; do if isUbuntu "$OS"; then From 3f3932c62377d3f6e3a3f42179b113ab6d418d9a Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 10:23:02 -0700 Subject: [PATCH 50/78] chore: revert packer template --- vhdbuilder/packer/vhd-image-builder-base.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index f0796bef2c7..1bb630a346e 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -811,12 +811,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} /bin/bash -eux /home/packer/post-install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/opt/azure/vhd-install.complete", - "destination": "release-notes.txt" - }, { "type": "file", "source": "vhdbuilder/packer/list-images.sh", @@ -834,6 +828,12 @@ "source": "/opt/azure/containers/image-bom.json", "destination": "image-bom.json" }, + { + "type": "file", + "direction": "download", + "source": "/opt/azure/vhd-install.complete", + "destination": "release-notes.txt" + }, { "type": "file", "direction": "download", From 6e2e083889e1c189ce5278befd44b0fa99d1c035 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 10:30:37 -0700 Subject: [PATCH 51/78] chore: don't download bcc tools logs --- vhdbuilder/packer/vhd-image-builder-acl-arm64.json | 12 ------------ vhdbuilder/packer/vhd-image-builder-acl.json | 12 ------------ vhdbuilder/packer/vhd-image-builder-arm64-gb.json | 10 ---------- vhdbuilder/packer/vhd-image-builder-arm64-gen2.json | 12 ------------ vhdbuilder/packer/vhd-image-builder-base.json | 12 ------------ vhdbuilder/packer/vhd-image-builder-cvm.json | 12 ------------ .../packer/vhd-image-builder-mariner-arm64.json | 12 ------------ vhdbuilder/packer/vhd-image-builder-mariner-cvm.json | 12 ------------ vhdbuilder/packer/vhd-image-builder-mariner.json | 12 ------------ 9 files changed, 106 deletions(-) diff --git a/vhdbuilder/packer/vhd-image-builder-acl-arm64.json b/vhdbuilder/packer/vhd-image-builder-acl-arm64.json index e212a973da5..8993f7649c6 100644 --- a/vhdbuilder/packer/vhd-image-builder-acl-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-acl-arm64.json @@ -726,12 +726,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, { "type": "shell", "inline": [ @@ -744,12 +738,6 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, - { - "type": "shell", - "inline": [ - "sudo rm /var/log/bcc_installation.log" - ] - }, { "type": "shell", "inline": ": reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-acl.json b/vhdbuilder/packer/vhd-image-builder-acl.json index 8fc19ab5f7c..5a715f11f0f 100644 --- a/vhdbuilder/packer/vhd-image-builder-acl.json +++ b/vhdbuilder/packer/vhd-image-builder-acl.json @@ -726,12 +726,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, { "type": "shell", "inline": [ @@ -744,12 +738,6 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, - { - "type": "shell", - "inline": [ - "sudo rm /var/log/bcc_installation.log" - ] - }, { "type": "shell", "inline": ": reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json index fa25d264ec0..2f3b88e7741 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json @@ -831,16 +831,6 @@ "sudo CONTINUE_ON_LOCAL_REPO_DOWNLOAD_ERROR={{user `continue_on_local_repo_download_error`}} LOCAL_DOCA_REPO_URL=\"{{user `local_doca_repo_url`}}\" FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} TELEPORTD_PLUGIN_DOWNLOAD_URL={{user `teleportd_plugin_download_url`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, - { - "type": "shell", - "inline": ["sudo rm /var/log/bcc_installation.log"] - }, { "type": "shell", "inline": ["sudo /bin/bash /home/packer/generate-disk-usage.sh"] diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json index cf571b3431c..1e16bed154c 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json @@ -776,12 +776,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, { "type": "shell", "inline": [ @@ -794,12 +788,6 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, - { - "type": "shell", - "inline": [ - "sudo rm /var/log/bcc_installation.log" - ] - }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index 1bb630a346e..1b4d7b79f60 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -774,12 +774,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, { "type": "shell", "inline": [ @@ -792,12 +786,6 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, - { - "type": "shell", - "inline": [ - "sudo rm /var/log/bcc_installation.log" - ] - }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-cvm.json b/vhdbuilder/packer/vhd-image-builder-cvm.json index a4b6e438c9d..587817d2f05 100644 --- a/vhdbuilder/packer/vhd-image-builder-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-cvm.json @@ -778,12 +778,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, { "type": "shell", "inline": [ @@ -796,12 +790,6 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, - { - "type": "shell", - "inline": [ - "sudo rm /var/log/bcc_installation.log" - ] - }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json index 96a506cd113..e14027cf66d 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json @@ -740,12 +740,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} ENABLE_CGROUPV2={{user `enable_cgroupv2`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, { "type": "shell", "inline": [ @@ -758,12 +752,6 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, - { - "type": "shell", - "inline": [ - "sudo rm /var/log/bcc_installation.log" - ] - }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json b/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json index f2260889fd8..5b5b53fc673 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json @@ -741,12 +741,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} ENABLE_CGROUPV2={{user `enable_cgroupv2`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, { "type": "shell", "inline": [ @@ -759,12 +753,6 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, - { - "type": "shell", - "inline": [ - "sudo rm /var/log/bcc_installation.log" - ] - }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner.json b/vhdbuilder/packer/vhd-image-builder-mariner.json index 7c43894b629..d01bc1035e2 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner.json @@ -742,12 +742,6 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} ENABLE_CGROUPV2={{user `enable_cgroupv2`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, - { - "type": "file", - "direction": "download", - "source": "/var/log/bcc_installation.log", - "destination": "bcc-tools-installation.log" - }, { "type": "shell", "inline": [ @@ -760,12 +754,6 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, - { - "type": "shell", - "inline": [ - "sudo rm /var/log/bcc_installation.log" - ] - }, { "type": "shell", "inline": "sudo reboot", From 9cd3b114d6bfc9740b7f898f3d527fb9195c82e4 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 10:35:42 -0700 Subject: [PATCH 52/78] chore: fix renovate tag --- parts/common/components.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/common/components.json b/parts/common/components.json index e3c24f6c8d2..23143b3530b 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2168,7 +2168,7 @@ "r2604": { "versionsV2": [ { - "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=26.04", + "renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=ubuntu, release=26.04", "latestVersion": "1:4.6.0-1" } ] From f2df495373cfca5aa5dab5fcba0a766b4387d86a Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 11:06:28 -0700 Subject: [PATCH 53/78] chore: fix quoting --- vhdbuilder/packer/packer_source.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index 5f65d1936aa..51499d646ea 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -409,7 +409,7 @@ copyPackerFiles() { cpAndMode $AKS_CHECK_NETWORK_SCRIPT_SRC $AKS_CHECK_NETWORK_SCRIPT_DEST 755 cpAndMode $AKS_CHECK_NETWORK_SERVICE_SRC $AKS_CHECK_NETWORK_SERVICE_DEST 644 - if [ ${UBUNTU_RELEASE} = "22.04" ] || [ ${UBUNTU_RELEASE} = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then + if [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then PAM_D_COMMON_AUTH_SRC=/home/packer/pam-d-common-auth-2204 fi From 7584119995b97b8ff0c78f68c9f1fc8066ae79f8 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 11:08:53 -0700 Subject: [PATCH 54/78] chore: feedback --- vhdbuilder/packer/trivy-scan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 119cb133b40..1bc9b51ef0b 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -118,6 +118,7 @@ install_azure_cli() { echo "Azure CLI is not installed properly." exit 1 fi + return 0 fi if [ "$OS_VERSION" = "24.04" ] || [ "$OS_VERSION" = "26.04" ]; then apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg From 3558624ecd9f666b022145b2deb78447da8d8d0d Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 11:09:56 -0700 Subject: [PATCH 55/78] chore: more feedback --- vhdbuilder/packer/trivy-scan.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 1bc9b51ef0b..58a4262424e 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -100,7 +100,7 @@ install_azure_cli() { export PATH="/home/$TEST_VM_ADMIN_USERNAME/venv/bin:$PATH" pip install azure-cli CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z $CHECKAZ ]; then + if [ -z "$CHECKAZ" ]; then echo "Azure CLI is not installed properly." exit 1 fi @@ -114,7 +114,7 @@ install_azure_cli() { pip install azure-cli export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z $CHECKAZ ]; then + if [ -z "$CHECKAZ" ]; then echo "Azure CLI is not installed properly." exit 1 fi From 91004caee76cc345172613290abef1ac269415dd Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 11:14:05 -0700 Subject: [PATCH 56/78] chore: fix typo --- parts/linux/cloud-init/artifacts/cse_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 0cbace81763..00ffd33042c 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -322,7 +322,7 @@ disableSystemdResolved() { cat /etc/resolv.conf UBUNTU_RELEASE=$(lsb_release -r -s 2>/dev/null || echo "") if [ "${UBUNTU_RELEASE}" = "20.04" ] || [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then - echo "Ingoring systemd-resolved query service but using its resolv.conf file" + echo "Ignoring systemd-resolved query service but using its resolv.conf file" echo "This is the simplest approach to workaround resolved issues without completely uninstall it" [ -f /run/systemd/resolve/resolv.conf ] && ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf ls -ltr /etc/resolv.conf From 3b2165f9f0bcba1a2caf257dd803f14fcfa3049a Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 11:39:31 -0700 Subject: [PATCH 57/78] chore: update assignRootPW for 26.04's python3.14 --- parts/linux/cloud-init/artifacts/cis.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/parts/linux/cloud-init/artifacts/cis.sh b/parts/linux/cloud-init/artifacts/cis.sh index 6eaa4235da5..948febb3c46 100644 --- a/parts/linux/cloud-init/artifacts/cis.sh +++ b/parts/linux/cloud-init/artifacts/cis.sh @@ -8,7 +8,17 @@ assignRootPW() { VERSION=$(grep DISTRIB_RELEASE /etc/*-release | cut -f 2 -d "=") SALT=$(openssl rand -base64 5) SECRET=$(openssl rand -base64 37) + CMD="import crypt, getpass, pwd; print(crypt.crypt('$SECRET', '\$6\$$SALT\$'))" + if [ "${VERSION}" = "26.04" ]; then + # Ubuntu 26.04 ships Python 3.14, which removed the 'crypt' module (removed in 3.13). + # passlib's sha512_crypt has a pure-Python backend that produces the same SHA-512 ($6$) + # hash without needing the crypt module. rounds=5000 matches the crypt(3) default cost. + # SALT is base64 and may contain chars (e.g. '+', '=') outside passlib's salt alphabet, + # so filter it to sha512_crypt.salt_chars before use. + CMD="from passlib.hash import sha512_crypt, getpass, pwd; salt=''.join(c for c in '$SALT' if c in sha512_crypt.salt_chars); print(sha512_crypt.using(salt=salt, rounds=5000).hash('$SECRET'))" + fi + if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ] || [ "${VERSION}" = "26.04" ]; then HASH=$(python3 -c "$CMD") else From c2a256b98257977d52016eb0529bf64c06c1a873 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 12:09:02 -0700 Subject: [PATCH 58/78] chore: fix python syntax --- parts/common/components.json | 2 +- parts/linux/cloud-init/artifacts/cis.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index 23143b3530b..6f49c161266 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -957,7 +957,7 @@ "versionsV2": [ { "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.1.4-ubuntu26.04u1" + "latestVersion": "1.1.4-ubuntu26.04u2" } ] }, diff --git a/parts/linux/cloud-init/artifacts/cis.sh b/parts/linux/cloud-init/artifacts/cis.sh index 948febb3c46..f93fa2ce518 100644 --- a/parts/linux/cloud-init/artifacts/cis.sh +++ b/parts/linux/cloud-init/artifacts/cis.sh @@ -16,7 +16,7 @@ assignRootPW() { # hash without needing the crypt module. rounds=5000 matches the crypt(3) default cost. # SALT is base64 and may contain chars (e.g. '+', '=') outside passlib's salt alphabet, # so filter it to sha512_crypt.salt_chars before use. - CMD="from passlib.hash import sha512_crypt, getpass, pwd; salt=''.join(c for c in '$SALT' if c in sha512_crypt.salt_chars); print(sha512_crypt.using(salt=salt, rounds=5000).hash('$SECRET'))" + CMD="import getpass, pwd; from passlib.hash import sha512_crypt; salt=''.join(c for c in '$SALT' if c in sha512_crypt.salt_chars); print(sha512_crypt.using(salt=salt, rounds=5000).hash('$SECRET'))" fi if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ] || [ "${VERSION}" = "26.04" ]; then From aee624eb01c966fa4380dd3abc83c704a015ac70 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 16:52:31 -0700 Subject: [PATCH 59/78] chore: don't use sudo-rs --- parts/linux/cloud-init/artifacts/cis.sh | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/parts/linux/cloud-init/artifacts/cis.sh b/parts/linux/cloud-init/artifacts/cis.sh index f93fa2ce518..0ee20b08005 100644 --- a/parts/linux/cloud-init/artifacts/cis.sh +++ b/parts/linux/cloud-init/artifacts/cis.sh @@ -279,7 +279,34 @@ EOF systemctl restart ssh } +# Ensures that the classic (Todd Miller) sudo is the selected sudo implementation +# rather than sudo-rs. On Ubuntu 26.04 sudo-rs is installed and enabled +# by default, but sudo-rs does NOT support the `Defaults logfile` sudoers setting that +# configureSudo relies on -- per the sudo-rs docs, "logfile is not supported; logging +# is always done via syslog". Leaving sudo-rs as the default would therefore be a CIS +# regression: we would lose the dedicated /var/log/sudo.log audit trail, and sudo-rs +# would also print "unknown setting: 'logfile'" on every sudo invocation. Both classic +# sudo and sudo-rs ship on the base image and are switchable via update-alternatives, +# so select classic sudo here so the logfile setting below is actually honored. +# This is a no-op on releases where sudo is not an update-alternatives group (e.g. +# 22.04/24.04, which only have classic sudo). +# TODO: revisit if/when sudo-rs gains support for file-based logging. +preferClassicSudo() { + # Classic sudo registers itself as the "/usr/bin/sudo.ws" alternative candidate + # (sudo.ws = Todd Miller's sudo). Only relevant where sudo is update-alternatives + # managed (i.e. sudo-rs is present); a no-op on releases that ship only classic sudo. + local classic_sudo="/usr/bin/sudo.ws" + if [ -x "${classic_sudo}" ] && update-alternatives --list sudo >/dev/null 2>&1; then + update-alternatives --set sudo "${classic_sudo}" || echo "warning: failed to set ${classic_sudo} as the default sudo implementation" + fi +} + configureSudo() { + # sudo-rs (the default on Ubuntu 26.04) can only log to syslog and rejects the + # `Defaults logfile` setting below. Select classic sudo first so CIS file-based + # sudo logging works; see preferClassicSudo for details. + preferClassicSudo + cat </etc/sudoers.d/99-cis Defaults logfile="/var/log/sudo.log" EOF From c821ec9efb776c2c8a5034fa228ae67f667400ba Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 15 Jul 2026 16:53:26 -0700 Subject: [PATCH 60/78] chore: cleanup --- vhdbuilder/packer/install-dependencies.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 646c3df50cf..7c83a00825f 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1130,16 +1130,6 @@ APT::Periodic::Unattended-Upgrade "0"; EOF fi -# # If the IMG_SKU does not contain "minimal", installDeps normally -# # shellcheck disable=SC3010 -# if [[ "$IMG_SKU" != *"minimal"* ]]; then -# installDeps -# else -# updateAptWithMicrosoftPkg -# fi - -# TODO(2604): confirm if this is right -# updateAptWithMicrosoftPkg installDeps CHRONYD_DIR=/etc/systemd/system/chronyd.service.d From 06530d8c325b8441fc5c26050adaf9d4e0da728d Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 10:51:46 -0700 Subject: [PATCH 61/78] chore: cleanup, add dcgm-exporter support --- e2e/components/components_test.go | 7 +++++ e2e/scenario_gpu_managed_experience_test.go | 4 +++ parts/common/components.json | 8 +++--- vhdbuilder/packer/install-dependencies.sh | 23 +++++----------- .../packer/vhd-image-builder-acl-arm64.json | 12 +++++++++ vhdbuilder/packer/vhd-image-builder-acl.json | 12 +++++++++ .../packer/vhd-image-builder-arm64-gb.json | 10 +++++++ .../packer/vhd-image-builder-arm64-gen2.json | 12 +++++++++ vhdbuilder/packer/vhd-image-builder-base.json | 12 +++++++++ vhdbuilder/packer/vhd-image-builder-cvm.json | 12 +++++++++ .../vhd-image-builder-mariner-arm64.json | 12 +++++++++ .../packer/vhd-image-builder-mariner-cvm.json | 12 +++++++++ .../packer/vhd-image-builder-mariner.json | 12 +++++++++ .../linux/ubuntu/tool_installs_ubuntu.sh | 27 +------------------ 14 files changed, 129 insertions(+), 46 deletions(-) diff --git a/e2e/components/components_test.go b/e2e/components/components_test.go index 2e03797442e..4fe890a3ed0 100644 --- a/e2e/components/components_test.go +++ b/e2e/components/components_test.go @@ -125,6 +125,13 @@ func TestDCGMExporterCompatibility(t *testing.T) { downloadURL: "https://packages.microsoft.com/repos/microsoft-ubuntu-noble-prod/pool/main/d/dcgm-exporter/dcgm-exporter_%s_amd64.deb", parseDeps: parseDebDeps, }, + { + name: "Ubuntu2604", + os: "ubuntu", + osVersion: "r2604", + downloadURL: "https://packages.microsoft.com/repos/microsoft-ubuntu-resolute-prod/pool/main/d/dcgm-exporter/dcgm-exporter_%s_amd64.deb", + parseDeps: parseDebDeps, + }, { name: "AzureLinux3", os: "azurelinux", diff --git a/e2e/scenario_gpu_managed_experience_test.go b/e2e/scenario_gpu_managed_experience_test.go index 12108028b3b..99c0d0fae64 100644 --- a/e2e/scenario_gpu_managed_experience_test.go +++ b/e2e/scenario_gpu_managed_experience_test.go @@ -88,21 +88,25 @@ type packageOSVariant struct { func Test_Version_Consistency_GPU_Managed_Components(t *testing.T) { allPackageVariants := [][]packageOSVariant{ { + {"nvidia-device-plugin", "ubuntu", "r2604"}, {"nvidia-device-plugin", "ubuntu", "r2404"}, {"nvidia-device-plugin", "ubuntu", "r2204"}, {"nvidia-device-plugin", "azurelinux", "v3.0"}, }, { + {"datacenter-gpu-manager-4-core", "ubuntu", "r2604"}, {"datacenter-gpu-manager-4-core", "ubuntu", "r2404"}, {"datacenter-gpu-manager-4-core", "ubuntu", "r2204"}, {"datacenter-gpu-manager-4-core", "azurelinux", "v3.0"}, }, { + {"datacenter-gpu-manager-4-proprietary", "ubuntu", "r2604"}, {"datacenter-gpu-manager-4-proprietary", "ubuntu", "r2404"}, {"datacenter-gpu-manager-4-proprietary", "ubuntu", "r2204"}, {"datacenter-gpu-manager-4-proprietary", "azurelinux", "v3.0"}, }, { + {"dcgm-exporter", "ubuntu", "r2604"}, {"dcgm-exporter", "ubuntu", "r2404"}, {"dcgm-exporter", "ubuntu", "r2204"}, {"dcgm-exporter", "azurelinux", "v3.0"}, diff --git a/parts/common/components.json b/parts/common/components.json index 912155ccb94..c8502ec4799 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2211,7 +2211,7 @@ "versionsV2": [ { "renovateTag": "name=dcgm-exporter, repository=production, os=ubuntu, release=26.04", - "latestVersion": "4.8.2-ubuntu26.04u9" + "latestVersion": "4.8.3-ubuntu26.04u1" } ] }, @@ -2219,7 +2219,7 @@ "versionsV2": [ { "renovateTag": "name=dcgm-exporter, repository=production, os=ubuntu, release=24.04", - "latestVersion": "4.8.2-ubuntu24.04u9" + "latestVersion": "4.8.3-ubuntu24.04u1" } ] }, @@ -2227,7 +2227,7 @@ "versionsV2": [ { "renovateTag": "name=dcgm-exporter, repository=production, os=ubuntu, release=22.04", - "latestVersion": "4.8.2-ubuntu22.04u9" + "latestVersion": "4.8.3-ubuntu22.04u1" } ] } @@ -2237,7 +2237,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=dcgm-exporter, os=azurelinux, release=3.0", - "latestVersion": "4.8.2-9.azl3" + "latestVersion": "4.8.3-1.azl3" } ] } diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 7c83a00825f..b7a4cdf9ae6 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -329,7 +329,7 @@ cacheKubePackageFromPrivateUrl() { fi } -starteBPFToolsInstallation() { +startEBPFToolsInstallation() { if [ "$OS_VERSION" = "26.04" ]; then echo "Ubuntu 26.04 - skipping eBPF tools installation" return 0 @@ -349,9 +349,12 @@ starteBPFToolsInstallation() { BCC_PID=$! } -finisheBPFToolsInstallation() { +finishEBPFToolsInstallation() { if [ "$OS_VERSION" = "26.04" ]; then echo "Ubuntu 26.04 - skipping eBPF tools installation" + # write out an empty bcc_installation.log since builds currently expect it to be present + touch /var/log/bcc_installation.log || true + chmod 644 /var/log/bcc_installation.log || true return 0 fi @@ -698,10 +701,6 @@ cachePackageAndBinaryComponents() { done ;; "dcgm-exporter") - if [ "$OS_VERSION" = "26.04" ]; then - echo "Ubuntu 26.04 - temporarily skipping dcgm-exporter installation" - continue - fi for version in ${PACKAGE_VERSIONS[@]}; do downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" echo " - dcgm-exporter version ${version}" >> ${VHD_LOGS_FILEPATH} @@ -1237,11 +1236,9 @@ capture_benchmark "${SCRIPT_NAME}_handle_os_specific_configurations" echo "VHD will be built with containerd as the container runtime" # Cache packages and binaries declared within components.json -# TODO(2604): uncomment once able cachePackageAndBinaryComponents # k8s will use images in the k8s.io namespaces - create it -# TODO(2604): uncomment all below once able ctr namespace create k8s.io # Fetch and pre-build the NVIDIA CUDA driver BEFORE starting the BCC background build and BEFORE caching @@ -1254,11 +1251,10 @@ buildNVIDIAKernelModule capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" # Start eBPF tool installation in the background while we pull container images in the foreground -starteBPFToolsInstallation +startEBPFToolsInstallation capture_benchmark "${SCRIPT_NAME}_start_install_ebpf_tools" # Cache container images declared within components.json -# TODO(2604): uncomment all below once able echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} cacheContainerImageComponents capture_benchmark "${SCRIPT_NAME}_caching_container_images" @@ -1268,11 +1264,9 @@ if [ -d "/opt/gpu" ] && [ "$(ls -A /opt/gpu)" ]; then ls -ltr /opt/gpu/* >> ${VHD_LOGS_FILEPATH} fi -# TODO(2604): uncomment once able retagAKSNodeCAWatcher capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" -# TODO(2604): uncomment once able pinPodSandboxImages capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" @@ -1321,7 +1315,7 @@ fi capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu" # Ensure eBPF tools installed successfully -finisheBPFToolsInstallation +finishEBPFToolsInstallation capture_benchmark "${SCRIPT_NAME}_finish_installing_ebpf_tools" configureLsmWithBpf @@ -1338,7 +1332,6 @@ if [ -n "${PRIVATE_PACKAGES_URL:-}" ]; then fi rm -f ./azcopy # cleanup immediately after usage will return in two downloads -# TODO(2604): uncomment once able extractAndCacheCoreDnsBinary collect_grid_compatibility_data @@ -1351,7 +1344,5 @@ capture_benchmark "${SCRIPT_NAME}_remove_nvidia_repos" echo "install-dependencies step completed successfully" -# --------------------------------------------------------- INSTALL DEPENDENCIES END --------------------------------------------------------- - capture_benchmark "${SCRIPT_NAME}_overall" true process_benchmarks diff --git a/vhdbuilder/packer/vhd-image-builder-acl-arm64.json b/vhdbuilder/packer/vhd-image-builder-acl-arm64.json index 8993f7649c6..e212a973da5 100644 --- a/vhdbuilder/packer/vhd-image-builder-acl-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-acl-arm64.json @@ -726,6 +726,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, { "type": "shell", "inline": [ @@ -738,6 +744,12 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, + { + "type": "shell", + "inline": [ + "sudo rm /var/log/bcc_installation.log" + ] + }, { "type": "shell", "inline": ": reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-acl.json b/vhdbuilder/packer/vhd-image-builder-acl.json index 5a715f11f0f..8fc19ab5f7c 100644 --- a/vhdbuilder/packer/vhd-image-builder-acl.json +++ b/vhdbuilder/packer/vhd-image-builder-acl.json @@ -726,6 +726,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, { "type": "shell", "inline": [ @@ -738,6 +744,12 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, + { + "type": "shell", + "inline": [ + "sudo rm /var/log/bcc_installation.log" + ] + }, { "type": "shell", "inline": ": reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json index 2f3b88e7741..fa25d264ec0 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json @@ -831,6 +831,16 @@ "sudo CONTINUE_ON_LOCAL_REPO_DOWNLOAD_ERROR={{user `continue_on_local_repo_download_error`}} LOCAL_DOCA_REPO_URL=\"{{user `local_doca_repo_url`}}\" FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} TELEPORTD_PLUGIN_DOWNLOAD_URL={{user `teleportd_plugin_download_url`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, + { + "type": "shell", + "inline": ["sudo rm /var/log/bcc_installation.log"] + }, { "type": "shell", "inline": ["sudo /bin/bash /home/packer/generate-disk-usage.sh"] diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json index 1e16bed154c..cf571b3431c 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json @@ -776,6 +776,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, { "type": "shell", "inline": [ @@ -788,6 +794,12 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, + { + "type": "shell", + "inline": [ + "sudo rm /var/log/bcc_installation.log" + ] + }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index 1b4d7b79f60..1bb630a346e 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -774,6 +774,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, { "type": "shell", "inline": [ @@ -786,6 +792,12 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, + { + "type": "shell", + "inline": [ + "sudo rm /var/log/bcc_installation.log" + ] + }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-cvm.json b/vhdbuilder/packer/vhd-image-builder-cvm.json index 587817d2f05..a4b6e438c9d 100644 --- a/vhdbuilder/packer/vhd-image-builder-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-cvm.json @@ -778,6 +778,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} VHD_BUILD_TIMESTAMP={{user `vhd_build_timestamp`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, { "type": "shell", "inline": [ @@ -790,6 +796,12 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, + { + "type": "shell", + "inline": [ + "sudo rm /var/log/bcc_installation.log" + ] + }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json index e14027cf66d..96a506cd113 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json @@ -740,6 +740,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} ENABLE_CGROUPV2={{user `enable_cgroupv2`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, { "type": "shell", "inline": [ @@ -752,6 +758,12 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, + { + "type": "shell", + "inline": [ + "sudo rm /var/log/bcc_installation.log" + ] + }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json b/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json index 5b5b53fc673..f2260889fd8 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json @@ -741,6 +741,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} ENABLE_CGROUPV2={{user `enable_cgroupv2`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, { "type": "shell", "inline": [ @@ -753,6 +759,12 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, + { + "type": "shell", + "inline": [ + "sudo rm /var/log/bcc_installation.log" + ] + }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner.json b/vhdbuilder/packer/vhd-image-builder-mariner.json index d01bc1035e2..7c43894b629 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner.json @@ -742,6 +742,12 @@ "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} HYPERV_GENERATION={{user `hyperv_generation`}} CONTAINER_RUNTIME={{user `container_runtime`}} ENABLE_FIPS={{user `enable_fips`}} ENABLE_CGROUPV2={{user `enable_cgroupv2`}} IMG_SKU={{user `img_sku`}} PRIVATE_PACKAGES_URL={{user `private_packages_url`}} /bin/bash -eux /home/packer/install-dependencies.sh" ] }, + { + "type": "file", + "direction": "download", + "source": "/var/log/bcc_installation.log", + "destination": "bcc-tools-installation.log" + }, { "type": "shell", "inline": [ @@ -754,6 +760,12 @@ "source": "/opt/azure/disk-usage.txt", "destination": "disk-usage.txt" }, + { + "type": "shell", + "inline": [ + "sudo rm /var/log/bcc_installation.log" + ] + }, { "type": "shell", "inline": "sudo reboot", diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 20f1723a152..dcadf961c88 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -27,12 +27,6 @@ installBcc() { VERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT - elif [ "${VERSION}" = "26.04" ]; then - # LLVM is pinned to 20 (not 26.04's default 22): BCC v0.37.0 links the individual clang - # static libs, and LLVM 21/22 reorganized clang's driver/options libraries, breaking that - # static link (undefined refs to clang::getDriverOptTable / GetResourcesPath / getLastArgIntValue). - # 20 is the newest major before that reorg and is in the 26.04 archive. - apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm20 llvm-20-dev libclang-20-dev python3 zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_install 120 5 300 build-essential git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev python zlib1g-dev libelf-dev python3-distutils libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi @@ -47,23 +41,12 @@ installBcc() { apt_get_install 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi - # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-20 that is downloaded for 26.04 - if [ "${VERSION}" = "26.04" ]; then - apt_get_install 120 5 300 libpolly-20-dev || exit $ERR_BCC_INSTALL_TIMEOUT - fi - - local bcc_version="v0.29.0" - if [ "${VERSION}" = "26.04" ]; then - # 26.04 bakes CMake v4 and thus requires a newer bcc version, otherwise CMake will fail with: "Compatibility with CMake < 3.5 has been removed from CMake" - bcc_version="v0.37.0" - fi - mkdir -p /tmp/bcc pushd /tmp/bcc || exit 1 git clone https://github.com/iovisor/bcc.git mkdir bcc/build; cd bcc/build || exit 1 - git checkout ${bcc_version} + git checkout v0.29.0 cmake -DENABLE_EXAMPLES=off .. || exit 1 make @@ -80,9 +63,6 @@ installBcc() { # only ensuring they are installed above. if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then apt_get_purge 120 5 300 bison cmake flex libedit-dev libllvm14 llvm-14-dev libclang-14-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT - elif [ "${VERSION}" = "26.04" ]; then - # we remove git to keep the image as minimal as possible - apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm20 llvm-20-dev libclang-20-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT else apt_get_purge 120 5 300 git bison cmake flex libedit-dev libllvm6.0 llvm-6.0-dev libclang-6.0-dev zlib1g-dev libelf-dev libfl-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi @@ -92,11 +72,6 @@ installBcc() { apt_get_purge 120 5 300 libpolly-14-dev || exit $ERR_BCC_INSTALL_TIMEOUT fi - # libPolly.a is needed for the make target that runs later, which is not present in the default patch version of llvm-20 that is downloaded for 26.04 - if [ "${VERSION}" = "26.04" ]; then - apt_get_purge 120 5 300 libpolly-20-dev || exit $ERR_BCC_INSTALL_TIMEOUT - fi - rm -rf /tmp/bcc } From de75f5590703dcb4b51a3e1d2d62b7315a7adda1 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 10:55:14 -0700 Subject: [PATCH 62/78] chore: update dcgm deps --- parts/common/components.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index c8502ec4799..630c399a62e 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2135,7 +2135,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=24.04", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] }, @@ -2143,7 +2143,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=22.04", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] } @@ -2153,7 +2153,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=azurelinux, release=3.0", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] } @@ -2177,7 +2177,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=ubuntu, release=24.04", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] }, @@ -2185,7 +2185,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=ubuntu, release=22.04", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] } @@ -2195,7 +2195,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=azurelinux, release=3.0", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] } From 4d99a10aa443ba4c6677b22f30bbeb34aaba8e51 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 10:58:29 -0700 Subject: [PATCH 63/78] chore: remove TODO --- vhdbuilder/packer/post-install-dependencies.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/vhdbuilder/packer/post-install-dependencies.sh b/vhdbuilder/packer/post-install-dependencies.sh index d25d45d076a..732328c29a9 100644 --- a/vhdbuilder/packer/post-install-dependencies.sh +++ b/vhdbuilder/packer/post-install-dependencies.sh @@ -79,7 +79,6 @@ fi # shellcheck disable=SC2129 echo "kubelet/kubectl downloaded:" >> ${VHD_LOGS_FILEPATH} -# TODO(2604): uncomment when able ls -ltr /opt/bin/kube* >> ${VHD_LOGS_FILEPATH} # shellcheck disable=SC2010 From 268bd9c77d63e53ba01ba3e23186546ee46f4ab5 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 11:34:17 -0700 Subject: [PATCH 64/78] chore: fix nvidia package versioning consistency --- .github/workflows/no-sudo-check.yml | 1 + parts/common/components.json | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/no-sudo-check.yml b/.github/workflows/no-sudo-check.yml index d82f6338e53..e22b7c2df53 100644 --- a/.github/workflows/no-sudo-check.yml +++ b/.github/workflows/no-sudo-check.yml @@ -15,6 +15,7 @@ jobs: | grep -v ':[[:space:]]*#' \ | grep -v 'sudo\.log' \ | grep -v 'sudoers' \ + | grep -v 'update-alternatives' \ | grep -v 'logrotate\.d/sudo' || true) if [ -n "$matches" ]; then echo "::error::Found sudo command invocations in CSE provisioning scripts." diff --git a/parts/common/components.json b/parts/common/components.json index 630c399a62e..2f46330125e 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2051,7 +2051,7 @@ "versionsV2": [ { "renovateTag": "name=nvidia-device-plugin, repository=production, os=ubuntu, release=24.04", - "latestVersion": "0.19.3-ubuntu24.04u3" + "latestVersion": "0.19.3-ubuntu24.04u6" } ] }, @@ -2059,7 +2059,7 @@ "versionsV2": [ { "renovateTag": "name=nvidia-device-plugin, repository=production, os=ubuntu, release=22.04", - "latestVersion": "0.19.3-ubuntu22.04u3" + "latestVersion": "0.19.3-ubuntu22.04u6" } ] } @@ -2069,7 +2069,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=nvidia-device-plugin, os=azurelinux, release=3.0", - "latestVersion": "0.19.3-3.azl3" + "latestVersion": "0.19.3-6.azl3" } ] } From b37bff29b40f45102925dee59d405e30abec89ba Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 12:05:37 -0700 Subject: [PATCH 65/78] chore: clenaup components --- parts/common/components.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/parts/common/components.json b/parts/common/components.json index 2f46330125e..eb64e25824b 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1460,7 +1460,8 @@ { "k8sVersion": "1.36", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.2-ubuntu26.04u2" + "latestVersion": "1.36.2-ubuntu26.04u2", + "previousLatestVersion": "1.36.1-ubuntu26.04u3" } ] }, @@ -1723,6 +1724,11 @@ "downloadURL": "mcr.microsoft.com/oss/binaries/kubernetes/kubernetes-node:${version}-linux-${CPU_ARCH}", "windowsDownloadURL": "https://packages.aks.azure.com/kubernetes/${version}/windowszip/${version}-1int.zip" } + }, + "ubuntu": { + "r2604": { + "versionsV2": [] + } } } }, @@ -1762,6 +1768,11 @@ "downloadURL": "mcr.microsoft.com/oss/binaries/kubernetes/azure-acr-credential-provider:${version}-linux-${CPU_ARCH}" } }, + "ubuntu": { + "r2604": { + "versionsV2": [] + } + }, "flatcar": { "current": { "versionsV2": [] From 9b5777c4e914b2bc6e568ec03046da0f2ed96b62 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 13:04:22 -0700 Subject: [PATCH 66/78] chore: fix packagekit purge, add E2Es --- e2e/config/vhd.go | 16 + e2e/scenario_gpu_managed_experience_test.go | 265 +++++++++++++++ e2e/scenario_localdns_hosts_test.go | 2 + e2e/scenario_rcv1p_test.go | 31 +- e2e/scenario_test.go | 312 +++++++++++++++++- e2e/test_helpers.go | 45 ++- .../packer/post-install-dependencies.sh | 16 +- 7 files changed, 672 insertions(+), 15 deletions(-) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 24a85d3dffc..7716a0c99d7 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -171,6 +171,22 @@ var ( Gallery: imageGalleryLinux, } + VHDUbuntu2604MinimalGen2Containerd = &Image{ + Name: "2604minimalgen2containerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuMinimalContainerd2604Gen2, + Gallery: imageGalleryLinux, + } + + VHDUbuntu2604MinimalArm64Gen2Containerd = &Image{ + Name: "2604minimalgen2arm64containerd", + OS: OSUbuntu, + Arch: "arm64", + Distro: datamodel.AKSUbuntuMinimalArm64Containerd2604Gen2, + Gallery: imageGalleryLinux, + } + VHDAzureLinuxV3Gen2Arm64 = &Image{ Name: "azurelinuxv3gen2arm64", OS: OSAzureLinux, diff --git a/e2e/scenario_gpu_managed_experience_test.go b/e2e/scenario_gpu_managed_experience_test.go index 99c0d0fae64..1afe2e85000 100644 --- a/e2e/scenario_gpu_managed_experience_test.go +++ b/e2e/scenario_gpu_managed_experience_test.go @@ -187,6 +187,19 @@ func Test_DCGM_Exporter_Compatibility(t *testing.T) { } testCases := []testCase{ + { + name: "Ubuntu2604Minimal", + vhd: config.VHDUbuntu2604MinimalGen2Containerd, + os: "ubuntu", + osVersion: "r2604", + description: "Tests that DCGM Exporter is compatible with its dependencies on Ubuntu 26.04 minimal GPU nodes", + downloadCmd: "curl -fL --retry 3 --retry-all-errors -o /tmp/dcgm-exporter.deb 'https://packages.microsoft.com/repos/microsoft-ubuntu-resolute-prod/pool/main/d/dcgm-exporter/dcgm-exporter_%s_amd64.deb'", + extractDepsCmd: "dpkg-deb -f /tmp/dcgm-exporter.deb Depends", + + // Parse output like: "..., datacenter-gpu-manager-4-core (= 1:4.4.2-1), datacenter-gpu-manager-4-proprietary (= 1:4.4.2-1), ..." + coreRegex: `datacenter-gpu-manager-4-core \(= ([^)]+)\)`, + propRegex: `datacenter-gpu-manager-4-proprietary \(= ([^)]+)\)`, + }, { name: "Ubuntu2404", vhd: config.VHDUbuntu2404Gen2Containerd, @@ -307,6 +320,84 @@ func Test_DCGM_Exporter_Compatibility(t *testing.T) { } } +func Test_Ubuntu2604Minimal_NvidiaDevicePluginRunning(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that NVIDIA device plugin and DCGM Exporter are running & functional on Ubuntu 26.04 minimal GPU nodes", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NV6ads_A10_v5" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = true + nbc.EnableNvidia = true + nbc.ManagedGPUExperienceAFECEnabled = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NV6ads_A10_v5") + if vmss.Tags == nil { + vmss.Tags = map[string]*string{} + } + vmss.Tags["EnableManagedGPUExperience"] = to.Ptr("true") + + // Enable the AKS VM extension for GPU nodes + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + os := "ubuntu" + osVersion := "r2604" + + // Validate that the NVIDIA device plugin binary was installed correctly + versions := components.GetExpectedPackageVersions("nvidia-device-plugin", os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one nvidia-device-plugin version for %s %s but got %d", os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, "nvidia-device-plugin", versions[0]) + + // Validate that the NVIDIA device plugin systemd service is running + ValidateNvidiaDevicePluginServiceRunning(ctx, s) + + // Validate that GPU resources are advertised by the device plugin + ValidateNodeAdvertisesGPUResources(ctx, s, 1, "nvidia.com/gpu") + + // Validate that GPU workloads can be scheduled + ValidateGPUWorkloadSchedulable(ctx, s, 1, "nvidia.com/gpu") + + // Validate that the NVIDIA DCGM packages were installed correctly + for _, packageName := range getDCGMPackageNames(os) { + versions := components.GetExpectedPackageVersions(packageName, os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one %s version for %s %s but got %d", packageName, os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, packageName, versions[0]) + } + + ValidateNvidiaDCGMExporterSystemDServiceRunning(ctx, s) + ValidateNvidiaDCGMExporterIsScrapable(ctx, s) + ValidateNvidiaDCGMExporterScrapeCommonMetric(ctx, s, "DCGM_FI_DEV_GPU_UTIL") + ValidateNodeHasLabel(ctx, s, "kubernetes.azure.com/dcgm-exporter", "enabled") + + // Let's run the NPD validation tests to verify that the nvidia + // device plugin & DCGM services are reporting status correctly + ValidateNodeProblemDetector(ctx, s) + // Restart NPD to ensure it picks up the managed GPU experience marker file, + // which may have been created after NPD's initial startup during provisioning. + RestartNodeProblemDetector(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePlugin(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePluginCondition(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePluginAfterFailure(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServices(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServicesCondition(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServicesAfterFailure(ctx, s) + // verify nvidia grid license status checks are reporting status correctly + ValidateNPDHealthyNvidiaGridLicenseStatus(ctx, s) + ValidateNPDUnhealthyNvidiaGridLicenseStatusAfterFailure(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404_NvidiaDevicePluginRunning(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that NVIDIA device plugin and DCGM Exporter are running & functional on Ubuntu 24.04 GPU nodes", @@ -537,6 +628,85 @@ func Test_AzureLinux3_NvidiaDevicePluginRunning(t *testing.T) { }) } +func Test_Ubuntu2604Minimal_NvidiaDevicePluginRunning_MIG(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that NVIDIA device plugin and DCGM Exporter work with MIG enabled on Ubuntu 26.04 minimal GPU nodes", + Location: "westus2", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + SkipScriptlessNBC: true, + WaitForSSHAfterReboot: 5 * time.Minute, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NC24ads_A100_v4" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = true + nbc.EnableNvidia = true + nbc.GPUInstanceProfile = "MIG2g" + nbc.EnableManagedGPU = true + nbc.MigStrategy = "Single" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NC24ads_A100_v4") + + // Enable the AKS VM extension for GPU nodes + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + os := "ubuntu" + osVersion := "r2604" + + // Validate that the NVIDIA device plugin binary was installed correctly + versions := components.GetExpectedPackageVersions("nvidia-device-plugin", os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one nvidia-device-plugin version for %s %s but got %d", os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, "nvidia-device-plugin", versions[0]) + + // Validate that the NVIDIA device plugin systemd service is running + ValidateNvidiaDevicePluginServiceRunning(ctx, s) + + // Validate that MIG mode is enabled via nvidia-smi + ValidateMIGModeEnabled(ctx, s) + + // Validate that MIG instances are created + ValidateMIGInstancesCreated(ctx, s, "MIG 2g.20gb") + + // Validate that GPU resources are advertised by the device plugin + ValidateNodeAdvertisesGPUResources(ctx, s, 3, "nvidia.com/gpu") + + // Validate that MIG workloads can be scheduled + ValidateGPUWorkloadSchedulable(ctx, s, 3, "nvidia.com/gpu") + + // Validate that the NVIDIA DCGM packages were installed correctly + for _, packageName := range getDCGMPackageNames(os) { + versions := components.GetExpectedPackageVersions(packageName, os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one %s version for %s %s but got %d", packageName, os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, packageName, versions[0]) + } + + ValidateNvidiaDCGMExporterSystemDServiceRunning(ctx, s) + ValidateNvidiaDCGMExporterIsScrapable(ctx, s) + ValidateNvidiaDCGMExporterScrapeCommonMetric(ctx, s, "DCGM_FI_DEV_GPU_TEMP") + ValidateNodeHasLabel(ctx, s, "kubernetes.azure.com/dcgm-exporter", "enabled") + + // Let's run the NPD validation tests to verify that the nvidia + // device plugin & DCGM services are reporting status correctly + ValidateNodeProblemDetector(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePlugin(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePluginCondition(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePluginAfterFailure(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServices(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServicesCondition(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServicesAfterFailure(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that NVIDIA device plugin and DCGM Exporter work with MIG enabled on Ubuntu 24.04 GPU nodes", @@ -731,6 +901,63 @@ func Test_CreateVMExtensionLinuxAKSNode_Timing(t *testing.T) { "both calls should return the same extension version") } +func Test_Ubuntu2604Minimal_NvidiaDevicePluginRunning_MIG_Mixed(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that NVIDIA device plugin work with MIG Mixed mode on Ubuntu 26.04 minimal GPU nodes", + Location: "westus2", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + WaitForSSHAfterReboot: 5 * time.Minute, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NC24ads_A100_v4" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = true + nbc.EnableNvidia = true + nbc.GPUInstanceProfile = "MIG1g" + nbc.EnableManagedGPU = true + nbc.MigStrategy = "Mixed" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NC24ads_A100_v4") + + // Enable the AKS VM extension for GPU nodes + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + os := "ubuntu" + osVersion := "r2604" + + // Validate that the NVIDIA device plugin binary was installed correctly + versions := components.GetExpectedPackageVersions("nvidia-device-plugin", os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one nvidia-device-plugin version for %s %s but got %d", os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, "nvidia-device-plugin", versions[0]) + + // Validate that the NVIDIA device plugin systemd service is running + ValidateNvidiaDevicePluginServiceRunning(ctx, s) + + // Validate that MIG mode is enabled via nvidia-smi + ValidateMIGModeEnabled(ctx, s) + + // Validate that MIG instances are created + ValidateMIGInstancesCreated(ctx, s, "MIG 1g.10gb") + + // Validate that MIG profile-specific GPU resources are advertised by the device plugin + migResourceName := "nvidia.com/mig-1g.10gb" + ValidateNodeAdvertisesGPUResources(ctx, s, 7, migResourceName) + + // Validate that MIG workloads can be scheduled + ValidateGPUWorkloadSchedulable(ctx, s, 2, migResourceName) + }, + }, + }) +} + func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_Mixed(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that NVIDIA device plugin work with MIG Mixed mode on Ubuntu 24.04 GPU nodes", @@ -788,6 +1015,44 @@ func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_Mixed(t *testing.T) { }) } +func Test_Ubuntu2604Minimal_DraDriverNvidiaGpuRunning(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests DRA driver works on Ubuntu 26.04 minimal VHD with containerd v2", + Tags: Tags{ + GPU: true, + }, + + Config: Config{ + Cluster: ClusterKubenet, + SkipScriptlessNBC: true, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NV6ads_A10_v5" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableNvidia = true + nbc.EnableManagedGPUDRA = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NV6ads_A10_v5") + + // Enable the AKS VM extension for GPU nodes + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2404") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2404") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateDraDriverNvidiaGpuServiceRunning(ctx, s) + ValidateDRAWorkloadSchedulable(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404_DraDriverNvidiaGpuRunning(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests DRA driver works on Ubuntu 24.04 VHD with containerd v2", diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index f9aa4452716..5e26b73e57b 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -24,6 +24,7 @@ func Test_LocalDNSHostsPlugin(t *testing.T) { }{ {name: "Ubuntu2204", vhd: config.VHDUbuntu2204Gen2Containerd}, {name: "Ubuntu2404", vhd: config.VHDUbuntu2404Gen2Containerd}, + {name: "Ubuntu2604Minimal", vhd: config.VHDUbuntu2604MinimalGen2Containerd}, {name: "AzureLinuxV3", vhd: config.VHDAzureLinuxV3Gen2}, {name: "ACL", vhd: config.VHDACLGen2TL, vmConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) @@ -61,6 +62,7 @@ func Test_LocalDNSHostsPlugin_Scriptless(t *testing.T) { }{ {name: "Ubuntu2204", vhd: config.VHDUbuntu2204Gen2Containerd}, {name: "Ubuntu2404", vhd: config.VHDUbuntu2404Gen2Containerd}, + {name: "Ubuntu2604Minimal", vhd: config.VHDUbuntu2604MinimalGen2Containerd}, {name: "AzureLinuxV3", vhd: config.VHDAzureLinuxV3Gen2}, {name: "ACL", vhd: config.VHDACLGen2TL, vmConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) diff --git a/e2e/scenario_rcv1p_test.go b/e2e/scenario_rcv1p_test.go index f6e2ea6c3b9..4a32198a540 100644 --- a/e2e/scenario_rcv1p_test.go +++ b/e2e/scenario_rcv1p_test.go @@ -310,7 +310,7 @@ func rcv1pOptInVMConfigMutator(vmss *armcompute.VirtualMachineScaleSet) { func Test_RCV1P_Ubuntu2204(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Ubuntu 22.04 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Ubuntu 22.04 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -325,13 +325,34 @@ func Test_RCV1P_Ubuntu2204(t *testing.T) { }) } +// Test_RCV1P_Ubuntu2604Minimal validates RCV1P cert download and trust store installation on Ubuntu 26.04 minimal. +// Covers the newer Ubuntu LTS release to ensure the cert endpoint and trust store integration +// work correctly across Ubuntu versions. +func Test_RCV1P_Ubuntu2604Minimal(t *testing.T) { + skipIfRCV1PNotConfigured(t) + RunScenario(t, &Scenario{ + Description: "Tests RCV1P cert mode on Ubuntu 26.04 minimal with VM opt-in tag", + Tags: Tags{ + RCV1PCertMode: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + VMConfigMutator: rcv1pVMConfigMutator(), + Validator: func(ctx context.Context, s *Scenario) { + ValidateRCV1PCertMode(ctx, s) + }, + }, + }) +} + // Test_RCV1P_Ubuntu2404 validates RCV1P cert download and trust store installation on Ubuntu 24.04. // Covers the newer Ubuntu LTS release to ensure the cert endpoint and trust store integration // work correctly across Ubuntu versions. func Test_RCV1P_Ubuntu2404(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Ubuntu 24.04 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Ubuntu 24.04 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -352,7 +373,7 @@ func Test_RCV1P_Ubuntu2404(t *testing.T) { func Test_RCV1P_AzureLinuxV3(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Azure Linux V3 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Azure Linux V3 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -373,7 +394,7 @@ func Test_RCV1P_AzureLinuxV3(t *testing.T) { func Test_RCV1P_ACL(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on ACL with VM opt-in tag", + Description: "Tests RCV1P cert mode on ACL with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -403,7 +424,7 @@ func Test_RCV1P_ACL(t *testing.T) { func Test_RCV1P_NotOptedIn(t *testing.T) { skipIfRCV1PNotExplicit(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode without VM opt-in tag; expects no cert installation", + Description: "Tests RCV1P cert mode without VM opt-in tag; expects no cert installation", Tags: Tags{ RCV1PCertMode: true, }, diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 66596d0ac4e..cc12bf7a966 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -2205,6 +2205,284 @@ func Test_Ubuntu2404Gen2(t *testing.T) { }) } +func Test_Ubuntu2604Minimal(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal VHD can be properly bootstrapped with containerd v2", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a new Ubuntu 2604 minimal node using self contained installer can be properly bootstrapped", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully") + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {}, + }, + }) +} + +func Test_Ubuntu2604Minimal_AzureCNI(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal VHD can be properly bootstrapped with containerd v2 on an Azure CNI cluster", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: clusterAzureNetwork, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_AzureCNI_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a new Ubuntu 2604 minimal node using self contained installer can be properly bootstrapped on an Azure CNI cluster", + Config: Config{ + Cluster: clusterAzureNetwork, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully") + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {}, + }, + }) +} + +func Test_Ubuntu2604Minimal_NPD_Basic(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Test that a node using Ubuntu 26.04 minimal with AKS VM Extension enabled can report simulated node problem detector events", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNodeProblemDetector(ctx, s) + ValidateNPDFilesystemCorruption(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecondaryNIC(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a secondary NIC is properly configured via configureSecondaryNICs on Ubuntu 26.04 minimal", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + // configureSecondaryNICs is new and not yet baked into released VHDs. + // The scriptless_nbc path always uses VHD scripts (DisableCustomData=true), + // so it can't pick up the new function until the next VHD release. + SkipScriptlessNBC: true, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.StandardSecondaryNICCount = 1 + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + addSecondaryNIC(vmss) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false") + ValidateSecondaryNICUp(ctx, s, resolveSecondaryNICName(ctx, s)) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecondaryNIC_DualStack(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a dual-stack secondary NIC is properly configured on Ubuntu 26.04 minimal", + Config: Config{ + Cluster: ClusterAzureOverlayNetworkDualStack, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + SkipScriptlessNBC: true, + BootstrapConfigMutator: func(c *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.StandardSecondaryNICCount = 1 + if nbc.ContainerService.Properties.FeatureFlags == nil { + nbc.ContainerService.Properties.FeatureFlags = &datamodel.FeatureFlags{} + } + nbc.ContainerService.Properties.FeatureFlags.EnableIPv6DualStack = true + nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone) + nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone) + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/podnetwork-type"] = "overlay" + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/nodenetwork-vnetguid"] = c.VNetResourceGUID + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/azure-cni-overlay"] = "true" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + DualStackVMConfigMutator(vmss) + addDualStackSecondaryNIC(vmss) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4-overrides:") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6-overrides:") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false") + ValidateSecondaryNICDualStack(ctx, s, resolveSecondaryNICName(ctx, s)) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using an Ubuntu 2604 minimal Gen2 VHD can be properly bootstrapped even if secure TLS bootstrapping fails", + Tags: Tags{ + BootstrapTokenFallback: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{ + Enabled: true, + GetAccessTokenTimeout: (10 * time.Second).String(), + UserAssignedIdentityID: "invalid", // use an unexpected user-assigned identity ID to force a secure TLS bootstrapping failure + } + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_GPUA10(t *testing.T) { + runScenarioUbuntu2604MinimalGRID(t, "Standard_NV6ads_A10_v5") +} + +func Test_Ubuntu2604Minimal_GPU_H100(t *testing.T) { + RunScenario(t, runScenarioUbuntu2604MinimalGPUNPD(t, "Standard_ND96isr_H100_v5", "uaenorth", "")) +} + +func Test_Ubuntu2604Minimal_GPU_A100(t *testing.T) { + RunScenario(t, runScenarioUbuntu2604MinimalGPUNPD(t, "Standard_ND96asr_v4", "southcentralus", "Standard_D2s_v3")) +} + +func Test_Ubuntu2604MinimalArm64(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal ARM64 VHD can be properly bootstrapped with containerd v2", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a new Ubuntu 2604 minimal ARM64 node using self contained installer can be properly bootstrapped", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully") + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {}, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_AzureCNI(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal ARM64 VHD can be properly bootstrapped with containerd v2 on an Azure CNI cluster", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: clusterAzureNetwork, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_AzureCNI_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a new Ubuntu 2604 minimal ARM64 node using self contained installer can be properly bootstrapped on an Azure CNI cluster", + Config: Config{ + Cluster: clusterAzureNetwork, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully") + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {}, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_NPD_Basic(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Test that a node using Ubuntu 26.04 minimal ARM64 with AKS VM Extension enabled can report simulated node problem detector events", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNodeProblemDetector(ctx, s) + ValidateNPDFilesystemCorruption(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404Gen2_McrChinaCloud_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ Tags: Tags{ @@ -2408,6 +2686,36 @@ func runScenarioUbuntu2404GRID(t *testing.T, vmSize string) { }) } +func runScenarioUbuntu2604MinimalGRID(t *testing.T, vmSize string) { + RunScenario(t, &Scenario{ + Description: fmt.Sprintf("Tests that a GPU-enabled node with VM size %s using an Ubuntu 2604 minimal VHD can be properly bootstrapped, and that the GRID license is valid", vmSize), + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = vmSize + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = false + nbc.EnableNvidia = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(vmSize) + }, + Validator: func(ctx context.Context, s *Scenario) { + // Ensure nvidia-modprobe install does not restart kubelet and temporarily cause node to be unschedulable + ValidateNvidiaModProbeInstalled(ctx, s) + ValidateNvidiaGRIDLicenseValid(ctx, s) + ValidateKubeletHasNotStopped(ctx, s) + ValidateServicesDoNotRestartKubelet(ctx, s) + ValidateNvidiaPersistencedRunning(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404_GPUA10(t *testing.T) { runScenarioUbuntu2404GRID(t, "Standard_NV6ads_A10_v5") } @@ -2465,11 +2773,11 @@ func Test_Ubuntu2404_NPD_Basic(t *testing.T) { } func Test_Ubuntu2404_GPU_H100(t *testing.T) { - RunScenario(t, runScenarioGPUNPD(t, "Standard_ND96isr_H100_v5", "uaenorth", "")) + RunScenario(t, runScenarioUbuntu2404GPUNPD(t, "Standard_ND96isr_H100_v5", "uaenorth", "")) } func Test_Ubuntu2404_GPU_A100(t *testing.T) { - RunScenario(t, runScenarioGPUNPD(t, "Standard_ND96asr_v4", "southcentralus", "Standard_D2s_v3")) + RunScenario(t, runScenarioUbuntu2404GPUNPD(t, "Standard_ND96asr_v4", "southcentralus", "Standard_D2s_v3")) } func Test_AzureLinux3_PMC_Install(t *testing.T) { diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index 1dca0a2be2b..310ed96345a 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -1039,7 +1039,7 @@ func attemptSSHConnection(ctx context.Context, s *Scenario) error { return nil } -func runScenarioGPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) *Scenario { +func runScenarioUbuntu2404GPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) *Scenario { t.Helper() return &Scenario{ Description: fmt.Sprintf("Tests that a GPU-enabled node with VM size %s using an Ubuntu 2404 VHD can be properly bootstrapped and NPD tests are valid", vmSize), @@ -1081,3 +1081,46 @@ func runScenarioGPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) }, }} } + +func runScenarioUbuntu2604MinimalGPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) *Scenario { + t.Helper() + return &Scenario{ + Description: fmt.Sprintf("Tests that a GPU-enabled node with VM size %s using an Ubuntu 2604 minimal VHD can be properly bootstrapped and NPD tests are valid", vmSize), + Location: location, + K8sSystemPoolSKU: k8sSystemPoolSKU, + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = vmSize + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableNvidia = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(vmSize) + + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + // First, ensure nvidia-modprobe install does not restart kubelet and temporarily cause node to be unschedulable + ValidateNvidiaModProbeInstalled(ctx, s) + ValidateKubeletHasNotStopped(ctx, s) + ValidateServicesDoNotRestartKubelet(ctx, s) + + // Then validate NPD configuration and GPU monitoring + ValidateNPDGPUCountPlugin(ctx, s) + ValidateNPDGPUCountCondition(ctx, s) + ValidateNPDGPUCountAfterFailure(ctx, s) + + // Validate the if IB NPD is reporting the flapping condition + ValidateNPDIBLinkFlappingCondition(ctx, s) + ValidateNPDIBLinkFlappingAfterFailure(ctx, s) + }, + }} +} diff --git a/vhdbuilder/packer/post-install-dependencies.sh b/vhdbuilder/packer/post-install-dependencies.sh index 732328c29a9..72509dae75c 100644 --- a/vhdbuilder/packer/post-install-dependencies.sh +++ b/vhdbuilder/packer/post-install-dependencies.sh @@ -49,13 +49,15 @@ if [ $OS = $UBUNTU_OS_NAME ]; then # remove apport retrycmd_if_failure 10 2 60 apt-get purge --auto-remove apport open-vm-tools -y || exit 1 - # Remove PackageKit: an unused desktop D-Bus package manager whose apt hook (/etc/apt/apt.conf.d/20packagekit) - # runs `gdbus call --system` after every apt update. During early-boot CSE the system bus isn't ready yet, so - # that benign `Error connecting: ... Broken pipe` stderr trips the node-bootstrap apt error-check -> CSE exit 99 - # -> node never joins. Purging it also drops packagekit-tools + software-properties-common (add-apt-repository, - # unused at node runtime; the build's only add-apt-repository usage is earlier in pre-install-dependencies.sh). - # No-op on the minimal image, which does not ship these. - retrycmd_if_failure 10 2 60 apt-get purge --auto-remove packagekit packagekit-tools software-properties-common -y || exit 1 + if ! isMinimalImage; then + # Remove PackageKit: an unused desktop D-Bus package manager whose apt hook (/etc/apt/apt.conf.d/20packagekit) + # runs `gdbus call --system` after every apt update. During early-boot CSE the system bus isn't ready yet, so + # that benign `Error connecting: ... Broken pipe` stderr trips the node-bootstrap apt error-check -> CSE exit 99 + # -> node never joins. Purging it also drops packagekit-tools + software-properties-common (add-apt-repository, + # unused at node runtime; the build's only add-apt-repository usage is earlier in pre-install-dependencies.sh). + # No-op on the minimal image, which does not ship these. + retrycmd_if_failure 10 2 60 apt-get purge --auto-remove packagekit packagekit-tools software-properties-common -y || exit 1 + fi # strip old kernels/packages retrycmd_if_failure 10 2 60 apt-get -y autoclean || exit 1 From 5bdeaea8263089743b44894909d2a0bc144d8b82 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 13:49:38 -0700 Subject: [PATCH 67/78] chore: skip testBccTools --- vhdbuilder/packer/test/linux-vhd-content-test.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index 493ad55b0cf..29ff58a398d 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -1731,11 +1731,16 @@ testContainerImagePrefetchScript() { testBccTools () { local test="BCCInstallTest" os_sku="${1}" + os_version="${2}" echo "$test: checking if BCC tools were successfully installed" if [ "$os_sku" = "AzureLinuxOSGuard" ]; then echo "$test: Skipping check on AzureLinuxOSGuard - BCC tools are not installed" return 0 fi + if [ "$os_version" = "26.04" ]; then + echo "$test: Skipping check on Ubuntu 26.04 - BCC tools are not installed" + return 0 + fi for line in ' - bcc-tools' ' - libbcc-examples'; do if ! grep -F -x -e "$line" $VHD_LOGS_FILEPATH; then err "BCC tools were not successfully installed" @@ -2523,7 +2528,7 @@ testContainerNetworkingPluginsInstalled() { # This will keep the VM alive after the tests are run and we can SSH/Bastion into the VM to run the test manually. # Therefore, for example, you can run "sudo bash /var/lib/waagent/run-command/download/0/script.sh" to run the tests manually. checkPerformanceData -testBccTools $OS_SKU +testBccTools $OS_SKU $OS_VERSION testVHDBuildLogsExist testCriticalTools testPackagesInstalled From 84374b6bae51ab0cf0689127d633c069f16da1b3 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 14:34:26 -0700 Subject: [PATCH 68/78] chore: correctly install azcli on 26.04 scanning VM --- vhdbuilder/packer/trivy-scan.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 58a4262424e..6378f18c3fd 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -107,6 +107,19 @@ install_azure_cli() { return 0 fi + if [ "$OS_VERSION" = "26.04" ]; then + apt_get_update + apt_get_install 5 1 60 python3-pip + pip install azure-cli + export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" + CHECKAZ=$(pip freeze | grep "azure-cli==") + if [ -z "$CHECKAZ" ]; then + echo "Azure CLI is not installed properly." + exit 1 + fi + return 0 + fi + if [ "${ARCHITECTURE,,}" = "arm64" ]; then if [ "$OS_VERSION" = "22.04" ]; then apt_get_update @@ -120,7 +133,7 @@ install_azure_cli() { fi return 0 fi - if [ "$OS_VERSION" = "24.04" ] || [ "$OS_VERSION" = "26.04" ]; then + if [ "$OS_VERSION" = "24.04" ]; then apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list From b49bcbdf569c49ff27456885b2a77d6df59ea0ad Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 15:20:42 -0700 Subject: [PATCH 69/78] chore: use python venv for azcli installation --- vhdbuilder/packer/trivy-scan.sh | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 6378f18c3fd..d10b180e0d6 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -95,7 +95,7 @@ install_azure_cli() { return 0 fi - if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ]; then + if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ] || { [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "26.04" ]; }; then python3 -m venv "/home/$TEST_VM_ADMIN_USERNAME/venv" export PATH="/home/$TEST_VM_ADMIN_USERNAME/venv/bin:$PATH" pip install azure-cli @@ -107,19 +107,6 @@ install_azure_cli() { return 0 fi - if [ "$OS_VERSION" = "26.04" ]; then - apt_get_update - apt_get_install 5 1 60 python3-pip - pip install azure-cli - export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" - CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z "$CHECKAZ" ]; then - echo "Azure CLI is not installed properly." - exit 1 - fi - return 0 - fi - if [ "${ARCHITECTURE,,}" = "arm64" ]; then if [ "$OS_VERSION" = "22.04" ]; then apt_get_update From 9457045c52f580ad5543b81b7402403b93ba4758 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Thu, 16 Jul 2026 16:23:25 -0700 Subject: [PATCH 70/78] chore: fixes --- parts/linux/cloud-init/artifacts/cis.sh | 2 +- parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh | 2 +- vhdbuilder/packer/trivy-scan.sh | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cis.sh b/parts/linux/cloud-init/artifacts/cis.sh index 0ee20b08005..628494ac193 100644 --- a/parts/linux/cloud-init/artifacts/cis.sh +++ b/parts/linux/cloud-init/artifacts/cis.sh @@ -297,7 +297,7 @@ preferClassicSudo() { # managed (i.e. sudo-rs is present); a no-op on releases that ship only classic sudo. local classic_sudo="/usr/bin/sudo.ws" if [ -x "${classic_sudo}" ] && update-alternatives --list sudo >/dev/null 2>&1; then - update-alternatives --set sudo "${classic_sudo}" || echo "warning: failed to set ${classic_sudo} as the default sudo implementation" + update-alternatives --set sudo "${classic_sudo}" >/dev/null 2>&1 || echo "warning: failed to set ${classic_sudo} as the default sudo implementation" fi } diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index c51c95b89cc..d22644482b6 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -122,7 +122,7 @@ installDeps() { if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then pkg_list+=("aznfs=3.0.14") elif [ "${OSVERSION}" = "26.04" ]; then - pkg_list+=("aznfs=3.0.18") + pkg_list+=("aznfs=3.0.19") fi aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index d10b180e0d6..3b173c54ca0 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -96,6 +96,10 @@ install_azure_cli() { fi if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ] || { [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "26.04" ]; }; then + if [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "26.04" ]; then + apt_get_update + apt_get_install 5 1 60 python3-pip + fi python3 -m venv "/home/$TEST_VM_ADMIN_USERNAME/venv" export PATH="/home/$TEST_VM_ADMIN_USERNAME/venv/bin:$PATH" pip install azure-cli From 607b7dfdc6be5fce088f8ee81f449594f810fe0e Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 17 Jul 2026 08:37:29 -0700 Subject: [PATCH 71/78] chore: fix trivy-scan.sh --- vhdbuilder/packer/trivy-scan.sh | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 3b173c54ca0..069c0ec4267 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -17,6 +17,9 @@ TRIVY_DEB_2204_VERSION="0.70.0-ubuntu22.04u9" # renovate: datasource=custom.deb2404 depName=trivy versioning=deb TRIVY_DEB_2404_VERSION="0.70.0-ubuntu24.04u9" +# renovate: datasource=custom.deb2604 depName=trivy versioning=deb +TRIVY_DEB_2604_VERSION="0.70.0-ubuntu26.04u12" + # renovate: datasource=rpm depName=trivy registryUrl=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata TRIVY_RPM_VERSION="0.70.0-9.azl3" @@ -77,6 +80,19 @@ retrycmd_if_failure() { echo Executed \"$@\" $i times; } +addPMCAptKey() { + local OS_VERSION=${1} + + retrycmd_if_failure 120 5 25 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + + if [ "${OS_VERSION}" = "26.04" ]; then + # Ubuntu 26.04 (Resolute) PMC repo is signed with Microsoft's newer 2025 gpg key + retrycmd_if_failure 120 5 25 curl https://packages.microsoft.com/keys/microsoft-2025.asc | gpg --dearmor > /tmp/microsoft-2025.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft-2025.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + fi +} + install_azure_cli() { OS_SKU=${1} OS_VERSION=${2} @@ -97,8 +113,9 @@ install_azure_cli() { if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ] || { [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "26.04" ]; }; then if [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "26.04" ]; then + # TODO(2604): switch to PMC-based azcli installation after resolute azcli package is published to PMC apt_get_update - apt_get_install 5 1 60 python3-pip + apt_get_install 5 1 60 python3-pip python3-venv fi python3 -m venv "/home/$TEST_VM_ADMIN_USERNAME/venv" export PATH="/home/$TEST_VM_ADMIN_USERNAME/venv/bin:$PATH" @@ -126,14 +143,12 @@ install_azure_cli() { fi if [ "$OS_VERSION" = "24.04" ]; then apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list apt_get_update apt_get_install 5 1 60 azure-cli fi else apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list apt_get_update apt_get_install 5 1 60 azure-cli @@ -152,15 +167,15 @@ login_with_user_assigned_managed_identity() { echo "logging into azure with flags: $LOGIN_FLAGS" az login $LOGIN_FLAGS } + login_with_umsi_object_id() { login_with_user_assigned_managed_identity "--object-id" "$1" } + login_with_umsi_resource_id() { login_with_user_assigned_managed_identity "--resource-id" "$1" } -install_azure_cli $OS_SKU $OS_VERSION $ARCHITECTURE $TEST_VM_ADMIN_USERNAME - install_trivy_from_github() { # Use the dedicated GitHub fallback version — PMC versions (e.g., 0.68.2) don't have # matching GitHub releases, so we pin to an actual upstream release separately. @@ -197,6 +212,7 @@ install_trivy() { 20.04) deb_version="${TRIVY_DEB_2004_VERSION}" ;; 22.04) deb_version="${TRIVY_DEB_2204_VERSION}" ;; 24.04) deb_version="${TRIVY_DEB_2404_VERSION}" ;; + 26.04) deb_version="${TRIVY_DEB_2604_VERSION}" ;; *) echo "No tracked trivy deb version for Ubuntu $os_version, downloading from GitHub" install_trivy_from_github @@ -217,6 +233,12 @@ install_trivy() { esac } +if [ "$OS_SKU" = "Ubuntu" ]; then + addPMCAptKey $OS_VERSION +fi + +install_azure_cli $OS_SKU $OS_VERSION $ARCHITECTURE $TEST_VM_ADMIN_USERNAME + install_trivy "$OS_SKU" "$OS_VERSION" login_with_umsi_object_id ${UMSI_PRINCIPAL_ID} From 4b3382958ac5f6d03d9ac75b7ee67280f7f28831 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 17 Jul 2026 08:46:38 -0700 Subject: [PATCH 72/78] chore: support aks-secure-tls-bootstrap-client --- vhdbuilder/packer/install-dependencies.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index b7a4cdf9ae6..db5d08c8653 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -592,10 +592,6 @@ cachePackageAndBinaryComponents() { done ;; "aks-secure-tls-bootstrap-client") - if [ "$OS_VERSION" = "26.04" ]; then - echo "Ubuntu 26.04 - temporarily skipping aks-secure-tls-bootstrap-client installation" - continue - fi for version in ${PACKAGE_VERSIONS[@]}; do # removed at provisioning time if secure TLS bootstrapping is disabled if isUbuntu; then From 1196c4950f9916a34bd8e6802187fe1622b3b4b3 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 17 Jul 2026 08:53:31 -0700 Subject: [PATCH 73/78] refactor: trivy-scan.sh azcli installation --- vhdbuilder/packer/trivy-scan.sh | 68 +++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 069c0ec4267..e82834d19e3 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -93,6 +93,22 @@ addPMCAptKey() { fi } +installAzCLIFromUbuntuPMC() { + local arch + arch="$(uname -m)" + + apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg + + if [ "${arch,,}" = "arm64" ]; then + echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + else + echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + fi + + apt_get_update + apt_get_install 5 1 60 azure-cli +} + install_azure_cli() { OS_SKU=${1} OS_VERSION=${2} @@ -128,31 +144,33 @@ install_azure_cli() { return 0 fi - if [ "${ARCHITECTURE,,}" = "arm64" ]; then - if [ "$OS_VERSION" = "22.04" ]; then - apt_get_update - apt_get_install 5 1 60 python3-pip - pip install azure-cli - export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" - CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z "$CHECKAZ" ]; then - echo "Azure CLI is not installed properly." - exit 1 - fi - return 0 - fi - if [ "$OS_VERSION" = "24.04" ]; then - apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - apt_get_update - apt_get_install 5 1 60 azure-cli - fi - else - apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - apt_get_update - apt_get_install 5 1 60 azure-cli - fi + installAzCLIFromUbuntuPMC + + # if [ "${ARCHITECTURE,,}" = "arm64" ]; then + # if [ "$OS_VERSION" = "22.04" ]; then + # apt_get_update + # apt_get_install 5 1 60 python3-pip + # pip install azure-cli + # export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" + # CHECKAZ=$(pip freeze | grep "azure-cli==") + # if [ -z "$CHECKAZ" ]; then + # echo "Azure CLI is not installed properly." + # exit 1 + # fi + # return 0 + # fi + # if [ "$OS_VERSION" = "24.04" ]; then + # apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg + # echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + # apt_get_update + # apt_get_install 5 1 60 azure-cli + # fi + # else + # apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg + # echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + # apt_get_update + # apt_get_install 5 1 60 azure-cli + # fi } login_with_user_assigned_managed_identity() { From 00361d7e2e0e9a4789ce0ac71c565b6212e9ca90 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 17 Jul 2026 09:03:01 -0700 Subject: [PATCH 74/78] chore: fix e2e --- e2e/scenario_gpu_managed_experience_test.go | 6 ++---- e2e/scenario_test.go | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/e2e/scenario_gpu_managed_experience_test.go b/e2e/scenario_gpu_managed_experience_test.go index 07e859cf38f..72f77c27bfb 100644 --- a/e2e/scenario_gpu_managed_experience_test.go +++ b/e2e/scenario_gpu_managed_experience_test.go @@ -638,7 +638,6 @@ func Test_Ubuntu2604Minimal_NvidiaDevicePluginRunning_MIG(t *testing.T) { Config: Config{ Cluster: ClusterKubenet, VHD: config.VHDUbuntu2604MinimalGen2Containerd, - SkipScriptlessNBC: true, WaitForSSHAfterReboot: 5 * time.Minute, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.VMSize = "Standard_NC24ads_A100_v4" @@ -1021,9 +1020,8 @@ func Test_Ubuntu2604Minimal_DraDriverNvidiaGpuRunning(t *testing.T) { }, Config: Config{ - Cluster: ClusterKubenet, - SkipScriptlessNBC: true, - VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.VMSize = "Standard_NV6ads_A10_v5" nbc.ConfigGPUDriverIfNeeded = true diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 7ad7aedbd84..ff9a5ec43f4 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -2282,7 +2282,6 @@ func Test_Ubuntu2604Minimal_SecondaryNIC(t *testing.T) { // configureSecondaryNICs is new and not yet baked into released VHDs. // The scriptless_nbc path always uses VHD scripts (DisableCustomData=true), // so it can't pick up the new function until the next VHD release. - SkipScriptlessNBC: true, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.StandardSecondaryNICCount = 1 }, @@ -2304,9 +2303,8 @@ func Test_Ubuntu2604Minimal_SecondaryNIC_DualStack(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that a dual-stack secondary NIC is properly configured on Ubuntu 26.04 minimal", Config: Config{ - Cluster: ClusterAzureOverlayNetworkDualStack, - VHD: config.VHDUbuntu2604MinimalGen2Containerd, - SkipScriptlessNBC: true, + Cluster: ClusterAzureOverlayNetworkDualStack, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, BootstrapConfigMutator: func(c *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.StandardSecondaryNICCount = 1 if nbc.ContainerService.Properties.FeatureFlags == nil { From 62c12ba0b1192f38785551afc758f099a74607c0 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 17 Jul 2026 09:03:31 -0700 Subject: [PATCH 75/78] chore: fix lint --- e2e/validators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/validators.go b/e2e/validators.go index 2a7c14da38e..74613b6d1f6 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -989,7 +989,7 @@ func ValidateNoFailedSystemdUnits(ctx context.Context, s *Scenario) { Name string `json:"unit,omitempty"` } var failedUnits []systemdUnit - result := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl list-units --failed --output json", 0, fmt.Sprintf("unable to list failed systemd units")) + result := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl list-units --failed --output json", 0, "unable to list failed systemd units") assert.NoError(s.T, json.Unmarshal([]byte(result.stdout), &failedUnits), `unable to parse and unmarshal "systemctl list-units" command output`) failedUnits = lo.Filter(failedUnits, func(unit systemdUnit, _ int) bool { if unitFailureAllowList[unit.Name] { From ae483d12d71ce80d57f826e3cc1c3c2cd682a386 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 17 Jul 2026 09:56:45 -0700 Subject: [PATCH 76/78] chore: fix scanning --- vhdbuilder/packer/trivy-scan.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index e82834d19e3..30efbf1de88 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -127,12 +127,7 @@ install_azure_cli() { return 0 fi - if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ] || { [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "26.04" ]; }; then - if [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "26.04" ]; then - # TODO(2604): switch to PMC-based azcli installation after resolute azcli package is published to PMC - apt_get_update - apt_get_install 5 1 60 python3-pip python3-venv - fi + if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ]; then python3 -m venv "/home/$TEST_VM_ADMIN_USERNAME/venv" export PATH="/home/$TEST_VM_ADMIN_USERNAME/venv/bin:$PATH" pip install azure-cli @@ -144,7 +139,20 @@ install_azure_cli() { return 0 fi - installAzCLIFromUbuntuPMC + if [ "$OS_VERSION" = "26.04" ]; then + # TODO(2604): install azcli from PMC once the resolute package is available + apt_get_update + apt_get_install 5 1 60 python3-pip + pip install azure-cli --break-system-packages + export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" + CHECKAZ=$(pip freeze | grep "azure-cli==") + if [ -z "$CHECKAZ" ]; then + echo "Azure CLI is not installed properly." + exit 1 + fi + else + installAzCLIFromUbuntuPMC + fi # if [ "${ARCHITECTURE,,}" = "arm64" ]; then # if [ "$OS_VERSION" = "22.04" ]; then From 58029331ce08f417f2c162e95940d61dc074f893 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 17 Jul 2026 11:15:42 -0700 Subject: [PATCH 77/78] chore: skip CIS for 26.04 since no benchmarks are yet available --- vhdbuilder/packer/vhd-scanning.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/vhdbuilder/packer/vhd-scanning.sh b/vhdbuilder/packer/vhd-scanning.sh index c1ce9e7e17b..42c24da07ca 100755 --- a/vhdbuilder/packer/vhd-scanning.sh +++ b/vhdbuilder/packer/vhd-scanning.sh @@ -249,10 +249,17 @@ isCISUnsupportedUbuntu() { local os="$1" local version="$2" - # Only 22.04+ are supported - if [ "$os" = "Ubuntu" ] && { [ "$version" = "20.04" ]; }; then + # Only 22.04+ are supported. + if [ "$os" = "Ubuntu" ] && [ "$version" = "20.04" ]; then return 0 fi + + # No CIS benchmarks yet available for Ubuntu 26.04 (resolute) + # TODO(2604): enable 26.04 CIS scanning when support is added by upstream CIS. + if [ "$os" = "Ubuntu" ] && [ "$version" = "26.04" ]; then + return 0 + fi + return 1 } isFlatcar() { From eae6da24f2a1c1f1bd1588ed61c73262be58b505 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Fri, 17 Jul 2026 11:57:32 -0700 Subject: [PATCH 78/78] chore: cleanup --- vhdbuilder/packer/trivy-scan.sh | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 30efbf1de88..8de07d7d489 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -153,32 +153,6 @@ install_azure_cli() { else installAzCLIFromUbuntuPMC fi - - # if [ "${ARCHITECTURE,,}" = "arm64" ]; then - # if [ "$OS_VERSION" = "22.04" ]; then - # apt_get_update - # apt_get_install 5 1 60 python3-pip - # pip install azure-cli - # export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" - # CHECKAZ=$(pip freeze | grep "azure-cli==") - # if [ -z "$CHECKAZ" ]; then - # echo "Azure CLI is not installed properly." - # exit 1 - # fi - # return 0 - # fi - # if [ "$OS_VERSION" = "24.04" ]; then - # apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - # echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - # apt_get_update - # apt_get_install 5 1 60 azure-cli - # fi - # else - # apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - # echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - # apt_get_update - # apt_get_install 5 1 60 azure-cli - # fi } login_with_user_assigned_managed_identity() {