diff --git a/.github/workflows/no-sudo-check.yml b/.github/workflows/no-sudo-check.yml index d82f6338e53..e22b7c2df53 100644 --- a/.github/workflows/no-sudo-check.yml +++ b/.github/workflows/no-sudo-check.yml @@ -15,6 +15,7 @@ jobs: | grep -v ':[[:space:]]*#' \ | grep -v 'sudo\.log' \ | grep -v 'sudoers' \ + | grep -v 'update-alternatives' \ | grep -v 'logrotate\.d/sudo' || true) if [ -n "$matches" ]; then echo "::error::Found sudo command invocations in CSE provisioning scripts." diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index d75574b211d..3a4512bfd9d 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -117,12 +117,12 @@ parameters: displayName: Build 2204 TL Gen2 Containerd type: boolean default: true - - name: build2204minimalgen1containerd - displayName: Build 2204 Minimal Gen1 Containerd + - name: build2604minimalgen2containerd + displayName: Build 2604 Minimal Gen2 Containerd type: boolean default: false - - name: build2204minimalgen2containerd - displayName: Build 2204 Minimal Gen2 Containerd + - name: build2604minimalarm64gen2containerd + displayName: Build 2604 Minimal ARM64 Gen2 Containerd type: boolean default: false - name: buildacltlgen2 @@ -763,21 +763,21 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2204-tl-gen2-containerd - - job: build2204minimalgen1containerd - condition: eq('${{ parameters.build2204minimalgen1containerd }}', true) + - job: build2604minimalgen2containerd + condition: eq('${{ parameters.build2604minimalgen2containerd }}', true) dependsOn: [ ] timeoutInMinutes: 360 steps: - bash: | echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' - echo '##vso[task.setvariable variable=OS_VERSION]22.04' + echo '##vso[task.setvariable variable=OS_VERSION]26.04' echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' - echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-minimal-jammy-aks-daily-preview' - echo '##vso[task.setvariable variable=IMG_SKU]minimal-aks-22_04-daily-lts' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-26_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]minimal' echo '##vso[task.setvariable variable=IMG_VERSION]latest' - echo '##vso[task.setvariable variable=HYPERV_GENERATION]V1' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]minimal' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' @@ -786,23 +786,23 @@ stages: parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} - artifactName: 2204-minimal-gen1-containerd - - job: build2204minimalgen2containerd - condition: eq('${{ parameters.build2204minimalgen2containerd }}', true) + artifactName: 2604-minimal-gen2-containerd + - job: build2604minimalarm64gen2containerd + condition: eq('${{ parameters.build2604minimalarm64gen2containerd }}', true) dependsOn: [ ] timeoutInMinutes: 360 steps: - bash: | echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' - echo '##vso[task.setvariable variable=OS_VERSION]22.04' + echo '##vso[task.setvariable variable=OS_VERSION]26.04' echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' - echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-minimal-jammy-aks-daily-preview' - echo '##vso[task.setvariable variable=IMG_SKU]minimal-aks-22_04-daily-lts-gen2' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-26_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]minimal-arm64' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' - echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]minimal' + echo '##vso[task.setvariable variable=ARCHITECTURE]ARM64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' displayName: Setup Build Variables @@ -810,7 +810,7 @@ stages: parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} - artifactName: 2204-minimal-gen2-containerd + artifactName: 2604-minimal-arm64-gen2-containerd - job: buildacltlgen2 condition: eq('${{ parameters.buildacltlgen2 }}', true) dependsOn: [ ] diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 730505b375f..2cd7fdd8cda 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -75,12 +75,12 @@ steps: - bash: | SKU_NAME=${OS_VERSION} && \ + if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \ if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ - if [[ "${IMG_SKU}" == *"minimal"* ]]; then SKU_NAME="${SKU_NAME}minimal"; fi && \ if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \ SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ @@ -97,7 +97,7 @@ steps: make -f packer.mk run-packer condition: ne(variables['BUILDER'], 'imagecustomizer') displayName: Build VHD - retryCountOnTaskFailure: 3 + # retryCountOnTaskFailure: 3 env: OS_TYPE: Linux GIT_VERSION: $(Build.SourceVersion) @@ -404,6 +404,7 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: PublishPipelineArtifact@0 + condition: always() displayName: Publish Release Notes inputs: artifactName: 'vhd-release-notes-${{ parameters.artifactName }}' diff --git a/e2e/components/components_test.go b/e2e/components/components_test.go index 2e03797442e..4fe890a3ed0 100644 --- a/e2e/components/components_test.go +++ b/e2e/components/components_test.go @@ -125,6 +125,13 @@ func TestDCGMExporterCompatibility(t *testing.T) { downloadURL: "https://packages.microsoft.com/repos/microsoft-ubuntu-noble-prod/pool/main/d/dcgm-exporter/dcgm-exporter_%s_amd64.deb", parseDeps: parseDebDeps, }, + { + name: "Ubuntu2604", + os: "ubuntu", + osVersion: "r2604", + downloadURL: "https://packages.microsoft.com/repos/microsoft-ubuntu-resolute-prod/pool/main/d/dcgm-exporter/dcgm-exporter_%s_amd64.deb", + parseDeps: parseDebDeps, + }, { name: "AzureLinux3", os: "azurelinux", diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 24a85d3dffc..7716a0c99d7 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -171,6 +171,22 @@ var ( Gallery: imageGalleryLinux, } + VHDUbuntu2604MinimalGen2Containerd = &Image{ + Name: "2604minimalgen2containerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuMinimalContainerd2604Gen2, + Gallery: imageGalleryLinux, + } + + VHDUbuntu2604MinimalArm64Gen2Containerd = &Image{ + Name: "2604minimalgen2arm64containerd", + OS: OSUbuntu, + Arch: "arm64", + Distro: datamodel.AKSUbuntuMinimalArm64Containerd2604Gen2, + Gallery: imageGalleryLinux, + } + VHDAzureLinuxV3Gen2Arm64 = &Image{ Name: "azurelinuxv3gen2arm64", OS: OSAzureLinux, diff --git a/e2e/scenario_gpu_managed_experience_test.go b/e2e/scenario_gpu_managed_experience_test.go index 23305e15ddf..72f77c27bfb 100644 --- a/e2e/scenario_gpu_managed_experience_test.go +++ b/e2e/scenario_gpu_managed_experience_test.go @@ -88,21 +88,25 @@ type packageOSVariant struct { func Test_Version_Consistency_GPU_Managed_Components(t *testing.T) { allPackageVariants := [][]packageOSVariant{ { + {"nvidia-device-plugin", "ubuntu", "r2604"}, {"nvidia-device-plugin", "ubuntu", "r2404"}, {"nvidia-device-plugin", "ubuntu", "r2204"}, {"nvidia-device-plugin", "azurelinux", "v3.0"}, }, { + {"datacenter-gpu-manager-4-core", "ubuntu", "r2604"}, {"datacenter-gpu-manager-4-core", "ubuntu", "r2404"}, {"datacenter-gpu-manager-4-core", "ubuntu", "r2204"}, {"datacenter-gpu-manager-4-core", "azurelinux", "v3.0"}, }, { + {"datacenter-gpu-manager-4-proprietary", "ubuntu", "r2604"}, {"datacenter-gpu-manager-4-proprietary", "ubuntu", "r2404"}, {"datacenter-gpu-manager-4-proprietary", "ubuntu", "r2204"}, {"datacenter-gpu-manager-4-proprietary", "azurelinux", "v3.0"}, }, { + {"dcgm-exporter", "ubuntu", "r2604"}, {"dcgm-exporter", "ubuntu", "r2404"}, {"dcgm-exporter", "ubuntu", "r2204"}, {"dcgm-exporter", "azurelinux", "v3.0"}, @@ -183,6 +187,19 @@ func Test_DCGM_Exporter_Compatibility(t *testing.T) { } testCases := []testCase{ + { + name: "Ubuntu2604Minimal", + vhd: config.VHDUbuntu2604MinimalGen2Containerd, + os: "ubuntu", + osVersion: "r2604", + description: "Tests that DCGM Exporter is compatible with its dependencies on Ubuntu 26.04 minimal GPU nodes", + downloadCmd: "curl -fL --retry 3 --retry-all-errors -o /tmp/dcgm-exporter.deb 'https://packages.microsoft.com/repos/microsoft-ubuntu-resolute-prod/pool/main/d/dcgm-exporter/dcgm-exporter_%s_amd64.deb'", + extractDepsCmd: "dpkg-deb -f /tmp/dcgm-exporter.deb Depends", + + // Parse output like: "..., datacenter-gpu-manager-4-core (= 1:4.4.2-1), datacenter-gpu-manager-4-proprietary (= 1:4.4.2-1), ..." + coreRegex: `datacenter-gpu-manager-4-core \(= ([^)]+)\)`, + propRegex: `datacenter-gpu-manager-4-proprietary \(= ([^)]+)\)`, + }, { name: "Ubuntu2404", vhd: config.VHDUbuntu2404Gen2Containerd, @@ -303,6 +320,84 @@ func Test_DCGM_Exporter_Compatibility(t *testing.T) { } } +func Test_Ubuntu2604Minimal_NvidiaDevicePluginRunning(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that NVIDIA device plugin and DCGM Exporter are running & functional on Ubuntu 26.04 minimal GPU nodes", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NV6ads_A10_v5" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = true + nbc.EnableNvidia = true + nbc.ManagedGPUExperienceAFECEnabled = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NV6ads_A10_v5") + if vmss.Tags == nil { + vmss.Tags = map[string]*string{} + } + vmss.Tags["EnableManagedGPUExperience"] = to.Ptr("true") + + // Enable the AKS VM extension for GPU nodes + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + os := "ubuntu" + osVersion := "r2604" + + // Validate that the NVIDIA device plugin binary was installed correctly + versions := components.GetExpectedPackageVersions("nvidia-device-plugin", os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one nvidia-device-plugin version for %s %s but got %d", os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, "nvidia-device-plugin", versions[0]) + + // Validate that the NVIDIA device plugin systemd service is running + ValidateNvidiaDevicePluginServiceRunning(ctx, s) + + // Validate that GPU resources are advertised by the device plugin + ValidateNodeAdvertisesGPUResources(ctx, s, 1, "nvidia.com/gpu") + + // Validate that GPU workloads can be scheduled + ValidateGPUWorkloadSchedulable(ctx, s, 1, "nvidia.com/gpu") + + // Validate that the NVIDIA DCGM packages were installed correctly + for _, packageName := range getDCGMPackageNames(os) { + versions := components.GetExpectedPackageVersions(packageName, os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one %s version for %s %s but got %d", packageName, os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, packageName, versions[0]) + } + + ValidateNvidiaDCGMExporterSystemDServiceRunning(ctx, s) + ValidateNvidiaDCGMExporterIsScrapable(ctx, s) + ValidateNvidiaDCGMExporterScrapeCommonMetric(ctx, s, "DCGM_FI_DEV_GPU_UTIL") + ValidateNodeHasLabel(ctx, s, "kubernetes.azure.com/dcgm-exporter", "enabled") + + // Let's run the NPD validation tests to verify that the nvidia + // device plugin & DCGM services are reporting status correctly + ValidateNodeProblemDetector(ctx, s) + // Restart NPD to ensure it picks up the managed GPU experience marker file, + // which may have been created after NPD's initial startup during provisioning. + RestartNodeProblemDetector(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePlugin(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePluginCondition(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePluginAfterFailure(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServices(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServicesCondition(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServicesAfterFailure(ctx, s) + // verify nvidia grid license status checks are reporting status correctly + ValidateNPDHealthyNvidiaGridLicenseStatus(ctx, s) + ValidateNPDUnhealthyNvidiaGridLicenseStatusAfterFailure(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404_NvidiaDevicePluginRunning(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that NVIDIA device plugin and DCGM Exporter are running & functional on Ubuntu 24.04 GPU nodes", @@ -533,6 +628,84 @@ func Test_AzureLinux3_NvidiaDevicePluginRunning(t *testing.T) { }) } +func Test_Ubuntu2604Minimal_NvidiaDevicePluginRunning_MIG(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that NVIDIA device plugin and DCGM Exporter work with MIG enabled on Ubuntu 26.04 minimal GPU nodes", + Location: "westus2", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + WaitForSSHAfterReboot: 5 * time.Minute, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NC24ads_A100_v4" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = true + nbc.EnableNvidia = true + nbc.GPUInstanceProfile = "MIG2g" + nbc.EnableManagedGPU = true + nbc.MigStrategy = "Single" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NC24ads_A100_v4") + + // Enable the AKS VM extension for GPU nodes + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + os := "ubuntu" + osVersion := "r2604" + + // Validate that the NVIDIA device plugin binary was installed correctly + versions := components.GetExpectedPackageVersions("nvidia-device-plugin", os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one nvidia-device-plugin version for %s %s but got %d", os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, "nvidia-device-plugin", versions[0]) + + // Validate that the NVIDIA device plugin systemd service is running + ValidateNvidiaDevicePluginServiceRunning(ctx, s) + + // Validate that MIG mode is enabled via nvidia-smi + ValidateMIGModeEnabled(ctx, s) + + // Validate that MIG instances are created + ValidateMIGInstancesCreated(ctx, s, "MIG 2g.20gb") + + // Validate that GPU resources are advertised by the device plugin + ValidateNodeAdvertisesGPUResources(ctx, s, 3, "nvidia.com/gpu") + + // Validate that MIG workloads can be scheduled + ValidateGPUWorkloadSchedulable(ctx, s, 3, "nvidia.com/gpu") + + // Validate that the NVIDIA DCGM packages were installed correctly + for _, packageName := range getDCGMPackageNames(os) { + versions := components.GetExpectedPackageVersions(packageName, os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one %s version for %s %s but got %d", packageName, os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, packageName, versions[0]) + } + + ValidateNvidiaDCGMExporterSystemDServiceRunning(ctx, s) + ValidateNvidiaDCGMExporterIsScrapable(ctx, s) + ValidateNvidiaDCGMExporterScrapeCommonMetric(ctx, s, "DCGM_FI_DEV_GPU_TEMP") + ValidateNodeHasLabel(ctx, s, "kubernetes.azure.com/dcgm-exporter", "enabled") + + // Let's run the NPD validation tests to verify that the nvidia + // device plugin & DCGM services are reporting status correctly + ValidateNodeProblemDetector(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePlugin(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePluginCondition(ctx, s) + ValidateNPDUnhealthyNvidiaDevicePluginAfterFailure(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServices(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServicesCondition(ctx, s) + ValidateNPDUnhealthyNvidiaDCGMServicesAfterFailure(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that NVIDIA device plugin and DCGM Exporter work with MIG enabled on Ubuntu 24.04 GPU nodes", @@ -725,6 +898,63 @@ func Test_CreateVMExtensionLinuxAKSNode_Timing(t *testing.T) { "both calls should return the same extension version") } +func Test_Ubuntu2604Minimal_NvidiaDevicePluginRunning_MIG_Mixed(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that NVIDIA device plugin work with MIG Mixed mode on Ubuntu 26.04 minimal GPU nodes", + Location: "westus2", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + WaitForSSHAfterReboot: 5 * time.Minute, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NC24ads_A100_v4" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = true + nbc.EnableNvidia = true + nbc.GPUInstanceProfile = "MIG1g" + nbc.EnableManagedGPU = true + nbc.MigStrategy = "Mixed" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NC24ads_A100_v4") + + // Enable the AKS VM extension for GPU nodes + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + os := "ubuntu" + osVersion := "r2604" + + // Validate that the NVIDIA device plugin binary was installed correctly + versions := components.GetExpectedPackageVersions("nvidia-device-plugin", os, osVersion) + require.Lenf(s.T, versions, 1, "Expected exactly one nvidia-device-plugin version for %s %s but got %d", os, osVersion, len(versions)) + ValidateInstalledPackageVersion(ctx, s, "nvidia-device-plugin", versions[0]) + + // Validate that the NVIDIA device plugin systemd service is running + ValidateNvidiaDevicePluginServiceRunning(ctx, s) + + // Validate that MIG mode is enabled via nvidia-smi + ValidateMIGModeEnabled(ctx, s) + + // Validate that MIG instances are created + ValidateMIGInstancesCreated(ctx, s, "MIG 1g.10gb") + + // Validate that MIG profile-specific GPU resources are advertised by the device plugin + migResourceName := "nvidia.com/mig-1g.10gb" + ValidateNodeAdvertisesGPUResources(ctx, s, 7, migResourceName) + + // Validate that MIG workloads can be scheduled + ValidateGPUWorkloadSchedulable(ctx, s, 2, migResourceName) + }, + }, + }) +} + func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_Mixed(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that NVIDIA device plugin work with MIG Mixed mode on Ubuntu 24.04 GPU nodes", @@ -782,6 +1012,43 @@ func Test_Ubuntu2404_NvidiaDevicePluginRunning_MIG_Mixed(t *testing.T) { }) } +func Test_Ubuntu2604Minimal_DraDriverNvidiaGpuRunning(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests DRA driver works on Ubuntu 26.04 minimal VHD with containerd v2", + Tags: Tags{ + GPU: true, + }, + + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NV6ads_A10_v5" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableNvidia = true + nbc.EnableManagedGPUDRA = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NV6ads_A10_v5") + + // Enable the AKS VM extension for GPU nodes + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2404") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2404") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateDraDriverNvidiaGpuServiceRunning(ctx, s) + ValidateDRAWorkloadSchedulable(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404_DraDriverNvidiaGpuRunning(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests DRA driver works on Ubuntu 24.04 VHD with containerd v2", diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index 4a1f6ab0c7b..ec14eed6b39 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -24,6 +24,7 @@ func Test_LocalDNSHostsPlugin(t *testing.T) { }{ {name: "Ubuntu2204", vhd: config.VHDUbuntu2204Gen2Containerd}, {name: "Ubuntu2404", vhd: config.VHDUbuntu2404Gen2Containerd}, + {name: "Ubuntu2604Minimal", vhd: config.VHDUbuntu2604MinimalGen2Containerd}, {name: "AzureLinuxV3", vhd: config.VHDAzureLinuxV3Gen2}, {name: "ACL", vhd: config.VHDACLGen2TL, vmConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) @@ -62,6 +63,7 @@ func Test_LocalDNSHostsPlugin_Scriptless(t *testing.T) { }{ {name: "Ubuntu2204", vhd: config.VHDUbuntu2204Gen2Containerd}, {name: "Ubuntu2404", vhd: config.VHDUbuntu2404Gen2Containerd}, + {name: "Ubuntu2604Minimal", vhd: config.VHDUbuntu2604MinimalGen2Containerd}, {name: "AzureLinuxV3", vhd: config.VHDAzureLinuxV3Gen2}, {name: "ACL", vhd: config.VHDACLGen2TL, vmConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) diff --git a/e2e/scenario_rcv1p_test.go b/e2e/scenario_rcv1p_test.go index f6e2ea6c3b9..4a32198a540 100644 --- a/e2e/scenario_rcv1p_test.go +++ b/e2e/scenario_rcv1p_test.go @@ -310,7 +310,7 @@ func rcv1pOptInVMConfigMutator(vmss *armcompute.VirtualMachineScaleSet) { func Test_RCV1P_Ubuntu2204(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Ubuntu 22.04 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Ubuntu 22.04 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -325,13 +325,34 @@ func Test_RCV1P_Ubuntu2204(t *testing.T) { }) } +// Test_RCV1P_Ubuntu2604Minimal validates RCV1P cert download and trust store installation on Ubuntu 26.04 minimal. +// Covers the newer Ubuntu LTS release to ensure the cert endpoint and trust store integration +// work correctly across Ubuntu versions. +func Test_RCV1P_Ubuntu2604Minimal(t *testing.T) { + skipIfRCV1PNotConfigured(t) + RunScenario(t, &Scenario{ + Description: "Tests RCV1P cert mode on Ubuntu 26.04 minimal with VM opt-in tag", + Tags: Tags{ + RCV1PCertMode: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + VMConfigMutator: rcv1pVMConfigMutator(), + Validator: func(ctx context.Context, s *Scenario) { + ValidateRCV1PCertMode(ctx, s) + }, + }, + }) +} + // Test_RCV1P_Ubuntu2404 validates RCV1P cert download and trust store installation on Ubuntu 24.04. // Covers the newer Ubuntu LTS release to ensure the cert endpoint and trust store integration // work correctly across Ubuntu versions. func Test_RCV1P_Ubuntu2404(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Ubuntu 24.04 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Ubuntu 24.04 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -352,7 +373,7 @@ func Test_RCV1P_Ubuntu2404(t *testing.T) { func Test_RCV1P_AzureLinuxV3(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Azure Linux V3 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Azure Linux V3 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -373,7 +394,7 @@ func Test_RCV1P_AzureLinuxV3(t *testing.T) { func Test_RCV1P_ACL(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on ACL with VM opt-in tag", + Description: "Tests RCV1P cert mode on ACL with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -403,7 +424,7 @@ func Test_RCV1P_ACL(t *testing.T) { func Test_RCV1P_NotOptedIn(t *testing.T) { skipIfRCV1PNotExplicit(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode without VM opt-in tag; expects no cert installation", + Description: "Tests RCV1P cert mode without VM opt-in tag; expects no cert installation", Tags: Tags{ RCV1PCertMode: true, }, diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 105f12c9282..ff9a5ec43f4 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -2182,6 +2182,282 @@ func Test_Ubuntu2404Gen2(t *testing.T) { }) } +func Test_Ubuntu2604Minimal(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal VHD can be properly bootstrapped with containerd v2", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a new Ubuntu 2604 minimal node using self contained installer can be properly bootstrapped", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully") + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {}, + }, + }) +} + +func Test_Ubuntu2604Minimal_AzureCNI(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal VHD can be properly bootstrapped with containerd v2 on an Azure CNI cluster", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: clusterAzureNetwork, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_AzureCNI_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a new Ubuntu 2604 minimal node using self contained installer can be properly bootstrapped on an Azure CNI cluster", + Config: Config{ + Cluster: clusterAzureNetwork, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully") + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {}, + }, + }) +} + +func Test_Ubuntu2604Minimal_NPD_Basic(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Test that a node using Ubuntu 26.04 minimal with AKS VM Extension enabled can report simulated node problem detector events", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNodeProblemDetector(ctx, s) + ValidateNPDFilesystemCorruption(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecondaryNIC(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a secondary NIC is properly configured via configureSecondaryNICs on Ubuntu 26.04 minimal", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + // configureSecondaryNICs is new and not yet baked into released VHDs. + // The scriptless_nbc path always uses VHD scripts (DisableCustomData=true), + // so it can't pick up the new function until the next VHD release. + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.StandardSecondaryNICCount = 1 + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + addSecondaryNIC(vmss) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false") + ValidateSecondaryNICUp(ctx, s, resolveSecondaryNICName(ctx, s)) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecondaryNIC_DualStack(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a dual-stack secondary NIC is properly configured on Ubuntu 26.04 minimal", + Config: Config{ + Cluster: ClusterAzureOverlayNetworkDualStack, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(c *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.StandardSecondaryNICCount = 1 + if nbc.ContainerService.Properties.FeatureFlags == nil { + nbc.ContainerService.Properties.FeatureFlags = &datamodel.FeatureFlags{} + } + nbc.ContainerService.Properties.FeatureFlags.EnableIPv6DualStack = true + nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone) + nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone) + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/podnetwork-type"] = "overlay" + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/nodenetwork-vnetguid"] = c.VNetResourceGUID + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/azure-cni-overlay"] = "true" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + DualStackVMConfigMutator(vmss) + addDualStackSecondaryNIC(vmss) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4-overrides:") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6-overrides:") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false") + ValidateSecondaryNICDualStack(ctx, s, resolveSecondaryNICName(ctx, s)) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using an Ubuntu 2604 minimal Gen2 VHD can be properly bootstrapped even if secure TLS bootstrapping fails", + Tags: Tags{ + BootstrapTokenFallback: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{ + Enabled: true, + GetAccessTokenTimeout: (10 * time.Second).String(), + UserAssignedIdentityID: "invalid", // use an unexpected user-assigned identity ID to force a secure TLS bootstrapping failure + } + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_GPUA10(t *testing.T) { + runScenarioUbuntu2604MinimalGRID(t, "Standard_NV6ads_A10_v5") +} + +func Test_Ubuntu2604Minimal_GPU_H100(t *testing.T) { + RunScenario(t, runScenarioUbuntu2604MinimalGPUNPD(t, "Standard_ND96isr_H100_v5", "uaenorth", "")) +} + +func Test_Ubuntu2604Minimal_GPU_A100(t *testing.T) { + RunScenario(t, runScenarioUbuntu2604MinimalGPUNPD(t, "Standard_ND96asr_v4", "southcentralus", "Standard_D2s_v3")) +} + +func Test_Ubuntu2604MinimalArm64(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal ARM64 VHD can be properly bootstrapped with containerd v2", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a new Ubuntu 2604 minimal ARM64 node using self contained installer can be properly bootstrapped", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully") + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {}, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_AzureCNI(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal ARM64 VHD can be properly bootstrapped with containerd v2 on an Azure CNI cluster", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: clusterAzureNetwork, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_AzureCNI_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a new Ubuntu 2604 minimal ARM64 node using self contained installer can be properly bootstrapped on an Azure CNI cluster", + Config: Config{ + Cluster: clusterAzureNetwork, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully") + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {}, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_NPD_Basic(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Test that a node using Ubuntu 26.04 minimal ARM64 with AKS VM Extension enabled can report simulated node problem detector events", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNodeProblemDetector(ctx, s) + ValidateNPDFilesystemCorruption(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404Gen2_McrChinaCloud_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ Tags: Tags{ @@ -2385,6 +2661,36 @@ func runScenarioUbuntu2404GRID(t *testing.T, vmSize string) { }) } +func runScenarioUbuntu2604MinimalGRID(t *testing.T, vmSize string) { + RunScenario(t, &Scenario{ + Description: fmt.Sprintf("Tests that a GPU-enabled node with VM size %s using an Ubuntu 2604 minimal VHD can be properly bootstrapped, and that the GRID license is valid", vmSize), + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = vmSize + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = false + nbc.EnableNvidia = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(vmSize) + }, + Validator: func(ctx context.Context, s *Scenario) { + // Ensure nvidia-modprobe install does not restart kubelet and temporarily cause node to be unschedulable + ValidateNvidiaModProbeInstalled(ctx, s) + ValidateNvidiaGRIDLicenseValid(ctx, s) + ValidateKubeletHasNotStopped(ctx, s) + ValidateServicesDoNotRestartKubelet(ctx, s) + ValidateNvidiaPersistencedRunning(ctx, s) + }, + }, + }) +} + func Test_Ubuntu2404_GPUA10(t *testing.T) { runScenarioUbuntu2404GRID(t, "Standard_NV6ads_A10_v5") } @@ -2442,11 +2748,11 @@ func Test_Ubuntu2404_NPD_Basic(t *testing.T) { } func Test_Ubuntu2404_GPU_H100(t *testing.T) { - RunScenario(t, runScenarioGPUNPD(t, "Standard_ND96isr_H100_v5", "uaenorth", "")) + RunScenario(t, runScenarioUbuntu2404GPUNPD(t, "Standard_ND96isr_H100_v5", "uaenorth", "")) } func Test_Ubuntu2404_GPU_A100(t *testing.T) { - RunScenario(t, runScenarioGPUNPD(t, "Standard_ND96asr_v4", "southcentralus", "Standard_D2s_v3")) + RunScenario(t, runScenarioUbuntu2404GPUNPD(t, "Standard_ND96asr_v4", "southcentralus", "Standard_D2s_v3")) } func Test_AzureLinux3_PMC_Install(t *testing.T) { diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index b5a4ac2e1ae..47c7900d9e2 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -1032,7 +1032,7 @@ func attemptSSHConnection(ctx context.Context, s *Scenario) error { return nil } -func runScenarioGPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) *Scenario { +func runScenarioUbuntu2404GPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) *Scenario { t.Helper() return &Scenario{ Description: fmt.Sprintf("Tests that a GPU-enabled node with VM size %s using an Ubuntu 2404 VHD can be properly bootstrapped and NPD tests are valid", vmSize), @@ -1075,6 +1075,49 @@ func runScenarioGPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) }} } +func runScenarioUbuntu2604MinimalGPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) *Scenario { + t.Helper() + return &Scenario{ + Description: fmt.Sprintf("Tests that a GPU-enabled node with VM size %s using an Ubuntu 2604 minimal VHD can be properly bootstrapped and NPD tests are valid", vmSize), + Location: location, + K8sSystemPoolSKU: k8sSystemPoolSKU, + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = vmSize + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableNvidia = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(vmSize) + + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + // First, ensure nvidia-modprobe install does not restart kubelet and temporarily cause node to be unschedulable + ValidateNvidiaModProbeInstalled(ctx, s) + ValidateKubeletHasNotStopped(ctx, s) + ValidateServicesDoNotRestartKubelet(ctx, s) + + // Then validate NPD configuration and GPU monitoring + ValidateNPDGPUCountPlugin(ctx, s) + ValidateNPDGPUCountCondition(ctx, s) + ValidateNPDGPUCountAfterFailure(ctx, s) + + // Validate the if IB NPD is reporting the flapping condition + ValidateNPDIBLinkFlappingCondition(ctx, s) + ValidateNPDIBLinkFlappingAfterFailure(ctx, s) + }, + }} +} + func vmSKUGeneration(sku string) (int, error) { // Extract the generation number from the SKU string (e.g., "Standard_D2s_v3" -> 3) sku = strings.ToLower(sku) diff --git a/e2e/validators.go b/e2e/validators.go index 2a7c14da38e..74613b6d1f6 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -989,7 +989,7 @@ func ValidateNoFailedSystemdUnits(ctx context.Context, s *Scenario) { Name string `json:"unit,omitempty"` } var failedUnits []systemdUnit - result := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl list-units --failed --output json", 0, fmt.Sprintf("unable to list failed systemd units")) + result := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl list-units --failed --output json", 0, "unable to list failed systemd units") assert.NoError(s.T, json.Unmarshal([]byte(result.stdout), &failedUnits), `unable to parse and unmarshal "systemctl list-units" command output`) failedUnits = lo.Filter(failedUnits, func(unit systemdUnit, _ int) bool { if unitFailureAllowList[unit.Name] { diff --git a/parts/common/components.json b/parts/common/components.json index e5e3d5d3c5b..ffc5548be57 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -953,6 +953,14 @@ "windowsDownloadLocation": "c:\\akse-cache\\aks-secure-tls-bootstrap-client\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.1.4-ubuntu26.04u2" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1055,6 +1063,14 @@ "latestVersion": "1.4.3-ubuntu24.04u1" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=moby-runc, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.4.3-ubuntu26.04u2" + } + ] } } } @@ -1065,6 +1081,14 @@ "windowsDownloadLocation": "c:\\akse-cache\\containerd\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=moby-containerd, repository=production, os=ubuntu, release=26.04", + "latestVersion": "2.3.2-ubuntu26.04u2" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1254,6 +1278,14 @@ "latestVersion": "1.9.0-ubuntu24.04u1" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=containernetworking-plugins, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.9.0-ubuntu26.04u3" + } + ] } }, "azurelinux": { @@ -1357,6 +1389,14 @@ "latestVersion": "1.34.0-ubuntu24.04u4" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.34.0-ubuntu26.04u5" + } + ] } }, "mariner": { @@ -1415,6 +1455,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\kubelet\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.2-ubuntu26.04u2", + "previousLatestVersion": "1.36.1-ubuntu26.04u3" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1513,6 +1563,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\kubectl\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.2-ubuntu26.04u5", + "previousLatestVersion": "1.36.1-ubuntu26.04u9" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1664,6 +1724,11 @@ "downloadURL": "mcr.microsoft.com/oss/binaries/kubernetes/kubernetes-node:${version}-linux-${CPU_ARCH}", "windowsDownloadURL": "https://packages.aks.azure.com/kubernetes/${version}/windowszip/${version}-1int.zip" } + }, + "ubuntu": { + "r2604": { + "versionsV2": [] + } } } }, @@ -1703,6 +1768,11 @@ "downloadURL": "mcr.microsoft.com/oss/binaries/kubernetes/azure-acr-credential-provider:${version}-linux-${CPU_ARCH}" } }, + "ubuntu": { + "r2604": { + "versionsV2": [] + } + }, "flatcar": { "current": { "versionsV2": [] @@ -1715,6 +1785,14 @@ "downloadLocation": "/opt/azure/ig", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=ig, repository=production, os=ubuntu, release=26.04", + "latestVersion": "0.53.2-ubuntu26.04u4" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1758,6 +1836,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\azure-acr-credential-provider\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.3-ubuntu26.04u2", + "previousLatestVersion": "1.36.2-ubuntu26.04u2" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1962,11 +2050,19 @@ "downloadLocation": "/opt/nvidia-device-plugin/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=nvidia-device-plugin, repository=production, os=ubuntu, release=26.04", + "latestVersion": "0.19.3-ubuntu26.04u6" + } + ] + }, "r2404": { "versionsV2": [ { "renovateTag": "name=nvidia-device-plugin, repository=production, os=ubuntu, release=24.04", - "latestVersion": "0.19.3-ubuntu24.04u3" + "latestVersion": "0.19.3-ubuntu24.04u6" } ] }, @@ -1974,7 +2070,7 @@ "versionsV2": [ { "renovateTag": "name=nvidia-device-plugin, repository=production, os=ubuntu, release=22.04", - "latestVersion": "0.19.3-ubuntu22.04u3" + "latestVersion": "0.19.3-ubuntu22.04u6" } ] } @@ -1984,7 +2080,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=nvidia-device-plugin, os=azurelinux, release=3.0", - "latestVersion": "0.19.3-3.azl3" + "latestVersion": "0.19.3-6.azl3" } ] } @@ -1996,6 +2092,14 @@ "downloadLocation": "/opt/dra-driver-nvidia-gpu/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=dra-driver-nvidia-gpu, repository=production, os=ubuntu, release=26.04", + "latestVersion": "0.4.1-ubuntu26.04u5" + } + ] + }, "r2404": { "versionsV2": [ { @@ -2030,11 +2134,19 @@ "downloadLocation": "/opt/datacenter-gpu-manager-4-core/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=26.04", + "latestVersion": "1:4.6.0-1" + } + ] + }, "r2404": { "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=24.04", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] }, @@ -2042,7 +2154,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=22.04", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] } @@ -2052,7 +2164,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=azurelinux, release=3.0", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] } @@ -2064,11 +2176,19 @@ "downloadLocation": "/opt/datacenter-gpu-manager-4-proprietary/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=ubuntu, release=26.04", + "latestVersion": "1:4.6.0-1" + } + ] + }, "r2404": { "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=ubuntu, release=24.04", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] }, @@ -2076,7 +2196,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=ubuntu, release=22.04", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] } @@ -2086,7 +2206,7 @@ "versionsV2": [ { "renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=azurelinux, release=3.0", - "latestVersion": "1:4.5.3-1" + "latestVersion": "1:4.6.0-1" } ] } @@ -2098,11 +2218,19 @@ "downloadLocation": "/opt/dcgm-exporter/downloads", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=dcgm-exporter, repository=production, os=ubuntu, release=26.04", + "latestVersion": "4.8.3-ubuntu26.04u1" + } + ] + }, "r2404": { "versionsV2": [ { "renovateTag": "name=dcgm-exporter, repository=production, os=ubuntu, release=24.04", - "latestVersion": "4.8.2-ubuntu24.04u9" + "latestVersion": "4.8.3-ubuntu24.04u1" } ] }, @@ -2110,7 +2238,7 @@ "versionsV2": [ { "renovateTag": "name=dcgm-exporter, repository=production, os=ubuntu, release=22.04", - "latestVersion": "4.8.2-ubuntu22.04u9" + "latestVersion": "4.8.3-ubuntu22.04u1" } ] } @@ -2120,7 +2248,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=dcgm-exporter, os=azurelinux, release=3.0", - "latestVersion": "4.8.2-9.azl3" + "latestVersion": "4.8.3-1.azl3" } ] } @@ -2132,6 +2260,14 @@ "downloadLocation": "/opt/node-exporter", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.9.1-ubuntu26.04u21" + } + ] + }, "r2404": { "versionsV2": [ { @@ -2358,6 +2494,14 @@ "latestVersion": "2.5.4" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=blobfuse2, repository=production, os=ubuntu, release=26.04", + "latestVersion": "2.5.3" + } + ] } } } diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index 07d128156c3..c09c023d9a5 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -88,6 +88,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh b/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh index 9193afdfde3..0874dde7d3e 100644 --- a/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh +++ b/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh @@ -89,6 +89,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/cis.sh b/parts/linux/cloud-init/artifacts/cis.sh index dcdd110c9f6..628494ac193 100644 --- a/parts/linux/cloud-init/artifacts/cis.sh +++ b/parts/linux/cloud-init/artifacts/cis.sh @@ -8,8 +8,18 @@ assignRootPW() { VERSION=$(grep DISTRIB_RELEASE /etc/*-release | cut -f 2 -d "=") SALT=$(openssl rand -base64 5) SECRET=$(openssl rand -base64 37) + CMD="import crypt, getpass, pwd; print(crypt.crypt('$SECRET', '\$6\$$SALT\$'))" - if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then + if [ "${VERSION}" = "26.04" ]; then + # Ubuntu 26.04 ships Python 3.14, which removed the 'crypt' module (removed in 3.13). + # passlib's sha512_crypt has a pure-Python backend that produces the same SHA-512 ($6$) + # hash without needing the crypt module. rounds=5000 matches the crypt(3) default cost. + # SALT is base64 and may contain chars (e.g. '+', '=') outside passlib's salt alphabet, + # so filter it to sha512_crypt.salt_chars before use. + CMD="import getpass, pwd; from passlib.hash import sha512_crypt; salt=''.join(c for c in '$SALT' if c in sha512_crypt.salt_chars); print(sha512_crypt.using(salt=salt, rounds=5000).hash('$SECRET'))" + fi + + if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ] || [ "${VERSION}" = "26.04" ]; then HASH=$(python3 -c "$CMD") else HASH=$(python -c "$CMD") @@ -269,7 +279,34 @@ EOF systemctl restart ssh } +# Ensures that the classic (Todd Miller) sudo is the selected sudo implementation +# rather than sudo-rs. On Ubuntu 26.04 sudo-rs is installed and enabled +# by default, but sudo-rs does NOT support the `Defaults logfile` sudoers setting that +# configureSudo relies on -- per the sudo-rs docs, "logfile is not supported; logging +# is always done via syslog". Leaving sudo-rs as the default would therefore be a CIS +# regression: we would lose the dedicated /var/log/sudo.log audit trail, and sudo-rs +# would also print "unknown setting: 'logfile'" on every sudo invocation. Both classic +# sudo and sudo-rs ship on the base image and are switchable via update-alternatives, +# so select classic sudo here so the logfile setting below is actually honored. +# This is a no-op on releases where sudo is not an update-alternatives group (e.g. +# 22.04/24.04, which only have classic sudo). +# TODO: revisit if/when sudo-rs gains support for file-based logging. +preferClassicSudo() { + # Classic sudo registers itself as the "/usr/bin/sudo.ws" alternative candidate + # (sudo.ws = Todd Miller's sudo). Only relevant where sudo is update-alternatives + # managed (i.e. sudo-rs is present); a no-op on releases that ship only classic sudo. + local classic_sudo="/usr/bin/sudo.ws" + if [ -x "${classic_sudo}" ] && update-alternatives --list sudo >/dev/null 2>&1; then + update-alternatives --set sudo "${classic_sudo}" >/dev/null 2>&1 || echo "warning: failed to set ${classic_sudo} as the default sudo implementation" + fi +} + configureSudo() { + # sudo-rs (the default on Ubuntu 26.04) can only log to syslog and rejects the + # `Defaults logfile` setting below. Select classic sudo first so CIS file-based + # sudo logging works; see preferClassicSudo for details. + preferClassicSudo + cat </etc/sudoers.d/99-cis Defaults logfile="/var/log/sudo.log" EOF diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index b5272f210f1..fa0e797d2ea 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -321,8 +321,8 @@ disableSystemdResolved() { ls -ltr /etc/resolv.conf cat /etc/resolv.conf UBUNTU_RELEASE=$(lsb_release -r -s 2>/dev/null || echo "") - if [ "${UBUNTU_RELEASE}" = "20.04" ] || [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ]; then - echo "Ingoring systemd-resolved query service but using its resolv.conf file" + if [ "${UBUNTU_RELEASE}" = "20.04" ] || [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then + echo "Ignoring systemd-resolved query service but using its resolv.conf file" echo "This is the simplest approach to workaround resolved issues without completely uninstall it" [ -f /run/systemd/resolve/resolv.conf ] && ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf ls -ltr /etc/resolv.conf diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 77bffdc48f0..27260f8ac20 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -171,11 +171,11 @@ ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network inte # For unit tests, the OS and OS_VERSION will be set in the unit test script. # So whether it's if or else actually doesn't matter to our unit test. if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then - OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') - OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') - OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') + OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') + OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') + OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') else -# This is only for unit test purpose. For example, a Mac OS dev box doesn't have /etc/*-release, then the unit test will continue. + # This is only for unit test purpose. For example, a Mac OS dev box doesn't have /etc/*-release, then the unit test will continue. echo "/etc/*-release not found" fi @@ -970,6 +970,13 @@ isUbuntu() { return 1 } +isMinimalImage() { + if grep -q "minimal" <<< "$FEATURE_FLAGS"; then + return 0 + fi + return 1 +} + evalPackageDownloadURL() { local url=${1:-} if [ -n "$url" ]; then diff --git a/parts/linux/cloud-init/artifacts/cse_install.sh b/parts/linux/cloud-init/artifacts/cse_install.sh index 0958e3bab86..77f87136032 100755 --- a/parts/linux/cloud-init/artifacts/cse_install.sh +++ b/parts/linux/cloud-init/artifacts/cse_install.sh @@ -15,8 +15,8 @@ K8S_PRIVATE_PACKAGES_CACHE_DIR="/opt/kubernetes/downloads/private-packages" K8S_REGISTRY_REPO="oss/binaries/kubernetes" UBUNTU_RELEASE=$(lsb_release -r -s 2>/dev/null || echo "") # For Mariner 2.0, this returns "MARINER" and for AzureLinux 3.0, this returns "AZURELINUX" -OS=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"'; fi) -OS_VARIANT=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"'; fi) +OS=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]'; fi) +OS_VARIANT=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]'; fi) SECURE_TLS_BOOTSTRAP_CLIENT_DOWNLOAD_DIR="/opt/aks-secure-tls-bootstrap-client/downloads" SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR="/opt/bin" CREDENTIAL_PROVIDER_DOWNLOAD_DIR="/opt/credentialprovider/downloads" diff --git a/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh b/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh index cf45aba65d3..fbdea58bf58 100755 --- a/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh +++ b/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh @@ -73,6 +73,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh index 83d2c6425ad..3ca452e3926 100755 --- a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh +++ b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh @@ -674,6 +674,10 @@ cleanUpGPUDrivers() { done } +installMinimalBuildDeps() { + echo "installMinimalBuildDeps not implemented for mariner" +} + downloadContainerdFromVersion() { echo "downloadContainerdFromVersion not implemented for mariner" } diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index fbe169be48e..6a6ff548896 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -4,6 +4,33 @@ removeContainerd() { apt_get_purge 10 5 300 moby-containerd } +# Batch install all packages in a single apt_get_install call instead of looping one-by-one. +# On failure, fall back to individual installs for diagnostic clarity. A return code of 2 from +# apt_get_install signals a CSE timeout and is propagated immediately by exiting the script. +aptGetBatchInstallPackagesWithFallback() { + local -a pkg_list=("$@") + + apt_get_install 30 1 600 "${pkg_list[@]}" + local batch_rc=$? + if [ "$batch_rc" -eq 2 ]; then + exit "$batch_rc" + elif [ "$batch_rc" -ne 0 ]; then + echo "Batch install failed, falling back to individual package install" + local apt_package + for apt_package in "${pkg_list[@]}"; do + apt_get_install 30 1 600 "$apt_package" + local pkg_rc=$? + if [ "$pkg_rc" -eq 2 ]; then + exit "$pkg_rc" + elif [ "$pkg_rc" -ne 0 ]; then + tail -n 200 /var/log/apt/term.log || true + tail -n 200 /var/log/dpkg.log || true + exit $ERR_APT_INSTALL_TIMEOUT + fi + done + fi +} + blobfuseFallbackPackages() { local OSVERSION="${1}" # blobfuse/blobfuse2 started to be centralized in components.json around April 2026. @@ -40,6 +67,32 @@ blobfuseFallbackPackages() { fi } +# Installs any required dependencies needed to build the particular Ubuntu minimal image (currently only 26.04) +installMinimalBuildDeps() { + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + + if [ "${OSVERSION}" = "26.04" ]; then + installUbuntu2604MinimalBuildDeps + return 0 + fi + + echo "Unrecognized Ubuntu minimal version ${OSVERSION} - cannot install minimal build dependencies" + exit 1 +} + +installUbuntu2604MinimalBuildDeps() { + wait_for_apt_locks + retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT + retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL + + holdWALinuxAgent hold + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + + local -a pkg_list=(rsyslog gpg) + aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" +} + installDeps() { wait_for_apt_locks retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT @@ -48,44 +101,31 @@ installDeps() { holdWALinuxAgent hold apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - pkg_list=(apparmor-utils bind9-dnsutils ca-certificates ceph-common cgroup-lite cifs-utils conntrack cracklib-runtime ebtables ethtool glusterfs-client htop init-system-helpers inotify-tools iotop iproute2 ipset iptables nftables jq libpam-pwquality libpwquality-tools mount nfs-common pigz socat sysfsutils sysstat util-linux xz-utils netcat-openbsd zip rng-tools kmod gcc make dkms initramfs-tools linux-headers-$(uname -r) linux-modules-extra-$(uname -r)) + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + + pkg_list=(apparmor-utils bind9-dnsutils ca-certificates ceph-common cgroup-lite cifs-utils conntrack cracklib-runtime ebtables ethtool glusterfs-client htop init-system-helpers inotify-tools iotop iproute2 ipset iptables nftables jq libpam-pwquality libpwquality-tools mount nfs-common pigz socat sysfsutils sysstat util-linux xz-utils netcat-openbsd zip rng-tools kmod gcc make dkms initramfs-tools linux-headers-$(uname -r)) + + if [ "${OSVERSION}" != "26.04" ]; then + # linux-modules-extra-* content is bundled into linux-modules-* on 26.04 (resolute) + pkg_list+=(linux-modules-extra-$(uname -r)) + fi - local OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") while IFS= read -r fallback_pkg; do [ -n "${fallback_pkg}" ] && pkg_list+=("${fallback_pkg}") done < <(blobfuseFallbackPackages "${OSVERSION}") - if [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then pkg_list+=(irqbalance) fi - if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then pkg_list+=("aznfs=3.0.19") fi - # Batch install all packages in a single apt_get_install call instead of - # looping one-by-one. On failure, fall back to individual installs for - # diagnostic clarity. Exit immediately on return code 2 (CSE timeout). - apt_get_install 30 1 600 "${pkg_list[@]}" - local batch_rc=$? - if [ "$batch_rc" -eq 2 ]; then - exit "$batch_rc" - elif [ "$batch_rc" -ne 0 ]; then - echo "Batch install failed, falling back to individual package install" - for apt_package in "${pkg_list[@]}"; do - apt_get_install 30 1 600 "$apt_package" - local pkg_rc=$? - if [ "$pkg_rc" -eq 2 ]; then - exit "$pkg_rc" - elif [ "$pkg_rc" -ne 0 ]; then - tail -n 200 /var/log/apt/term.log || true - tail -n 200 /var/log/dpkg.log || true - exit $ERR_APT_INSTALL_TIMEOUT - fi - done - fi + aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" - if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then # disable aznfswatchdog since aznfs install and enable aznfswatchdog and aznfswatchdogv4 services at the same time while we only need aznfswatchdogv4 systemctl disable aznfswatchdog systemctl stop aznfswatchdog @@ -93,6 +133,9 @@ installDeps() { } updateAptWithMicrosoftPkg() { + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + retrycmd_silent 120 5 25 curl https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/prod.list > /tmp/microsoft-prod.list || exit $ERR_MOBY_APT_LIST_TIMEOUT retrycmd_if_failure 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit $ERR_MOBY_APT_LIST_TIMEOUT @@ -100,6 +143,13 @@ updateAptWithMicrosoftPkg() { retrycmd_silent 120 5 25 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT retrycmd_if_failure 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + + if [ "${OSVERSION}" = "26.04" ]; then + # Ubuntu 26.04 (Resolute) PMC repo is signed with Microsoft's newer 2025 gpg key + retrycmd_silent 120 5 25 curl https://packages.microsoft.com/keys/microsoft-2025.asc | gpg --dearmor > /tmp/microsoft-2025.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft-2025.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + fi + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT } @@ -132,7 +182,7 @@ updatePMCRepository() { } updateAptWithNvidiaPkg() { - readonly nvidia_gpg_keyring_path="/etc/apt/keyrings/nvidia.pub" + readonly nvidia_gpg_keyring_path="/etc/apt/keyrings/nvidia.gpg" mkdir -p "$(dirname "${nvidia_gpg_keyring_path}")" readonly nvidia_sources_list_path="/etc/apt/sources.list.d/nvidia.list" @@ -153,6 +203,8 @@ updateAptWithNvidiaPkg() { nvidia_ubuntu_release="ubuntu2204" elif [ "${UBUNTU_RELEASE}" = "24.04" ]; then nvidia_ubuntu_release="ubuntu2404" + elif [ "${UBUNTU_RELEASE}" = "26.04" ]; then + nvidia_ubuntu_release="ubuntu2604" else echo "NVIDIA repo setup is not supported on Ubuntu ${UBUNTU_RELEASE}" return @@ -162,10 +214,21 @@ updateAptWithNvidiaPkg() { echo "deb [arch=${cpu_arch} signed-by=${nvidia_gpg_keyring_path}] https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch} /" > ${nvidia_sources_list_path} # Add NVIDIA repository - local nvidia_gpg_key_url="https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch}/3bf863cc.pub" - - # Download and add the GPG key for the NVIDIA repository - retrycmd_curl_file 120 5 25 ${nvidia_gpg_keyring_path} ${nvidia_gpg_key_url} 300 || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + local nvidia_gpg_key_name="3bf863cc.pub" + if [ "${UBUNTU_RELEASE}" = "26.04" ]; then + nvidia_gpg_key_name="60DF8A40.pub" + fi + local nvidia_gpg_key_url="https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch}/${nvidia_gpg_key_name}" + + # Download the armored NVIDIA repo key and dearmor it into a binary keyring. + # apt only accepts a signed-by keyring with a .gpg (binary) or .asc (armored) extension; + # NVIDIA publishes an ASCII-armored *.pub, so a raw .pub file is rejected as an "unsupported + # filetype" and the key is ignored (the repo then fails to verify with NO_PUBKEY). Newer apt + # (e.g. 3.x on Ubuntu 26.04) enforces this strictly, so dearmor to nvidia.gpg. + local nvidia_gpg_key_tmp="/tmp/${nvidia_gpg_key_name}" + retrycmd_curl_file 120 5 25 "${nvidia_gpg_key_tmp}" "${nvidia_gpg_key_url}" 300 || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + gpg --dearmor < "${nvidia_gpg_key_tmp}" > "${nvidia_gpg_keyring_path}" || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + rm -f "${nvidia_gpg_key_tmp}" apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT } @@ -225,9 +288,13 @@ removeNvidiaRepos() { rm -f /etc/apt/sources.list.d/nvidia.list echo "Removed NVIDIA apt repository" fi + if [ -f /etc/apt/keyrings/nvidia.gpg ]; then + rm -f /etc/apt/keyrings/nvidia.gpg + echo "Removed NVIDIA GPG key (nvidia.gpg)" + fi if [ -f /etc/apt/keyrings/nvidia.pub ]; then rm -f /etc/apt/keyrings/nvidia.pub - echo "Removed NVIDIA GPG key" + echo "Removed NVIDIA GPG key (nvidia.pub)" fi } diff --git a/schemas/components.cue b/schemas/components.cue index 442f736d96e..88cd1a8902a 100644 --- a/schemas/components.cue +++ b/schemas/components.cue @@ -54,6 +54,7 @@ package components r2004?: #ReleaseDownloadURI r2204?: #ReleaseDownloadURI r2404?: #ReleaseDownloadURI + r2604?: #ReleaseDownloadURI } #DefaultOSDistro: { diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 3762d1141c5..db5d08c8653 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -14,13 +14,13 @@ ACL_OS_NAME="AZURECONTAINERLINUX" # Real world examples from the command outputs # For Azure Linux V3: ID=azurelinux VERSION_ID="3.0" # For Azure Linux V2: ID=mariner VERSION_ID="2.0" -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') IS_KATA="false" if grep -q "kata" <<< "$FEATURE_FLAGS"; then IS_KATA="true" fi -OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') +OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') THIS_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)" @@ -38,207 +38,13 @@ CPU_ARCH=$(getCPUArch) #amd64 or arm64 SYSTEMD_ARCH=$(getSystemdArch) # x86-64 or arm64 VHD_LOGS_FILEPATH=/opt/azure/vhd-install.complete COMPONENTS_FILEPATH=/opt/azure/components.json +LOCALDNS_BINARY_PATH="/opt/azure/containers/localdns/binary" PERFORMANCE_DATA_FILE=/opt/azure/vhd-build-performance-data.json GRID_COMPATIBILITY_DATA_FILE=/opt/azure/vhd-grid-compatibility-data.json -resolve_packages_source_url - -echo "" -echo "Components downloaded in this VHD build (some of the below components might get deleted during cluster provisioning if they are not needed):" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" - -echo "Logging the kernel after purge and reinstall + reboot: $(uname -r)" -# fix grub issue with cvm by reinstalling before other deps -# other VHDs use grub-pc, not grub-efi -if [ "$OS" = "$UBUNTU_OS_NAME" ] && echo "$FEATURE_FLAGS" | grep -q "cvm"; then - apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - wait_for_apt_locks - apt_get_install 10 2 120 grub-efi || exit 1 -fi -capture_benchmark "${SCRIPT_NAME}_reinstall_grub_for_cvm" - -if [ "$OS" = "$UBUNTU_OS_NAME" ]; then - # disable and mask all UU timers/services - # save some background io/latency - systemctl mask apt-daily.service apt-daily-upgrade.service || exit 1 - systemctl disable apt-daily.service apt-daily-upgrade.service || exit 1 - systemctl disable apt-daily.timer apt-daily-upgrade.timer || exit 1 - - tee /etc/apt/apt.conf.d/99periodic > /dev/null <> "${CHRONYD_DIR}"/10-chrony-restarts.conf < /dev/null <<'EOF' -Compress=yes -Storage=persistent -SystemMaxUse=1G -RuntimeMaxUse=1G -ForwardToSyslog=yes -EOF -capture_benchmark "${SCRIPT_NAME}_install_deps_and_set_configs" - -if [ "$(isARM64)" -eq 1 ]; then - # shellcheck disable=SC3010 - if [[ ${HYPERV_GENERATION,,} == "v1" ]]; then - echo "No arm64 support on V1 VM, exiting..." - exit 1 - fi -fi - -# Always override network config and disable NTP + Timesyncd and install Chrony -# Mariner does this differently, so only do it for Ubuntu -if ! isMarinerOrAzureLinux "$OS"; then - overrideNetworkConfig || exit 1 - disableNtpAndTimesyncdInstallChrony || exit 1 -fi - -# ACL inherits Azure Linux behaviors but isMarinerOrAzureLinux returns false, -# so these must be called separately (mirrored in the Mariner/AzureLinux block below). -# Other Mariner functions are safe to skip for ACL: -# setMarinerNetworkdConfig — ACL doesn't ship systemd-bootstrap's 99-dhcp-en.network -# fixCBLMarinerPermissions — product_uuid already 444; no rsyslog on ACL -# addMarinerNvidiaRepo / updateDnfWithNvidiaPkg / disableDNFAutomatic / enableCheckRestart — ACL has no dnf/rpm -# activateNfConntrack — nf_conntrack auto-loads via iptables dependency chain -# disableTimesyncd — ACL handles chrony separately above via disableNtpAndTimesyncdInstallChrony -if isACL "$OS" "$OS_VARIANT"; then - # ACL's iptables.service loads host firewall rules that conflict with Cilium eBPF routing. - disableSystemdIptables || exit 1 - # Repoint /etc/resolv.conf from the stub (127.0.0.53) to the real upstream file - # so DNS queries go directly through localdns. - disableSystemdResolvedCache -fi -capture_benchmark "${SCRIPT_NAME}_validate_container_runtime_and_override_ubuntu_net_config" - -# Configure SSH service during VHD build for Ubuntu 22.10+ -configureSSHService "$OS" "$OS_VERSION" || echo "##vso[task.logissue type=warning]SSH Service configuration failed, but continuing VHD build" - -CONTAINERD_SERVICE_DIR="/etc/systemd/system/containerd.service.d" -mkdir -p "${CONTAINERD_SERVICE_DIR}" -# Explicitly set LimitNOFILE=1048576 (the value that 'infinity' resolves to on Ubuntu 22.04) for both Ubuntu and Mariner/AzureLinux. -# On Ubuntu 24.04 (Containerd 2.0), LimitNOFILE is removed upstream and systemd falls back to an implicit soft:hard limit -# (for example 1024:524288), so containerd inherits a very low soft file descriptor limit (1024) unless we override it here. -# On Mariner/AzureLinux this is redundant with the base containerd.service unit but harmless. -# Not removing LimitNOFILE from parts/linux/cloud-init/artifacts/containerd.service, -# to avoid compatibility issues between new VHDs and old CSE scripts. -tee "${CONTAINERD_SERVICE_DIR}/exec_start.conf" > /dev/null < /dev/null < /etc/udev/rules.d/99-nfs.rules < /etc/udev/rules.d/80-azure-disk.rules <, then skip and move on to next package - if [ "${#PACKAGE_VERSIONS[@]}" -eq 0 ] || [ "${PACKAGE_VERSIONS[0]}" = "" ]; then - echo "INFO: ${name} package versions array is either empty or the first element is . Skipping ${name} installation." - continue +installAndConfigureArtifactStreaming() { + local downloadURL="$1" + local version="$2" + # The arm64 packages have "-arm64" inserted before the file extension, + # e.g. acr-mirror-2204-arm64.deb instead of acr-mirror-2204.deb + if [ "$(isARM64)" -eq 1 ]; then + downloadURL="${downloadURL%.*}-arm64.${downloadURL##*.}" fi - downloadDir=$(echo "${p}" | jq .downloadLocation -r) - #download the package - case $name in - "kubernetes-cri-tools") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadAndInstallCriTools "${downloadDir}" "${evaluatedURL}" "${version}" - done - ;; - "azure-cni") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadAzureCNI "${downloadDir}" "${evaluatedURL}" - unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" - echo " - Azure CNI version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "cni-plugins") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadCNIPlugins "${downloadDir}" "${evaluatedURL}" - unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" - echo " - CNI plugin version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "containernetworking-plugins") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installCNI "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - containernetworking-plugins version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "runc") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - ensureRunc "${version}" "${evaluatedURL}" "${downloadDir}" - echo " - runc version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "containerd") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - if [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then - installContainerd "${downloadDir}" "${evaluatedURL}" "${version}" - elif isMarinerOrAzureLinux "$OS"; then - installStandaloneContainerd "${version}" - fi - echo " - containerd version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "oras") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installOras "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - oras version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "aks-secure-tls-bootstrap-client") - for version in ${PACKAGE_VERSIONS[@]}; do - # removed at provisioning time if secure TLS bootstrapping is disabled - if isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - installPackageFromCache "${name}" "${version}" "/opt/bin/${name}" || exit $? - elif isMarinerOrAzureLinux; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - installRPMPackageFromFile "${name}" "${version}" "/opt/bin/${name}" || exit $? - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? - installSecureTLSBootstrapClientSysext "${version}" || exit $? - fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "azure-acr-credential-provider") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadCredentialProvider "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - azure-acr-credential-provider version ${version}" >> ${VHD_LOGS_FILEPATH} - # ORAS will be used to install other packages for network isolated clusters, it must go first. - done - ;; - "inspektor-gadget") - if isMariner "$OS" || isFlatcar "$OS" || isACL "$OS" "$OS_VARIANT" || isAzureLinuxOSGuard "$OS" "$OS_VARIANT" || [ "${IS_KATA}" = "true" ]; then - echo "Skipping inspektor-gadget installation for ${OS} ${OS_VARIANT:-default} (IS_KATA=${IS_KATA})" - else - ig_version="${PACKAGE_VERSIONS[0]}" - if isUbuntu "$OS"; then - # Ubuntu: download ig deb via apt; ig_install_deb_stack expects it at downloadDir - downloadPkgFromVersion "ig" "${ig_version}" "${downloadDir}" - installIG "${ig_version}" "${downloadDir}" - elif isAzureLinux "$OS"; then - # Azure Linux 3.0: ig_install_rpm_stack handles its own RPM downloads - installIG "${ig_version}" "${downloadDir}" - fi - fi - ;; - "kubernetes-binaries") - # kubelet and kubectl - # need to cover previously supported version for VMAS scale up scenario - # So keeping as many versions as we can - those unsupported version can be removed when we don't have enough space - # NOTE that we only keep the latest one per k8s patch version as kubelet/kubectl is decided by VHD version - # Please do not use the .1 suffix, because that's only for the base image patches - # regular version >= v1.17.0 or hotfixes >= 20211009 has arm64 binaries. - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - extractKubeBinaries "${version}" "${evaluatedURL}" false "${downloadDir}" - echo " - kubernetes-binaries version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - azure-acr-credential-provider-pmc) - name=${name%-pmc} - for version in ${PACKAGE_VERSIONS[@]}; do - if isMarinerOrAzureLinux || isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? - fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - kubelet|kubectl) - for version in ${PACKAGE_VERSIONS[@]}; do - if isMarinerOrAzureLinux || isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - cacheVersionedKubernetesPackageBinary "${name}" "${version}" "${downloadDir}" || exit $ERR_K8S_INSTALL_ERR - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? - fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done + local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" + retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} + case "$downloadURL" in + *.deb) + apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD ;; - "${K8S_DEVICE_PLUGIN_PKG}") - for version in ${PACKAGE_VERSIONS[@]}; do - if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isMarinerOrAzureLinux "$OS"; then - downloadPkgFromVersion "${K8S_DEVICE_PLUGIN_PKG}" "${version}" "${downloadDir}" - fi - echo " - ${K8S_DEVICE_PLUGIN_PKG} version ${version}" >> ${VHD_LOGS_FILEPATH} - done + *.rpm) + dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD ;; - "dra-driver-nvidia-gpu") - for version in ${PACKAGE_VERSIONS[@]}; do - if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isAzureLinux "$OS"; then - downloadPkgFromVersion "dra-driver-nvidia-gpu" "${version}" "${downloadDir}" - fi - echo " - dra-driver-nvidia-gpu version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "datacenter-gpu-manager-4-core") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "datacenter-gpu-manager-4-core" "${version}" "${downloadDir}" - echo " - datacenter-gpu-manager-4-core version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "datacenter-gpu-manager-4-proprietary") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "datacenter-gpu-manager-4-proprietary" "${version}" "${downloadDir}" - echo " - datacenter-gpu-manager-4-proprietary version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "dcgm-exporter") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" - echo " - dcgm-exporter version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "node-exporter") - # Skipping is handled by empty versionsV2 arrays in components.json - # for mariner, flatcar, acl, and osguard. Kata is skipped explicitly here. - if [ "${IS_KATA}" = "true" ]; then - echo "Skipping node-exporter installation for kata (IS_KATA=${IS_KATA})" - else - # Download and install node-exporter-kubernetes at VHD build time. - # node-exporter is installed on the VHD so CSE only needs to enable+start it. - installNodeExporter "${PACKAGE_VERSIONS[0]}" - fi - ;; - "acr-mirror") - # acr-mirror is handled separately below via installAndConfigureArtifactStreaming. - ;; - "aznfs") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL "${PACKAGE_DOWNLOAD_URL}") - mkdir -p "${downloadDir}" - aznfsFilename=$(basename "${evaluatedURL}") - echo "Downloading aznfs RPM from ${evaluatedURL} to ${downloadDir}/${aznfsFilename}" - retrycmd_curl_file 120 5 25 "${downloadDir}/${aznfsFilename}" "${evaluatedURL}" || exit $ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT - echo " - aznfs version ${version}" >> ${VHD_LOGS_FILEPATH} - done - installAznfsPackage || exit $ERR_AZNFS_INSTALL_FAIL - ;; - "blobfuse"|"blobfuse2") - for version in "${PACKAGE_VERSIONS[@]}"; do - if isUbuntu "$OS"; then - if ! apt_get_install 10 2 120 "${name}=${version}"; then - journalctl --no-pager -u "${name}" || true - tail -n 200 /var/log/apt/term.log || true - tail -n 200 /var/log/dpkg.log || true - exit $ERR_APT_INSTALL_TIMEOUT - fi - echo " - ${name} version ${version}" >> "${VHD_LOGS_FILEPATH}" - else - echo " - ${name} installation skipped for ${OS}" >> "${VHD_LOGS_FILEPATH}" - fi - done - ;; - *) - echo "Package name: ${name} not supported for download. Please implement the download logic in the script." - # We can add a common function to download a generic package here. - # However, installation could be different for different packages. - ;; - esac - capture_benchmark "${SCRIPT_NAME}_download_${name}" -done <<< "$packages" - -installAndConfigureArtifactStreaming() { - local downloadURL="$1" - local version="$2" - # The arm64 packages have "-arm64" inserted before the file extension, - # e.g. acr-mirror-2204-arm64.deb instead of acr-mirror-2204.deb - if [ "$(isARM64)" -eq 1 ]; then - downloadURL="${downloadURL%.*}-arm64.${downloadURL##*.}" - fi - local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" - retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} - case "$downloadURL" in - *.deb) - apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD - ;; - *.rpm) - dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD - ;; - *) - echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 - exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} + *) + echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 + exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} ;; esac rm "$MIRROR_DOWNLOAD_PATH" @@ -668,257 +223,6 @@ installAndConfigureArtifactStreaming() { echo " - acr-mirror version ${version}" >> ${VHD_LOGS_FILEPATH} } -# Artifact streaming (acr-mirror) - version and URLs resolved from components.json, -# OS filtering handled declaratively by components.json entries ( for unsupported OSes). -acrMirrorPackage=$(echo "${packages}" | jq -c 'select(.name == "acr-mirror")') -updatePackageVersions "${acrMirrorPackage}" "${OS}" "${OS_VERSION}" "${OS_VARIANT}" -updatePackageDownloadURL "${acrMirrorPackage}" "${OS}" "${OS_VERSION}" "${OS_VARIANT}" -if [ "${#PACKAGE_VERSIONS[@]}" -gt 0 ] && [ "${PACKAGE_VERSIONS[0]}" != "" ]; then - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installAndConfigureArtifactStreaming "${evaluatedURL}" "${version}" - done -fi -capture_benchmark "${SCRIPT_NAME}_install_artifact_streaming" - -# k8s will use images in the k8s.io namespaces - create it -ctr namespace create k8s.io -cliTool="ctr" - -INSTALLED_RUNC_VERSION=$(runc --version | head -n1 | sed 's/runc version //') -echo " - runc version ${INSTALLED_RUNC_VERSION}" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_install_crictl" - -GPUContainerImages=$(jq -c '.GPUContainerImages[]' $COMPONENTS_FILEPATH) - -NVIDIA_DRIVER_IMAGE="" -NVIDIA_DRIVER_IMAGE_TAG="" -NVIDIA_GRID_DRIVER_VERSION="" - -# Extract GRID driver version for release notes (applicable to all Linux distributions) -while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') - # shellcheck disable=SC2001 - imageName=$(echo "$downloadURL" | sed 's/:.*$//') - - if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-grid" ]; then - NVIDIA_GRID_DRIVER_VERSION=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') - # Continue to extract CUDA driver info as well - fi -done <<< "$GPUContainerImages" - -# For Ubuntu, pre-pull the CUDA driver image -if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now - gpu_action="copy" - - while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') - # shellcheck disable=SC2001 - imageName=$(echo "$downloadURL" | sed 's/:.*$//') - - if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-cuda-lts" ]; then - latestVersion=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') - NVIDIA_DRIVER_IMAGE="$imageName" - NVIDIA_DRIVER_IMAGE_TAG="$latestVersion" - break # Exit the loop once we find the image - fi - done <<< "$GPUContainerImages" - - # Check if the NVIDIA_DRIVER_IMAGE and NVIDIA_DRIVER_IMAGE_TAG were found - if [ -z "$NVIDIA_DRIVER_IMAGE" ] || [ -z "$NVIDIA_DRIVER_IMAGE_TAG" ]; then - echo "Error: Unable to find aks-gpu-cuda-lts image in components.json" - exit 1 - fi - - mkdir -p /opt/{actions,gpu} - - /opt/azure/containers/image-fetcher "$NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG" - - cat << EOF >> ${VHD_LOGS_FILEPATH} - - nvidia-cuda-driver=${NVIDIA_DRIVER_IMAGE_TAG} -EOF - - # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the - # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles - # and DKMS-registers the kernel module + stages userspace libs against THIS VHD's kernel, - # performs NO device access (safe on the GPU-less Packer builder), and writes the marker - # /opt/azure/aks-gpu/dkms-marker. At node boot, configGPUDrivers passes "install-skip-build" - # when that marker matches, running only the device-dependent steps. - # The driver image is intentionally LEFT in the VHD: boot-time device init still sources the - # container toolkit debs, fabric manager, containerd runtime config and udev rules from it. - # Dropping the image is a separate, deferred size optimization. - if grep -q "NVIDIA_CUDA_PREBAKE" <<< "$FEATURE_FLAGS"; then - echo "Pre-building NVIDIA CUDA kernel module into the VHD (build-only) for kernel $(uname -r)" - # nvidia-installer needs gcc/make + libc6-dev to compile; the builder lacks them here, so install - # them. A boot-time fallback recompile (marker mismatch) still has them: the VHD ships - # build-essential -> libc6-dev (release-notes manifests) -- the toolchain baseline GPU nodes compile - # with at boot (installDeps runs apt --no-install-recommends, so gcc alone doesn't pull libc6-dev). - apt_get_install 10 2 300 gcc make libc6-dev || exit 1 - CTR_GPU_PREBUILD_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind" - retrycmd_if_failure 3 10 600 bash -c "$CTR_GPU_PREBUILD_CMD $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG gpuprebuild /entrypoint.sh build-only" || exit 1 - if [ ! -f /opt/azure/aks-gpu/dkms-marker ]; then - echo "Error: NVIDIA CUDA prebake did not produce /opt/azure/aks-gpu/dkms-marker" - exit 1 - fi - cat << EOF >> ${VHD_LOGS_FILEPATH} - - nvidia-cuda-driver-prebaked=${NVIDIA_DRIVER_IMAGE_TAG} (kernel $(uname -r)) -EOF - fi -fi - -if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then - # NVIDIA GB setup is only supported on arm64 Ubuntu 24.04. - if [ "${CPU_ARCH}" = "arm64" ] && [ "${UBUNTU_RELEASE}" = "24.04" ]; then - # Replicate all functionality from github.com/azure/aks-gpu/install.sh. - # aks-gpu is designed to run at node boot/join time, whereas the NVIDIA GB VHD is set up - # to have all drivers installed at VHD build time. - - # 1. Blacklist nouveau driver - cat << EOF >> /etc/modprobe.d/blacklist-nouveau.conf -blacklist nouveau -options nouveau modeset=0 -EOF - update-initramfs -u - - # 2. install drivers - BOM_PATH="gb-mai-bom.json" - - # Install a custom repository if a doca-custom-repo is specified - DOCA_CUSTOM_REPO=$(jq -r '.["doca-custom-repo"]' $BOM_PATH) - if [ -n "$DOCA_CUSTOM_REPO" ]; then - mv /etc/apt/sources.list.d/doca-net.list /etc/apt/sources.list.d/doca-net.list.backup - echo "deb [arch=arm64 signed-by=/etc/apt/keyrings/doca-net.pub] $DOCA_CUSTOM_REPO ./" > /etc/apt/sources.list.d/doca-net.list - apt-get update - fi - - # Install DOCA/OFED before the GPU driver so nvidia-peermem can build against the RDMA APIs provided by OFED. - # Farcically, nvidia-dkms-580-open cannot be installed together with the CUDA toolkit. Something about that package changes the build environment in an incompatible way. I've seen people mention CUDA including an old version of gcc that somehow makes its way onto the PATH... - # Therefore we install DOCA/OFED first, the GPU driver and its dependencies second, then all downstream reverse-dependencies (CUDA, DCGM, and so forth) third. - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave1"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave2"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave3"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - - # 3. Add char device symlinks for NVIDIA devices - mkdir -p "$(dirname /lib/udev/rules.d/71-nvidia-dev-char.rules)" - cat << EOF >> /lib/udev/rules.d/71-nvidia-dev-char.rules -ACTION=="add", DEVPATH=="/bus/pci/drivers/nvidia", RUN+="/usr/bin/nvidia-ctk system create-dev-char-symlinks --create-all" -EOF - - # Create systemd drop-in to override nvidia-device-plugin dependencies - mkdir -p /etc/systemd/system/nvidia-device-plugin.service.d - cat << EOF > /etc/systemd/system/nvidia-device-plugin.service.d/override.conf -[Unit] -After=kubelet.service - -[Service] -ExecStartPre=-/usr/bin/mkdir -p /var/lib/kubelet/device-plugins -EOF - - # Now we are off-piste: enable DCGM, DCGM exporter, container device plugin, and the NVIDIA containerd config. - systemctl enable nvidia-dcgm - systemctl enable nvidia-dcgm-exporter - systemctl enable nvidia-device-plugin - systemctl enable openibd - - # One additional request from MAI: signal that NPD is pre-installed on the VHD. - # When this file is present, the Azure AKS VM Extension skips installing NPD at provision time. - mkdir -p /etc/node-problem-detector.d/ - touch /etc/node-problem-detector.d/skip_vhd_npd - fi -fi - -if [ -d "/opt/gpu" ] && [ "$(ls -A /opt/gpu)" ]; then - ls -ltr /opt/gpu/* >> ${VHD_LOGS_FILEPATH} -fi - -installBpftrace -echo " - $(bpftrace --version)" >> ${VHD_LOGS_FILEPATH} - -PRESENT_DIR=$(pwd) -# run installBcc in a subshell and continue on with container image pull in order to decrease total build time -( - cd $PRESENT_DIR || { echo "Subshell in the wrong directory" >&2; exit 1; } - installBcc - exit $? -) > /var/log/bcc_installation.log 2>&1 & - -BCC_PID=$! - -# Add a separate section for runtime-installed components -# This clearly distinguishes components installed during CSE from VHD build-time components -# Only add for Ubuntu -if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then - cat << EOF >> ${VHD_LOGS_FILEPATH} -Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): - - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} -EOF -fi - -echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_pull_nvidia_driver_and_start_ebpf_downloads" - -string_replace() { - echo ${1//\*/$2} -} - -# Limit number of parallel pulls to 2 less than number of processor cores in order to prevent issues with network, CPU, and disk resources -# Account for possibility that number of cores is 3 or less -num_proc=$(nproc) -if [ "$num_proc" -gt 3 ]; then - parallel_container_image_pull_limit=$(nproc --ignore=2) -else - parallel_container_image_pull_limit=1 -fi -echo "Limit for parallel container image pulls set to $parallel_container_image_pull_limit" - -declare -a image_pids=() - -ContainerImages=$(jq ".ContainerImages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) -while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq .downloadURL -r) - amd64OnlyVersionsStr=$(echo "${imageToBePulled}" | jq .amd64OnlyVersions -r) - updateMultiArchVersions "${imageToBePulled}" - amd64OnlyVersions="" - if [ "${amd64OnlyVersionsStr}" != "null" ]; then - amd64OnlyVersions=$(echo "${amd64OnlyVersionsStr}" | jq -r ".[]") - fi - - if [ "$(isARM64)" -eq 1 ]; then - versions="${MULTI_ARCH_VERSIONS[*]}" - else - versions="${amd64OnlyVersions} ${MULTI_ARCH_VERSIONS[*]}" - fi - - for version in ${versions}; do - CONTAINER_IMAGE=$(string_replace $downloadURL $version) - pullContainerImage "${cliTool}" "${CONTAINER_IMAGE}" & - image_pids+=($!) - echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} - while [ "$(jobs -p | wc -l)" -ge "$parallel_container_image_pull_limit" ]; do - wait -n || { - ret=$? - echo "A background job pullContainerImage failed: ${ret}, ${CONTAINER_IMAGE}. Exiting..." >&2 - for pid in "${image_pids[@]}"; do - kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" - done - exit "${ret}" - } - done - done -done <<< "$ContainerImages" -echo "Waiting for container image pulls to finish. PID: ${image_pids[@]}" -while [ "$(jobs -p | wc -l)" -gt 0 ]; do - wait -n || { - ret=$? - echo "A background job pullContainerImage failed: ${ret}. Exiting..." >&2 - for pid in "${image_pids[@]}"; do - kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" - done - exit "${ret}" - } -done -capture_benchmark "${SCRIPT_NAME}_caching_container_images" - retagAKSNodeCAWatcher() { # This function retags the aks-node-ca-watcher image to a static tag # The static tag is used to bootstrap custom CA trust when MCR egress may be intercepted by an untrusted TLS MITM firewall. @@ -935,11 +239,8 @@ retagAKSNodeCAWatcher() { # may be intercepted by an untrusted TLS MITM firewall. watcherStaticImg=${watcherBaseImg//\*/static} - # can't use $cliTool variable because crictl doesn't support retagging. retagContainerImage "ctr" ${watcherFullImg} ${watcherStaticImg} } -retagAKSNodeCAWatcher -capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" pinPodSandboxImages() { # This function pins the pod sandbox image(s) to avoid Kubelet's Garbage Collector (GC) from removing them. @@ -981,35 +282,6 @@ pinPodSandboxImages() { done <<< "$pause_images" } -pinPodSandboxImages -capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" - -# IPv6 nftables rules are only available on Ubuntu or Mariner/AzureLinux -if [ $OS = $UBUNTU_OS_NAME ] || isMarinerOrAzureLinux "$OS"; then - systemctlEnableAndStart ipv6_nftables 30 || exit 1 -fi - -mkdir -p /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events - -# Disable cgroup-memory-telemetry on AzureLinux due to incompatibility with cgroup2fs driver and absence of required azure.slice directory -if ! isMarinerOrAzureLinux "$OS"; then - systemctlEnableAndStart cgroup-memory-telemetry.timer 30 || exit 1 - systemctl enable cgroup-memory-telemetry.service || exit 1 - systemctl restart cgroup-memory-telemetry.service -fi - -CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) -if [ "$CGROUP_VERSION" = "cgroup2fs" ]; then - systemctlEnableAndStart cgroup-pressure-telemetry.timer 30 || exit 1 - systemctl enable cgroup-pressure-telemetry.service || exit 1 - systemctl restart cgroup-pressure-telemetry.service -fi - -if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ "$(ls -A /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/)" ]; then - cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/* - rm -r /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 -fi -capture_benchmark "${SCRIPT_NAME}_configure_telemetry" # download kubernetes package from the given URL using azcopy # if it is a kube-proxy package, extract image from the downloaded package @@ -1038,7 +310,6 @@ cacheKubePackageFromPrivateUrl() { echo "download private package ${kube_private_binary_url} and store as ${cached_pkg}" if ! ./azcopy copy "${kube_private_binary_url}" "${cached_pkg}"; then - azExitCode=$? # loop through azcopy log files shopt -s nullglob for f in "${AZCOPY_LOG_LOCATION}"/*.log; do @@ -1058,38 +329,53 @@ cacheKubePackageFromPrivateUrl() { fi } -if [ "$OS" = "$UBUNTU_OS_NAME" ]; then - # remove snapd, which is not used by container stack - apt_get_purge 20 30 120 snapd || exit 1 - apt_get_purge 20 30 120 apache2-utils || exit 1 - # CIS: Ensure telnet (/ftp) client is not installed - # CIS: Ufw is not used but interferes with log_martians rule - apt_get_purge 20 30 120 telnet ftp ufw tnftp inetutils-telnet || exit 1 +startEBPFToolsInstallation() { + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - skipping eBPF tools installation" + return 0 + fi - apt-get -y autoclean || exit 1 - apt-get -y autoremove --purge || exit 1 - apt-get -y clean || exit 1 - # update message-of-the-day to start after multi-user.target - # multi-user.target usually start at the end of the boot sequence - sed -i 's/After=network-online.target/After=multi-user.target/g' /lib/systemd/system/motd-news.service -fi -capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu" + installBpftrace + echo " - $(bpftrace --version)" >> ${VHD_LOGS_FILEPATH} -wait $BCC_PID -BCC_EXIT_CODE=$? -chmod 644 /var/log/bcc_installation.log + PRESENT_DIR=$(pwd) + # run installBcc in a subshell asynchronously so we can do other things in the meantime + ( + cd $PRESENT_DIR || { echo "Subshell in the wrong directory" >&2; exit 1; } + installBcc + exit $? + ) > /var/log/bcc_installation.log 2>&1 & -if [ "$BCC_EXIT_CODE" -eq 0 ]; then - echo "Bcc tools successfully installed." - cat << EOF >> ${VHD_LOGS_FILEPATH} + BCC_PID=$! +} + +finishEBPFToolsInstallation() { + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - skipping eBPF tools installation" + # write out an empty bcc_installation.log since builds currently expect it to be present + touch /var/log/bcc_installation.log || true + chmod 644 /var/log/bcc_installation.log || true + return 0 + fi + + local BCC_EXIT_CODE=0 + wait "$BCC_PID" || BCC_EXIT_CODE=$? + chmod 644 /var/log/bcc_installation.log || true + + if [ "$BCC_EXIT_CODE" -eq 0 ]; then + echo "Bcc tools successfully installed." + cat << EOF >> ${VHD_LOGS_FILEPATH} - bcc-tools - libbcc-examples EOF -else - echo "Error: installBcc subshell failed with exit code $BCC_EXIT_CODE" >&2 - exit $BCC_EXIT_CODE -fi -capture_benchmark "${SCRIPT_NAME}_finish_installing_bcc_tools" + else + echo "Error: installBcc subshell failed with exit code $BCC_EXIT_CODE" >&2 + echo "===== BEGIN /var/log/bcc_installation.log =====" >&2 + cat /var/log/bcc_installation.log >&2 2>/dev/null || echo "(bcc_installation.log missing)" >&2 + echo "===== END /var/log/bcc_installation.log =====" >&2 + exit "$BCC_EXIT_CODE" + fi +} # Configure LSM modules to include BPF configureLsmWithBpf() { @@ -1120,7 +406,7 @@ configureLsmWithBpf() { local new_lsm="bpf,$current_lsm" echo "New LSM configuration: $new_lsm" - if [ "$OS" = "$UBUNTU_OS_NAME" ] && [ "$OS_VERSION" = "24.04" ]; then + if [ "$OS" = "$UBUNTU_OS_NAME" ] && { [ "$OS_VERSION" = "24.04" ] || [ "$OS_VERSION" = "26.04" ]; }; then local grub_cfg="/etc/default/grub.d/50-cloudimg-settings.cfg" if [ -f "$grub_cfg" ]; then if grep -q "lsm=" "$grub_cfg"; then @@ -1128,7 +414,7 @@ configureLsmWithBpf() { else sed -i "s/GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"lsm=$new_lsm /" "$grub_cfg" fi - echo "Updating GRUB configuration for Ubuntu 24.04..." + echo "Updating GRUB configuration for Ubuntu..." update-grub2 /boot/grub/grub.cfg || echo "Warning: Failed to update GRUB configuration" else echo "Warning: $grub_cfg not found, skipping LSM configuration" @@ -1155,47 +441,577 @@ configureLsmWithBpf() { fi } -configureLsmWithBpf -capture_benchmark "${SCRIPT_NAME}_configure_lsm_with_bpf" - -# use the private_packages_url to download and cache packages -if [ -n "${PRIVATE_PACKAGES_URL:-}" ]; then - IFS=',' read -ra PRIVATE_URLS <<< "${PRIVATE_PACKAGES_URL}" +setUdevRules() { + # set udev rules + echo "set read ahead size to 15380 KB" + AWK_PATH=$(command -v awk) + cat > /etc/udev/rules.d/99-nfs.rules < /etc/udev/rules.d/80-azure-disk.rules </dev/null - rm -rf "${ctr_temp}" - fi - } - trap cleanup_coredns_imports EXIT ABRT ERR INT PIPE QUIT TERM +# Use "resource" type for local SCSI because some VM skus offer NVMe local disks in addition to a SCSI resource disk, e.g. LSv3 family. +# This logic is already in walinuxagent rules but we duplicate it here to avoid an unnecessary dependency for anyone requiring it. +ATTRS{device_id}=="?00000000-0001-*", ENV{AZURE_DISK_TYPE}="resource", ENV{AZURE_DISK_LUN}="", GOTO="azure_disk_symlink" +ATTRS{device_id}=="{f8b3781a-1e82-4818-a1c3-63d806ec15bb}", ENV{AZURE_DISK_LUN}=="1", ENV{AZURE_DISK_TYPE}="resource", ENV{AZURE_DISK_LUN}="", GOTO="azure_disk_symlink" +GOTO="azure_disk_end" - # Extract available coredns image tags (v1.12.0-1 format) and sort them in descending order. - local sorted_coredns_tags=($(for image in "${coredns_image_list[@]}"; do echo "${image##*:}"; done | sort -V -r)) +LABEL="azure_disk_nvme_direct_v1" +LABEL="azure_disk_nvme_direct_v2" +ATTRS{nsid}=="?*", ENV{AZURE_DISK_TYPE}="local", ENV{AZURE_DISK_SERIAL}="\$env{ID_SERIAL_SHORT}" +GOTO="azure_disk_nvme_id" - # Determine latest version. +LABEL="azure_disk_nvme_remote_v1" +ATTRS{nsid}=="1", ENV{AZURE_DISK_TYPE}="os", GOTO="azure_disk_nvme_id" +ATTRS{nsid}=="?*", ENV{AZURE_DISK_TYPE}="data", PROGRAM="/bin/sh -ec 'echo \$\$((%s{nsid}-2))'", ENV{AZURE_DISK_LUN}="\$result" + +LABEL="azure_disk_nvme_id" +ATTRS{nsid}=="?*", IMPORT{program}="/usr/sbin/azure-nvme-id --udev" + +LABEL="azure_disk_symlink" +# systemd v254 ships an updated 60-persistent-storage.rules that would allow +# these to be deduplicated using \$env{.PART_SUFFIX} +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="os|resource|root", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}" +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_INDEX}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-index/\$env{AZURE_DISK_INDEX}" +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_LUN}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-lun/\$env{AZURE_DISK_LUN}" +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_NAME}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-name/\$env{AZURE_DISK_NAME}" +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_SERIAL}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-serial/\$env{AZURE_DISK_SERIAL}" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="os|resource|root", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}-part%n" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_INDEX}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-index/\$env{AZURE_DISK_INDEX}-part%n" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_LUN}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-lun/\$env{AZURE_DISK_LUN}-part%n" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_NAME}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-name/\$env{AZURE_DISK_NAME}-part%n" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_SERIAL}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-serial/\$env{AZURE_DISK_SERIAL}-part%n" +LABEL="azure_disk_end" +EOF + udevadm control --reload +} + +cachePackageAndBinaryComponents() { + # Download/cache all declared packages and binaries within components.json that apply to the respective OS SKU + packages=$(jq ".Packages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) + # Iterate over each element in the packages array + while IFS= read -r p; do + #getting metadata for each package + name=$(echo "${p}" | jq .name -r) + os=${OS} + # TODO(mheberling): Remove this once kata uses standard containerd. This OS is referenced + # in file `parts/common/component.json` with the same ${MARINER_KATA_OS_NAME}. + if isMariner "${OS}" && [ "${IS_KATA}" = "true" ]; then + # This is temporary for kata-cc because it uses a modified version of containerd and + # name is referenced in parts/common.json marinerkata. + os=${MARINER_KATA_OS_NAME} + fi + if isAzureLinux "${OS}" && [ "${IS_KATA}" = "true" ]; then + # This is temporary for kata-cc because it uses a modified version of containerd and + # name is referenced in parts/common.json azurelinuxkata. + os=${AZURELINUX_KATA_OS_NAME} + fi + updatePackageVersions "${p}" "${os}" "${OS_VERSION}" "${OS_VARIANT}" + updatePackageDownloadURL "${p}" "${os}" "${OS_VERSION}" "${OS_VARIANT}" + echo "In components.json, processing components.packages \"${name}\" \"${PACKAGE_VERSIONS[@]}\" \"${PACKAGE_DOWNLOAD_URL}\"" + + # if ${PACKAGE_VERSIONS[@]} count is 0 or if the first element of the array is , then skip and move on to next package + if [ "${#PACKAGE_VERSIONS[@]}" -eq 0 ] || [ "${PACKAGE_VERSIONS[0]}" = "" ]; then + echo "INFO: ${name} package versions array is either empty or the first element is . Skipping ${name} installation." + continue + fi + downloadDir=$(echo "${p}" | jq .downloadLocation -r) + #download the package + case $name in + "kubernetes-cri-tools") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadAndInstallCriTools "${downloadDir}" "${evaluatedURL}" "${version}" + done + ;; + "azure-cni") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadAzureCNI "${downloadDir}" "${evaluatedURL}" + unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" + echo " - Azure CNI version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "cni-plugins") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadCNIPlugins "${downloadDir}" "${evaluatedURL}" + unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" + echo " - CNI plugin version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "containernetworking-plugins") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installCNI "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - containernetworking-plugins version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "runc") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + ensureRunc "${version}" "${evaluatedURL}" "${downloadDir}" + echo " - runc version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "containerd") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + if [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then + installContainerd "${downloadDir}" "${evaluatedURL}" "${version}" + elif isMarinerOrAzureLinux "$OS"; then + installStandaloneContainerd "${version}" + fi + echo " - containerd version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "oras") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installOras "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - oras version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "aks-secure-tls-bootstrap-client") + for version in ${PACKAGE_VERSIONS[@]}; do + # removed at provisioning time if secure TLS bootstrapping is disabled + if isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + installPackageFromCache "${name}" "${version}" "/opt/bin/${name}" || exit $? + elif isMarinerOrAzureLinux; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + installRPMPackageFromFile "${name}" "${version}" "/opt/bin/${name}" || exit $? + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + installSecureTLSBootstrapClientSysext "${version}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "azure-acr-credential-provider") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadCredentialProvider "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - azure-acr-credential-provider version ${version}" >> ${VHD_LOGS_FILEPATH} + # ORAS will be used to install other packages for network isolated clusters, it must go first. + done + ;; + "inspektor-gadget") + if isMariner "$OS" || isFlatcar "$OS" || isACL "$OS" "$OS_VARIANT" || isAzureLinuxOSGuard "$OS" "$OS_VARIANT" || [ "${IS_KATA}" = "true" ]; then + echo "Skipping inspektor-gadget installation for ${OS} ${OS_VARIANT:-default} (IS_KATA=${IS_KATA})" + else + ig_version="${PACKAGE_VERSIONS[0]}" + if isUbuntu "$OS"; then + # Ubuntu: download ig deb via apt; ig_install_deb_stack expects it at downloadDir + downloadPkgFromVersion "ig" "${ig_version}" "${downloadDir}" + installIG "${ig_version}" "${downloadDir}" + elif isAzureLinux "$OS"; then + # Azure Linux 3.0: ig_install_rpm_stack handles its own RPM downloads + installIG "${ig_version}" "${downloadDir}" + fi + fi + ;; + "kubernetes-binaries") + # kubelet and kubectl + # need to cover previously supported version for VMAS scale up scenario + # So keeping as many versions as we can - those unsupported version can be removed when we don't have enough space + # NOTE that we only keep the latest one per k8s patch version as kubelet/kubectl is decided by VHD version + # Please do not use the .1 suffix, because that's only for the base image patches + # regular version >= v1.17.0 or hotfixes >= 20211009 has arm64 binaries. + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + extractKubeBinaries "${version}" "${evaluatedURL}" false "${downloadDir}" + echo " - kubernetes-binaries version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + azure-acr-credential-provider-pmc) + name=${name%-pmc} + for version in ${PACKAGE_VERSIONS[@]}; do + if isMarinerOrAzureLinux || isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + kubelet|kubectl) + for version in ${PACKAGE_VERSIONS[@]}; do + if isMarinerOrAzureLinux || isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + cacheVersionedKubernetesPackageBinary "${name}" "${version}" "${downloadDir}" || exit $ERR_K8S_INSTALL_ERR + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "${K8S_DEVICE_PLUGIN_PKG}") + for version in ${PACKAGE_VERSIONS[@]}; do + if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isMarinerOrAzureLinux "$OS"; then + downloadPkgFromVersion "${K8S_DEVICE_PLUGIN_PKG}" "${version}" "${downloadDir}" + fi + echo " - ${K8S_DEVICE_PLUGIN_PKG} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "dra-driver-nvidia-gpu") + for version in ${PACKAGE_VERSIONS[@]}; do + if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isAzureLinux "$OS"; then + downloadPkgFromVersion "dra-driver-nvidia-gpu" "${version}" "${downloadDir}" + fi + echo " - dra-driver-nvidia-gpu version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "datacenter-gpu-manager-4-core") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "datacenter-gpu-manager-4-core" "${version}" "${downloadDir}" + echo " - datacenter-gpu-manager-4-core version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "datacenter-gpu-manager-4-proprietary") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "datacenter-gpu-manager-4-proprietary" "${version}" "${downloadDir}" + echo " - datacenter-gpu-manager-4-proprietary version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "dcgm-exporter") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" + echo " - dcgm-exporter version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "node-exporter") + # Skipping is handled by empty versionsV2 arrays in components.json + # for mariner, flatcar, acl, and osguard. Kata is skipped explicitly here. + if [ "${IS_KATA}" = "true" ]; then + echo "Skipping node-exporter installation for kata (IS_KATA=${IS_KATA})" + else + # Download and install node-exporter-kubernetes at VHD build time. + # node-exporter is installed on the VHD so CSE only needs to enable+start it. + installNodeExporter "${PACKAGE_VERSIONS[0]}" + fi + ;; + "acr-mirror") + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - temporarily skipping acr-mirror installation" + continue + fi + # Artifact streaming (acr-mirror) - version and URLs resolved from components.json, + # OS filtering handled declaratively by components.json entries ( for unsupported OSes). + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installAndConfigureArtifactStreaming "${evaluatedURL}" "${version}" + done + ;; + "aznfs") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL "${PACKAGE_DOWNLOAD_URL}") + mkdir -p "${downloadDir}" + aznfsFilename=$(basename "${evaluatedURL}") + echo "Downloading aznfs RPM from ${evaluatedURL} to ${downloadDir}/${aznfsFilename}" + retrycmd_curl_file 120 5 25 "${downloadDir}/${aznfsFilename}" "${evaluatedURL}" || exit $ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT + echo " - aznfs version ${version}" >> ${VHD_LOGS_FILEPATH} + done + installAznfsPackage || exit $ERR_AZNFS_INSTALL_FAIL + ;; + "blobfuse"|"blobfuse2") + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - temporarily skipping blobfuse2 installation" + continue + fi + for version in "${PACKAGE_VERSIONS[@]}"; do + if isUbuntu "$OS"; then + if ! apt_get_install 10 2 120 "${name}=${version}"; then + journalctl --no-pager -u "${name}" || true + tail -n 200 /var/log/apt/term.log || true + tail -n 200 /var/log/dpkg.log || true + exit $ERR_APT_INSTALL_TIMEOUT + fi + echo " - ${name} version ${version}" >> "${VHD_LOGS_FILEPATH}" + else + echo " - ${name} installation skipped for ${OS}" >> "${VHD_LOGS_FILEPATH}" + fi + done + ;; + *) + echo "Package name: ${name} not supported for download. Please implement the download logic in the script." + # We can add a common function to download a generic package here. + # However, installation could be different for different packages. + ;; + esac + capture_benchmark "${SCRIPT_NAME}_download_${name}" + done <<< "$packages" +} + +cacheContainerImageComponents() { + # Download/cache all declared container images within components.json that apply to the respective OS SKU + + # Limit number of parallel pulls to 2 less than number of processor cores in order to prevent issues with network, CPU, and disk resources + # Account for possibility that number of cores is 3 or less + num_proc=$(nproc) + if [ "$num_proc" -gt 3 ]; then + parallel_container_image_pull_limit=$(nproc --ignore=2) + else + parallel_container_image_pull_limit=1 + fi + echo "Limit for parallel container image pulls set to $parallel_container_image_pull_limit" + + declare -a image_pids=() + + ContainerImages=$(jq ".ContainerImages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq .downloadURL -r) + amd64OnlyVersionsStr=$(echo "${imageToBePulled}" | jq .amd64OnlyVersions -r) + updateMultiArchVersions "${imageToBePulled}" + amd64OnlyVersions="" + if [ "${amd64OnlyVersionsStr}" != "null" ]; then + amd64OnlyVersions=$(echo "${amd64OnlyVersionsStr}" | jq -r ".[]") + fi + + if [ "$(isARM64)" -eq 1 ]; then + versions="${MULTI_ARCH_VERSIONS[*]}" + else + versions="${amd64OnlyVersions} ${MULTI_ARCH_VERSIONS[*]}" + fi + + for version in ${versions}; do + CONTAINER_IMAGE=$(string_replace $downloadURL $version) + pullContainerImage "ctr" "${CONTAINER_IMAGE}" & + image_pids+=($!) + echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} + while [ "$(jobs -p | wc -l)" -ge "$parallel_container_image_pull_limit" ]; do + wait -n || { + ret=$? + echo "A background job pullContainerImage failed: ${ret}, ${CONTAINER_IMAGE}. Exiting..." >&2 + for pid in "${image_pids[@]}"; do + kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" + done + exit "${ret}" + } + done + done + done <<< "$ContainerImages" + echo "Waiting for container image pulls to finish. PID: ${image_pids[@]}" + while [ "$(jobs -p | wc -l)" -gt 0 ]; do + wait -n || { + ret=$? + echo "A background job pullContainerImage failed: ${ret}. Exiting..." >&2 + for pid in "${image_pids[@]}"; do + kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" + done + exit "${ret}" + } + done +} + +cacheGPUContainerImageComponents() { + # Download/install GPU container image components + GPUContainerImages=$(jq -c '.GPUContainerImages[]' $COMPONENTS_FILEPATH) + + NVIDIA_DRIVER_IMAGE="" + NVIDIA_DRIVER_IMAGE_TAG="" + NVIDIA_GRID_DRIVER_VERSION="" + + # Extract GRID driver version for release notes (applicable to all Linux distributions) + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') + # shellcheck disable=SC2001 + imageName=$(echo "$downloadURL" | sed 's/:.*$//') + + if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-grid" ]; then + NVIDIA_GRID_DRIVER_VERSION=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') + # Continue to extract CUDA driver info as well + fi + done <<< "$GPUContainerImages" + + # For Ubuntu, pre-pull the CUDA driver image + if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now + gpu_action="copy" + + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') + # shellcheck disable=SC2001 + imageName=$(echo "$downloadURL" | sed 's/:.*$//') + + if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-cuda-lts" ]; then + latestVersion=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') + NVIDIA_DRIVER_IMAGE="$imageName" + NVIDIA_DRIVER_IMAGE_TAG="$latestVersion" + break # Exit the loop once we find the image + fi + done <<< "$GPUContainerImages" + + # Check if the NVIDIA_DRIVER_IMAGE and NVIDIA_DRIVER_IMAGE_TAG were found + if [ -z "$NVIDIA_DRIVER_IMAGE" ] || [ -z "$NVIDIA_DRIVER_IMAGE_TAG" ]; then + echo "Error: Unable to find aks-gpu-cuda-lts image in components.json" + exit 1 + fi + + mkdir -p /opt/{actions,gpu} + + /opt/azure/containers/image-fetcher "$NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG" + + cat << EOF >> ${VHD_LOGS_FILEPATH} + - nvidia-cuda-driver=${NVIDIA_DRIVER_IMAGE_TAG} +EOF + fi + + # Add a separate section for runtime-installed components + # This clearly distinguishes components installed during CSE from VHD build-time components + # Only add for Ubuntu + if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then + cat << EOF >> ${VHD_LOGS_FILEPATH} +Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): + - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} +EOF + fi +} + +buildNVIDIAKernelModule() { + if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now + gpu_action="copy" + + # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the + # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles + # and DKMS-registers the kernel module + stages userspace libs against THIS VHD's kernel, + # performs NO device access (safe on the GPU-less Packer builder), and writes the marker + # /opt/azure/aks-gpu/dkms-marker. At node boot, configGPUDrivers passes "install-skip-build" + # when that marker matches, running only the device-dependent steps. + # The driver image is intentionally LEFT in the VHD: boot-time device init still sources the + # container toolkit debs, fabric manager, containerd runtime config and udev rules from it. + # Dropping the image is a separate, deferred size optimization. + if grep -q "NVIDIA_CUDA_PREBAKE" <<< "$FEATURE_FLAGS"; then + echo "Pre-building NVIDIA CUDA kernel module into the VHD (build-only) for kernel $(uname -r)" + # nvidia-installer needs gcc/make + libc6-dev to compile; the builder lacks them here, so install + # them. A boot-time fallback recompile (marker mismatch) still has them: the VHD ships + # build-essential -> libc6-dev (release-notes manifests) -- the toolchain baseline GPU nodes compile + # with at boot (installDeps runs apt --no-install-recommends, so gcc alone doesn't pull libc6-dev). + apt_get_install 10 2 300 gcc make libc6-dev || exit 1 + CTR_GPU_PREBUILD_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind" + retrycmd_if_failure 3 10 600 bash -c "$CTR_GPU_PREBUILD_CMD $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG gpuprebuild /entrypoint.sh build-only" || exit 1 + if [ ! -f /opt/azure/aks-gpu/dkms-marker ]; then + echo "Error: NVIDIA CUDA prebake did not produce /opt/azure/aks-gpu/dkms-marker" + exit 1 + fi + cat << EOF >> ${VHD_LOGS_FILEPATH} + - nvidia-cuda-driver-prebaked=${NVIDIA_DRIVER_IMAGE_TAG} (kernel $(uname -r)) +EOF + fi + fi +} + +configureGraceBlackwell() { + if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then + # NVIDIA GB setup is only supported on arm64 Ubuntu 24.04. + if [ "${CPU_ARCH}" = "arm64" ] && [ "${UBUNTU_RELEASE}" = "24.04" ]; then + # Replicate all functionality from github.com/azure/aks-gpu/install.sh. + # aks-gpu is designed to run at node boot/join time, whereas the NVIDIA GB VHD is set up + # to have all drivers installed at VHD build time. + + # 1. Blacklist nouveau driver + cat << EOF >> /etc/modprobe.d/blacklist-nouveau.conf +blacklist nouveau +options nouveau modeset=0 +EOF + update-initramfs -u + + # 2. install drivers + BOM_PATH="gb-mai-bom.json" + + # Install a custom repository if a doca-custom-repo is specified + DOCA_CUSTOM_REPO=$(jq -r '.["doca-custom-repo"]' $BOM_PATH) + if [ -n "$DOCA_CUSTOM_REPO" ]; then + mv /etc/apt/sources.list.d/doca-net.list /etc/apt/sources.list.d/doca-net.list.backup + echo "deb [arch=arm64 signed-by=/etc/apt/keyrings/doca-net.pub] $DOCA_CUSTOM_REPO ./" > /etc/apt/sources.list.d/doca-net.list + apt-get update + fi + + # Install DOCA/OFED before the GPU driver so nvidia-peermem can build against the RDMA APIs provided by OFED. + # Farcically, nvidia-dkms-580-open cannot be installed together with the CUDA toolkit. Something about that package changes the build environment in an incompatible way. I've seen people mention CUDA including an old version of gcc that somehow makes its way onto the PATH... + # Therefore we install DOCA/OFED first, the GPU driver and its dependencies second, then all downstream reverse-dependencies (CUDA, DCGM, and so forth) third. + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave1"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave2"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave3"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + + # 3. Add char device symlinks for NVIDIA devices + mkdir -p "$(dirname /lib/udev/rules.d/71-nvidia-dev-char.rules)" + cat << EOF >> /lib/udev/rules.d/71-nvidia-dev-char.rules +ACTION=="add", DEVPATH=="/bus/pci/drivers/nvidia", RUN+="/usr/bin/nvidia-ctk system create-dev-char-symlinks --create-all" +EOF + + # Create systemd drop-in to override nvidia-device-plugin dependencies + mkdir -p /etc/systemd/system/nvidia-device-plugin.service.d + cat << EOF > /etc/systemd/system/nvidia-device-plugin.service.d/override.conf +[Unit] +After=kubelet.service + +[Service] +ExecStartPre=-/usr/bin/mkdir -p /var/lib/kubelet/device-plugins +EOF + + # Now we are off-piste: enable DCGM, DCGM exporter, container device plugin, and the NVIDIA containerd config. + systemctl enable nvidia-dcgm + systemctl enable nvidia-dcgm-exporter + systemctl enable nvidia-device-plugin + systemctl enable openibd + + # One additional request from MAI: signal that NPD is pre-installed on the VHD. + # When this file is present, the Azure AKS VM Extension skips installing NPD at provision time. + mkdir -p /etc/node-problem-detector.d/ + touch /etc/node-problem-detector.d/skip_vhd_npd + fi + fi +} + +# This function extracts CoreDNS binary from cached coredns images (latest version) +# and copies it to - /opt/azure/containers/localdns/binary/coredns. +# The binary is later used by localdns systemd unit. +# The function also handles the cleanup of temporary directories and unmounting of images. +extractAndCacheCoreDnsBinary() { + local coredns_image_list=($(ctr -n k8s.io images list -q | grep coredns)) + if [ "${#coredns_image_list[@]}" -eq 0 ]; then + echo "Error: No coredns images found." + exit 1 + fi + + rm -rf "${LOCALDNS_BINARY_PATH}" || exit 1 + mkdir -p "${LOCALDNS_BINARY_PATH}" || exit 1 + + cleanup_coredns_imports() { + set +e + if [ -n "${ctr_temp}" ]; then + ctr -n k8s.io images unmount "${ctr_temp}" >/dev/null + rm -rf "${ctr_temp}" + fi + } + trap cleanup_coredns_imports EXIT ABRT ERR INT PIPE QUIT TERM + + # Extract available coredns image tags (v1.12.0-1 format) and sort them in descending order. + local sorted_coredns_tags=($(for image in "${coredns_image_list[@]}"; do echo "${image##*:}"; done | sort -V -r)) + + # Determine latest version. local latest_coredns_tag="${sorted_coredns_tags[0]}" # Extract the CoreDNS binary for the latest version. @@ -1240,11 +1056,6 @@ extractAndCacheCoreDnsBinary() { trap - EXIT ABRT ERR INT PIPE QUIT TERM } -extractAndCacheCoreDnsBinary - -rm -f ./azcopy # cleanup immediately after usage will return in two downloads -echo "install-dependencies step completed successfully" - # Collect grid compatibility data (placeholder for now - will be extended later) collect_grid_compatibility_data() { if [ -z "${GRID_COMPATIBILITY_DATA_FILE}" ] ; then @@ -1275,6 +1086,250 @@ collect_grid_compatibility_data() { chmod 755 "${GRID_COMPATIBILITY_DATA_FILE}" } +# --------------------------------------------------------- INSTALL DEPENDENCIES START --------------------------------------------------------- + +# check if COMPONENTS_FILEPATH exists +if [ ! -f "$COMPONENTS_FILEPATH" ]; then + echo "Components file not found at $COMPONENTS_FILEPATH. Exiting..." + exit 1 +fi + +resolve_packages_source_url + +echo "" +echo "Components downloaded in this VHD build (some of the below components might get deleted during cluster provisioning if they are not needed):" >> ${VHD_LOGS_FILEPATH} +capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" + +echo "Logging the kernel after purge and reinstall + reboot: $(uname -r)" +# fix grub issue with cvm by reinstalling before other deps +# other VHDs use grub-pc, not grub-efi +if [ "$OS" = "$UBUNTU_OS_NAME" ] && echo "$FEATURE_FLAGS" | grep -q "cvm"; then + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + wait_for_apt_locks + apt_get_install 10 2 120 grub-efi || exit 1 +fi +capture_benchmark "${SCRIPT_NAME}_reinstall_grub_for_cvm" + +if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + # disable and mask all UU timers/services + # save some background io/latency + systemctl mask apt-daily.service apt-daily-upgrade.service || exit 1 + systemctl disable apt-daily.service apt-daily-upgrade.service || exit 1 + systemctl disable apt-daily.timer apt-daily-upgrade.timer || exit 1 + + tee /etc/apt/apt.conf.d/99periodic > /dev/null <> "${CHRONYD_DIR}"/10-chrony-restarts.conf < /dev/null <<'EOF' +Compress=yes +Storage=persistent +SystemMaxUse=1G +RuntimeMaxUse=1G +ForwardToSyslog=yes +EOF +capture_benchmark "${SCRIPT_NAME}_install_deps_and_set_configs" + +if [ "$(isARM64)" -eq 1 ]; then + # shellcheck disable=SC3010 + if [[ ${HYPERV_GENERATION,,} == "v1" ]]; then + echo "No arm64 support on V1 VM, exiting..." + exit 1 + fi +fi + +# Always override network config and disable NTP + Timesyncd and install Chrony +# Mariner does this differently, so only do it for Ubuntu +if ! isMarinerOrAzureLinux "$OS"; then + overrideNetworkConfig || exit 1 + disableNtpAndTimesyncdInstallChrony || exit 1 +fi + +# ACL inherits Azure Linux behaviors but isMarinerOrAzureLinux returns false, +# so these must be called separately (mirrored in the Mariner/AzureLinux block below). +# Other Mariner functions are safe to skip for ACL: +# setMarinerNetworkdConfig — ACL doesn't ship systemd-bootstrap's 99-dhcp-en.network +# fixCBLMarinerPermissions — product_uuid already 444; no rsyslog on ACL +# addMarinerNvidiaRepo / updateDnfWithNvidiaPkg / disableDNFAutomatic / enableCheckRestart — ACL has no dnf/rpm +# activateNfConntrack — nf_conntrack auto-loads via iptables dependency chain +# disableTimesyncd — ACL handles chrony separately above via disableNtpAndTimesyncdInstallChrony +if isACL "$OS" "$OS_VARIANT"; then + # ACL's iptables.service loads host firewall rules that conflict with Cilium eBPF routing. + disableSystemdIptables || exit 1 + # Repoint /etc/resolv.conf from the stub (127.0.0.53) to the real upstream file + # so DNS queries go directly through localdns. + disableSystemdResolvedCache +fi +capture_benchmark "${SCRIPT_NAME}_validate_container_runtime_and_override_ubuntu_net_config" + +# Configure SSH service during VHD build for Ubuntu 22.10+ +configureSSHService "$OS" "$OS_VERSION" || echo "##vso[task.logissue type=warning]SSH Service configuration failed, but continuing VHD build" + +CONTAINERD_SERVICE_DIR="/etc/systemd/system/containerd.service.d" +mkdir -p "${CONTAINERD_SERVICE_DIR}" +# Explicitly set LimitNOFILE=1048576 (the value that 'infinity' resolves to on Ubuntu 22.04) for both Ubuntu and Mariner/AzureLinux. +# On Ubuntu 24.04 (Containerd 2.0), LimitNOFILE is removed upstream and systemd falls back to an implicit soft:hard limit +# (for example 1024:524288), so containerd inherits a very low soft file descriptor limit (1024) unless we override it here. +# On Mariner/AzureLinux this is redundant with the base containerd.service unit but harmless. +# Not removing LimitNOFILE from parts/linux/cloud-init/artifacts/containerd.service, +# to avoid compatibility issues between new VHDs and old CSE scripts. +tee "${CONTAINERD_SERVICE_DIR}/exec_start.conf" > /dev/null < /dev/null <> ${VHD_LOGS_FILEPATH} +cacheContainerImageComponents +capture_benchmark "${SCRIPT_NAME}_caching_container_images" + +configureGraceBlackwell +if [ -d "/opt/gpu" ] && [ "$(ls -A /opt/gpu)" ]; then + ls -ltr /opt/gpu/* >> ${VHD_LOGS_FILEPATH} +fi + +retagAKSNodeCAWatcher +capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" + +pinPodSandboxImages +capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" + +# IPv6 nftables rules are only available on Ubuntu or Mariner/AzureLinux +if [ $OS = $UBUNTU_OS_NAME ] || isMarinerOrAzureLinux "$OS"; then + systemctlEnableAndStart ipv6_nftables 30 || exit 1 +fi + +mkdir -p /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events + +# Disable cgroup-memory-telemetry on AzureLinux due to incompatibility with cgroup2fs driver and absence of required azure.slice directory +if ! isMarinerOrAzureLinux "$OS"; then + systemctlEnableAndStart cgroup-memory-telemetry.timer 30 || exit 1 + systemctl enable cgroup-memory-telemetry.service || exit 1 + systemctl restart cgroup-memory-telemetry.service +fi + +CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) +if [ "$CGROUP_VERSION" = "cgroup2fs" ]; then + systemctlEnableAndStart cgroup-pressure-telemetry.timer 30 || exit 1 + systemctl enable cgroup-pressure-telemetry.service || exit 1 + systemctl restart cgroup-pressure-telemetry.service +fi + +if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ "$(ls -A /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/)" ]; then + cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/* + rm -r /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 +fi +capture_benchmark "${SCRIPT_NAME}_configure_telemetry" + +if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + # remove snapd, which is not used by container stack + apt_get_purge 20 30 120 snapd || exit 1 + apt_get_purge 20 30 120 apache2-utils || exit 1 + # CIS: Ensure telnet (/ftp) client is not installed + # CIS: Ufw is not used but interferes with log_martians rule + apt_get_purge 20 30 120 telnet ftp ufw tnftp inetutils-telnet || exit 1 + + apt-get -y autoclean || exit 1 + apt-get -y autoremove --purge || exit 1 + apt-get -y clean || exit 1 + # update message-of-the-day to start after multi-user.target + # multi-user.target usually start at the end of the boot sequence + sed -i 's/After=network-online.target/After=multi-user.target/g' /lib/systemd/system/motd-news.service +fi +capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu" + +# Ensure eBPF tools installed successfully +finishEBPFToolsInstallation +capture_benchmark "${SCRIPT_NAME}_finish_installing_ebpf_tools" + +configureLsmWithBpf +capture_benchmark "${SCRIPT_NAME}_configure_lsm_with_bpf" + +# use the private_packages_url to download and cache packages +if [ -n "${PRIVATE_PACKAGES_URL:-}" ]; then + IFS=',' read -ra PRIVATE_URLS <<< "${PRIVATE_PACKAGES_URL}" + + for private_url in "${PRIVATE_URLS[@]}"; do + echo "download kube package from ${private_url}" + cacheKubePackageFromPrivateUrl "$private_url" + done +fi +rm -f ./azcopy # cleanup immediately after usage will return in two downloads + +extractAndCacheCoreDnsBinary + collect_grid_compatibility_data # nvidia repos are non msft public endpoints and should not be present on VHDs. @@ -1283,5 +1338,7 @@ collect_grid_compatibility_data removeNvidiaRepos capture_benchmark "${SCRIPT_NAME}_remove_nvidia_repos" +echo "install-dependencies step completed successfully" + capture_benchmark "${SCRIPT_NAME}_overall" true process_benchmarks diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index c2a6e2f4213..51499d646ea 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -409,7 +409,7 @@ copyPackerFiles() { cpAndMode $AKS_CHECK_NETWORK_SCRIPT_SRC $AKS_CHECK_NETWORK_SCRIPT_DEST 755 cpAndMode $AKS_CHECK_NETWORK_SERVICE_SRC $AKS_CHECK_NETWORK_SERVICE_DEST 644 - if [ ${UBUNTU_RELEASE} = "22.04" ] || [ ${UBUNTU_RELEASE} = "24.04" ]; then + if [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then PAM_D_COMMON_AUTH_SRC=/home/packer/pam-d-common-auth-2204 fi diff --git a/vhdbuilder/packer/post-install-dependencies.sh b/vhdbuilder/packer/post-install-dependencies.sh index 65eee5613c6..72509dae75c 100644 --- a/vhdbuilder/packer/post-install-dependencies.sh +++ b/vhdbuilder/packer/post-install-dependencies.sh @@ -1,6 +1,6 @@ #!/bin/bash -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') UBUNTU_OS_NAME="UBUNTU" FLATCAR_OS_NAME="FLATCAR" ACL_OS_NAME="AZURECONTAINERLINUX" @@ -49,13 +49,15 @@ if [ $OS = $UBUNTU_OS_NAME ]; then # remove apport retrycmd_if_failure 10 2 60 apt-get purge --auto-remove apport open-vm-tools -y || exit 1 - # Remove PackageKit: an unused desktop D-Bus package manager whose apt hook (/etc/apt/apt.conf.d/20packagekit) - # runs `gdbus call --system` after every apt update. During early-boot CSE the system bus isn't ready yet, so - # that benign `Error connecting: ... Broken pipe` stderr trips the node-bootstrap apt error-check -> CSE exit 99 - # -> node never joins. Purging it also drops packagekit-tools + software-properties-common (add-apt-repository, - # unused at node runtime; the build's only add-apt-repository usage is earlier in pre-install-dependencies.sh). - # No-op on the minimal image, which does not ship these. - retrycmd_if_failure 10 2 60 apt-get purge --auto-remove packagekit packagekit-tools software-properties-common -y || exit 1 + if ! isMinimalImage; then + # Remove PackageKit: an unused desktop D-Bus package manager whose apt hook (/etc/apt/apt.conf.d/20packagekit) + # runs `gdbus call --system` after every apt update. During early-boot CSE the system bus isn't ready yet, so + # that benign `Error connecting: ... Broken pipe` stderr trips the node-bootstrap apt error-check -> CSE exit 99 + # -> node never joins. Purging it also drops packagekit-tools + software-properties-common (add-apt-repository, + # unused at node runtime; the build's only add-apt-repository usage is earlier in pre-install-dependencies.sh). + # No-op on the minimal image, which does not ship these. + retrycmd_if_failure 10 2 60 apt-get purge --auto-remove packagekit packagekit-tools software-properties-common -y || exit 1 + fi # strip old kernels/packages retrycmd_if_failure 10 2 60 apt-get -y autoclean || exit 1 diff --git a/vhdbuilder/packer/pre-install-dependencies.sh b/vhdbuilder/packer/pre-install-dependencies.sh index 89ab9f0b885..12b2eef6a44 100644 --- a/vhdbuilder/packer/pre-install-dependencies.sh +++ b/vhdbuilder/packer/pre-install-dependencies.sh @@ -1,7 +1,7 @@ #!/bin/bash -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') THIS_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)" #the following sed removes all comments of the format {{/* */}} @@ -36,6 +36,11 @@ capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" copyPackerFiles +# Install required dependencies needed to build minimal images if needed (currently only Ubuntu 26.04) +if isMinimalImage && isUbuntu "$OS"; then + installMinimalBuildDeps +fi + # Update rsyslog configuration RSYSLOG_CONFIG_FILEPATH="/etc/rsyslog.d/60-CIS.conf" if isMarinerOrAzureLinux "$OS"; then diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index c6a6f89657a..34dfd3a35e1 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -1731,11 +1731,16 @@ testContainerImagePrefetchScript() { testBccTools () { local test="BCCInstallTest" os_sku="${1}" + os_version="${2}" echo "$test: checking if BCC tools were successfully installed" if [ "$os_sku" = "AzureLinuxOSGuard" ]; then echo "$test: Skipping check on AzureLinuxOSGuard - BCC tools are not installed" return 0 fi + if [ "$os_version" = "26.04" ]; then + echo "$test: Skipping check on Ubuntu 26.04 - BCC tools are not installed" + return 0 + fi for line in ' - bcc-tools' ' - libbcc-examples'; do if ! grep -F -x -e "$line" $VHD_LOGS_FILEPATH; then err "BCC tools were not successfully installed" @@ -2525,7 +2530,7 @@ testContainerNetworkingPluginsInstalled() { # This will keep the VM alive after the tests are run and we can SSH/Bastion into the VM to run the test manually. # Therefore, for example, you can run "sudo bash /var/lib/waagent/run-command/download/0/script.sh" to run the tests manually. checkPerformanceData -testBccTools $OS_SKU +testBccTools $OS_SKU $OS_VERSION testVHDBuildLogsExist testCriticalTools testPackagesInstalled diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 42f301f326b..8de07d7d489 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -17,6 +17,9 @@ TRIVY_DEB_2204_VERSION="0.70.0-ubuntu22.04u9" # renovate: datasource=custom.deb2404 depName=trivy versioning=deb TRIVY_DEB_2404_VERSION="0.70.0-ubuntu24.04u9" +# renovate: datasource=custom.deb2604 depName=trivy versioning=deb +TRIVY_DEB_2604_VERSION="0.70.0-ubuntu26.04u12" + # renovate: datasource=rpm depName=trivy registryUrl=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata TRIVY_RPM_VERSION="0.70.0-9.azl3" @@ -77,50 +80,78 @@ retrycmd_if_failure() { echo Executed \"$@\" $i times; } +addPMCAptKey() { + local OS_VERSION=${1} + + retrycmd_if_failure 120 5 25 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + + if [ "${OS_VERSION}" = "26.04" ]; then + # Ubuntu 26.04 (Resolute) PMC repo is signed with Microsoft's newer 2025 gpg key + retrycmd_if_failure 120 5 25 curl https://packages.microsoft.com/keys/microsoft-2025.asc | gpg --dearmor > /tmp/microsoft-2025.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft-2025.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + fi +} + +installAzCLIFromUbuntuPMC() { + local arch + arch="$(uname -m)" + + apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg + + if [ "${arch,,}" = "arm64" ]; then + echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + else + echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + fi + + apt_get_update + apt_get_install 5 1 60 azure-cli +} + install_azure_cli() { OS_SKU=${1} OS_VERSION=${2} ARCHITECTURE=${3} TEST_VM_ADMIN_USERNAME=${4} - if [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "22.04" ] && [ "${ARCHITECTURE,,}" = "arm64" ]; then - apt_get_update - apt_get_install 5 1 60 python3-pip - pip install azure-cli - export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" - CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z $CHECKAZ ]; then - echo "Azure CLI is not installed properly." - exit 1 - fi - elif [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "24.04" ] && [ "${ARCHITECTURE,,}" = "arm64" ]; then - apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - apt_get_update - apt_get_install 5 1 60 azure-cli - elif [ "$OS_SKU" = "Ubuntu" ] && { [ "$OS_VERSION" = "20.04" ] || [ "$OS_VERSION" = "22.04" ] || [ "$OS_VERSION" = "24.04" ]; } && [ "${ARCHITECTURE,,}" != "arm64" ]; then - apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - apt_get_update - apt_get_install 5 1 60 azure-cli - elif [ "$OS_SKU" = "CBLMariner" ] || [ "$OS_SKU" = "AzureLinux" ]; then + if [ "$OS_SKU" != "Ubuntu" ] && [ "$OS_SKU" != "CBLMariner" ] && [ "$OS_SKU" != "AzureLinux" ] && [ "$OS_SKU" != "Flatcar" ] && [ "$OS_SKU" != "AzureContainerLinux" ] && [ "$OS_SKU" != "AzureLinuxOSGuard" ]; then + echo "Unrecognized OS SKU for downloading az: $OS_SKU $OS_VERSION $ARCHITECTURE" + exit 1 + fi + + if [ "$OS_SKU" = "CBLMariner" ] || [ "$OS_SKU" = "AzureLinux" ]; then sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc sudo sh -c 'echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo' sudo dnf install -y azure-cli - elif [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ]; then + return 0 + fi + + if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ]; then python3 -m venv "/home/$TEST_VM_ADMIN_USERNAME/venv" export PATH="/home/$TEST_VM_ADMIN_USERNAME/venv/bin:$PATH" pip install azure-cli CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z $CHECKAZ ]; then + if [ -z "$CHECKAZ" ]; then + echo "Azure CLI is not installed properly." + exit 1 + fi + return 0 + fi + + if [ "$OS_VERSION" = "26.04" ]; then + # TODO(2604): install azcli from PMC once the resolute package is available + apt_get_update + apt_get_install 5 1 60 python3-pip + pip install azure-cli --break-system-packages + export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" + CHECKAZ=$(pip freeze | grep "azure-cli==") + if [ -z "$CHECKAZ" ]; then echo "Azure CLI is not installed properly." exit 1 fi else - echo "Unrecognized SKU, Version, and Architecture combination for downloading az: $OS_SKU $OS_VERSION $ARCHITECTURE" - exit 1 + installAzCLIFromUbuntuPMC fi } @@ -136,15 +167,15 @@ login_with_user_assigned_managed_identity() { echo "logging into azure with flags: $LOGIN_FLAGS" az login $LOGIN_FLAGS } + login_with_umsi_object_id() { login_with_user_assigned_managed_identity "--object-id" "$1" } + login_with_umsi_resource_id() { login_with_user_assigned_managed_identity "--resource-id" "$1" } -install_azure_cli $OS_SKU $OS_VERSION $ARCHITECTURE $TEST_VM_ADMIN_USERNAME - install_trivy_from_github() { # Use the dedicated GitHub fallback version — PMC versions (e.g., 0.68.2) don't have # matching GitHub releases, so we pin to an actual upstream release separately. @@ -181,6 +212,7 @@ install_trivy() { 20.04) deb_version="${TRIVY_DEB_2004_VERSION}" ;; 22.04) deb_version="${TRIVY_DEB_2204_VERSION}" ;; 24.04) deb_version="${TRIVY_DEB_2404_VERSION}" ;; + 26.04) deb_version="${TRIVY_DEB_2604_VERSION}" ;; *) echo "No tracked trivy deb version for Ubuntu $os_version, downloading from GitHub" install_trivy_from_github @@ -201,6 +233,12 @@ install_trivy() { esac } +if [ "$OS_SKU" = "Ubuntu" ]; then + addPMCAptKey $OS_VERSION +fi + +install_azure_cli $OS_SKU $OS_VERSION $ARCHITECTURE $TEST_VM_ADMIN_USERNAME + install_trivy "$OS_SKU" "$OS_VERSION" login_with_umsi_object_id ${UMSI_PRINCIPAL_ID} diff --git a/vhdbuilder/packer/vhd-scanning.sh b/vhdbuilder/packer/vhd-scanning.sh index c1ce9e7e17b..42c24da07ca 100755 --- a/vhdbuilder/packer/vhd-scanning.sh +++ b/vhdbuilder/packer/vhd-scanning.sh @@ -249,10 +249,17 @@ isCISUnsupportedUbuntu() { local os="$1" local version="$2" - # Only 22.04+ are supported - if [ "$os" = "Ubuntu" ] && { [ "$version" = "20.04" ]; }; then + # Only 22.04+ are supported. + if [ "$os" = "Ubuntu" ] && [ "$version" = "20.04" ]; then return 0 fi + + # No CIS benchmarks yet available for Ubuntu 26.04 (resolute) + # TODO(2604): enable 26.04 CIS scanning when support is added by upstream CIS. + if [ "$os" = "Ubuntu" ] && [ "$version" = "26.04" ]; then + return 0 + fi + return 1 } isFlatcar() { diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 807d67bae41..dcadf961c88 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -1,23 +1,21 @@ #!/bin/bash -{{/* FIPS-related error codes */}} -ERR_UA_TOOLS_INSTALL_TIMEOUT=180 {{/* Timeout waiting for ubuntu-advantage-tools install */}} -ERR_ADD_UA_APT_REPO=181 {{/* Error to add UA apt repository */}} -ERR_UA_ATTACH=182 {{/* Error attaching UA */}} -ERR_UA_DISABLE_LIVEPATCH=183 {{/* Error to disable UA livepatch */}} -ERR_UA_ENABLE_FIPS=184 {{/* Error to enable UA FIPS */}} -ERR_UA_DETACH=185 {{/* Error to detach UA */}} -ERR_LINUX_HEADER_INSTALL_TIMEOUT=186 {{/* Timeout to install linux header */}} -ERR_STRONGSWAN_INSTALL_TIMEOUT=187 {{/* Timeout to install strongswan */}} -ERR_UA_ESM_HOOK_CLEANUP=188 {{/* Error removing the apt ESM hook for Ubuntu Pro */}} -ERR_UA_MASK_UNIT=189 {{/* Error stopping/disabling/masking an Ubuntu Pro background unit */}} -ERR_UA_TOKEN_CLEANUP=190 {{/* Error removing the baked-in Ubuntu Pro machine token state */}} - -ERR_NTP_INSTALL_TIMEOUT=10 {{/*Unable to install NTP */}} -ERR_NTP_START_TIMEOUT=11 {{/* Unable to start NTP */}} -ERR_STOP_OR_DISABLE_SYSTEMD_TIMESYNCD_TIMEOUT=12 {{/* Timeout waiting for systemd-timesyncd stop */}} -ERR_STOP_OR_DISABLE_NTP_TIMEOUT=13 {{/* Timeout waiting for ntp stop */}} -ERR_CHRONY_INSTALL_TIMEOUT=14 {{/*Unable to install CHRONY */}} -ERR_CHRONY_START_TIMEOUT=15 {{/* Unable to start CHRONY */}} +ERR_UA_TOOLS_INSTALL_TIMEOUT=180 # Timeout waiting for ubuntu-advantage-tools install +ERR_ADD_UA_APT_REPO=181 # Error to add UA apt repository +ERR_UA_ATTACH=182 # Error attaching UA +ERR_UA_DISABLE_LIVEPATCH=183 # Error to disable UA livepatch +ERR_UA_ENABLE_FIPS=184 # Error to enable UA FIPS +ERR_UA_DETACH=185 # Error to detach UA +ERR_LINUX_HEADER_INSTALL_TIMEOUT=186 # Timeout to install linux header +ERR_STRONGSWAN_INSTALL_TIMEOUT=187 # Timeout to install strongswan +ERR_UA_ESM_HOOK_CLEANUP=188 # Error removing the apt ESM hook for Ubuntu Pro +ERR_UA_MASK_UNIT=189 # Error stopping/disabling/masking an Ubuntu Pro background unit +ERR_UA_TOKEN_CLEANUP=190 # Error removing the baked-in Ubuntu Pro machine token state +ERR_NTP_INSTALL_TIMEOUT=10 # Unable to install NTP +ERR_NTP_START_TIMEOUT=11 # Unable to start NTP +ERR_STOP_OR_DISABLE_SYSTEMD_TIMESYNCD_TIMEOUT=12 # Timeout waiting for systemd-timesyncd stop +ERR_STOP_OR_DISABLE_NTP_TIMEOUT=13 # Timeout waiting for ntp stop +ERR_CHRONY_INSTALL_TIMEOUT=14 # Unable to install CHRONY +ERR_CHRONY_START_TIMEOUT=15 # Unable to start CHRONY echo "Sourcing tool_installs_ubuntu.sh" @@ -44,9 +42,9 @@ installBcc() { fi mkdir -p /tmp/bcc - pushd /tmp/bcc + pushd /tmp/bcc || exit 1 git clone https://github.com/iovisor/bcc.git - mkdir bcc/build; cd bcc/build + mkdir bcc/build; cd bcc/build || exit 1 git checkout v0.29.0 @@ -54,11 +52,12 @@ installBcc() { make sudo make install || exit 1 cmake -DPYTHON_CMD=python3 .. || exit 1 # build python3 binding - pushd src/python/ + pushd src/python/ || exit 1 make sudo make install || exit 1 - popd - popd + popd || exit 1 + popd || exit 1 + # we explicitly do not remove build-essential or python # these are standard packages we want to keep, they should usually be in the final build anyway. # only ensuring they are installed above.