Skip to content

[AKS] Synchronize monitoring profiles and repair live scenarios #34047

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
FumingZhang wants to merge 2 commits into
base: dev
Choose a base branch
from
Draft

[AKS] Synchronize monitoring profiles and repair live scenarios #34047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e5905ee
{AKS} Synchronize monitoring profiles and repair live scenarios
FumingZhang Sep 9, 2026
d987836
{AKS} Assert disabled monitoring without discarding metadata
FumingZhang Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/azure-cli/azure/cli/command_modules/acs/custom.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1879,6 +1879,20 @@ def _update_addons(cmd, instance, subscription_id, resource_group_name, name, ad
"The addon {} is not installed.".format(addon))
addon_profiles[addon].config = None
addon_profiles[addon].enabled = enable
if addon == CONST_MONITORING_ADDON_NAME:
monitor_profile = getattr(instance, "azure_monitor_profile", None)
if getattr(monitor_profile, "container_insights", None) is not None:
# Reset the canonical profile along with the legacy addon config.
# Otherwise its old enabled/workspace/flow-log values survive the PUT.
ContainerInsights = cmd.get_models(
'ManagedClusterAzureMonitorProfileContainerInsights',
resource_type=ResourceType.MGMT_CONTAINERSERVICE,
operation_group='managed_clusters',
)
monitor_profile.container_insights = ContainerInsights(
enabled=enable,
log_analytics_workspace_resource_id=workspace_resource_id if enable else None,
)

instance.addon_profiles = addon_profiles

Expand Down
7 changes: 7 additions & 0 deletions src/azure-cli/azure/cli/command_modules/acs/managed_cluster_decorator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9208,6 +9208,13 @@ def update_monitoring_profile_flow_logs(self, mc: ManagedCluster) -> ManagedClus
config = monitoring_addon_profile.config or {}
config["enableRetinaNetworkFlags"] = str(container_network_logs_enabled)
mc.addon_profiles[monitoring_addon_key].config = config
# Newer API responses contain both representations; keep the canonical
# profile consistent with the legacy addon flag in the outgoing PUT.
container_insights = getattr(mc.azure_monitor_profile, "container_insights", None)
if container_insights is not None:
container_insights.container_network_logs = (
"Enabled" if container_network_logs_enabled else "Disabled"
)

# When CNL or HLSM flags are provided, mark that monitoring postprocessing is needed
# so the DCR gets updated with the correct streams
Expand Down
102 changes: 65 additions & 37 deletions src/azure-cli/azure/cli/command_modules/acs/tests/latest/test_aks_commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,10 @@ def _should_retry_for_provisioning_state(self, result):
@staticmethod
def _is_transient_operation_conflict(ex):
message = str(ex)
error_code = getattr(getattr(ex, "error", None), "code", None)
return (
error_code == "AKSOperationPreempted" or
"(AKSOperationPreempted)" in message or
"Another operation is in progress" in message or
"Operation is not allowed because there's an in-progress" in message or
"in-progress PutExtensionAddonHandler.PUT operation" in message or
Expand Down Expand Up @@ -158,6 +161,11 @@ def _execute_with_transient_conflict_retry(self, command, expect_failure):
):
show_command = self._build_show_command_for_already_existing_resource(command)
if show_command:
recovery_command = getattr(self, "_retried_create_recovery_command", None)
if recovery_command:
# A postprocessing conflict can leave metrics disabled on an
# otherwise Succeeded cluster. A GET alone cannot finish setup.
return self._execute_with_transient_conflict_retry(recovery_command, False)
return execute(self.cli_ctx, show_command, expect_failure=False)
if (
expect_failure or
Expand All @@ -176,13 +184,18 @@ def _execute_with_transient_conflict_retry(self, command, expect_failure):

raise AssertionError("unreachable")

def _cmd_with_retried_create_recovery(self, command, checks=None):
def _cmd_with_retried_create_recovery(self, command, checks=None, recovery_command=None):
previous_value = getattr(self, "_allow_retried_create_recovery", False)
previous_command = getattr(self, "_retried_create_recovery_command", None)
self._allow_retried_create_recovery = True
self._retried_create_recovery_command = (
self._apply_kwargs(recovery_command) if recovery_command else None
)
try:
return self.cmd(command, checks=checks)
finally:
self._allow_retried_create_recovery = previous_value
self._retried_create_recovery_command = previous_command

def _refetch_settled_aks_result(self, resource_id, fallback_result):
from azure.cli.testsdk.base import execute
Expand Down Expand Up @@ -370,33 +383,14 @@ def _get_latest_official_version(self, location):
return sorted_supported_versions[0] if sorted_supported_versions else None

def _create_container_insights_workspace(self, resource_group, location):
# These scenarios use MSI monitoring. The CLI provisions DCR/DCRA resources;
# the legacy OperationsManagement Containers solution is not required.
workspace_name = self.create_random_name("cliaksworkspace", 24)
workspace = self.cmd(
"monitor log-analytics workspace create "
f"--resource-group {resource_group} --name {workspace_name} --location {location}"
).get_output_in_json()
workspace_id = workspace["id"]
solution_name = f"Containers({workspace_name})"
solution_id = (
f"{workspace_id.rsplit('/providers/', 1)[0]}/providers/"
f"Microsoft.OperationsManagement/solutions/{solution_name}"
)
solution = {
"location": location,
"properties": {"workspaceResourceId": workspace_id},
"plan": {
"name": solution_name,
"publisher": "Microsoft",
"product": "OMSGallery/Containers",
"promotionCode": "",
},
}
self.kwargs["container_insights_solution"] = json.dumps(solution)
self.cmd(
f"resource create --id {solution_id} --api-version 2015-11-01-preview "
"--is-full-object --properties '{container_insights_solution}'"
)
return workspace_id
return workspace["id"]

def _create_azure_monitor_workspace(self, resource_group, location):
"""Pre-create a dedicated Azure Monitor Workspace (AMW) for a single test.
Expand All @@ -423,6 +417,25 @@ def _wait_for_cluster_update(self):
checks=[self.is_empty()],
)

def _require_availability_zones(self, location, zones):
if not (self.is_live or self.in_recording):
return
skus = self.cmd(
f'vm list-skus --location {location} --resource-type virtualMachines --all -o json'
).get_output_in_json()
location_info = [
info for sku in skus for info in (sku.get('locationInfo') or [])
if (info.get('location') or '').casefold() == location.casefold()
]
if not location_info:
raise AssertionError(f"Could not determine availability zone capabilities for {location}")
supported = {zone for info in location_info for zone in (info.get('zones') or [])}
if not set(zones).issubset(supported):
self.skipTest(
f"Scenario requires zones {zones} in {location}; Compute advertises {sorted(supported)}. "
"The configured test location is not changed."
)

def _wait_for_cluster_property(self, query, expected, attempts=20, delay=30):
if not (self.is_live or self.in_recording):
return
Expand Down Expand Up @@ -3004,6 +3017,7 @@ def test_aks_azure_service_mesh_with_egress_gateway(
preserve_default_location=True,
)
def test_aks_machine_cmds(self, resource_group, resource_group_location):
self._require_availability_zones(resource_group_location, ['1', '2', '3'])
aks_name = self.create_random_name('cliakstest', 16)
self.kwargs.update({
'resource_group': resource_group,
Expand Down Expand Up @@ -3086,6 +3100,9 @@ def test_aks_azure_service_mesh_with_ingress_gateway(self, resource_group, resou
'--aks-custom-headers=AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureServiceMeshPreview ' \
'--ssh-key-value={ssh_key_value} ' \
'--enable-azure-service-mesh --revision={revision} --output=json'
if self.is_live:
# Gateway configuration does not require a three-node cluster.
create_cmd += ' --node-count=1'
self.cmd(create_cmd, checks=[
self.check('provisioningState', 'Succeeded'),
self.check('serviceMeshProfile.mode', 'Istio'),
Expand Down Expand Up @@ -5481,16 +5498,14 @@ def test_aks_create_default_service_with_monitoring_addon_msi(self, resource_gro
])

# disable monitoring add-on
disable_addon_output = self.cmd('aks disable-addons -a monitoring -g {resource_group} -n {name}', checks=[
self.check('addonProfiles.omsagent.enabled', False),
]).get_output_in_json()
assert bool(disable_addon_output["addonProfiles"]["omsagent"]["config"]) == False
self.cmd('aks disable-addons -a monitoring -g {resource_group} -n {name}')
self._wait_for_cluster_update()
self._wait_for_cluster_property('addonProfiles.omsagent.enabled', False)

# show again
show_output = self.cmd('aks show -g {resource_group} -n {name}', checks=[
# Disabled monitoring may retain workspace and MSI authentication metadata.
self.cmd('aks show -g {resource_group} -n {name}', checks=[
self.check('addonProfiles.omsagent.enabled', False),
]).get_output_in_json()
assert bool(show_output["addonProfiles"]["omsagent"]["config"]) == False
])

# enable monitoring add-on
self.cmd('aks enable-addons -a monitoring -g {resource_group} -n {name}', checks=[
Expand Down Expand Up @@ -6620,6 +6635,7 @@ def test_aks_nodepool_scale_down_mode(self, resource_group, resource_group_locat
@AllowLargeResponse()
@AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='uksouth', preserve_default_location=True)
def test_aks_availability_zones_msi(self, resource_group, resource_group_location):
self._require_availability_zones(resource_group_location, ['1', '2', '3'])
# reset the count so in replay mode the random names will start with 0
self.test_resources_count = 0
# kwargs for string formatting
Expand Down Expand Up @@ -7517,6 +7533,7 @@ def test_aks_create_with_network_dataplane_cilium(self, resource_group, resource
@AllowLargeResponse()
@AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='uksouth', preserve_default_location=True)
def test_aks_enable_utlra_ssd(self, resource_group, resource_group_location):
self._require_availability_zones(resource_group_location, ['1', '2', '3'])
aks_name = self.create_random_name('cliakstest', 16)
self.kwargs.update({
'resource_group': resource_group,
Expand Down Expand Up @@ -8795,9 +8812,13 @@ def test_aks_create_with_azuremonitormetrics(self, resource_group, resource_grou
'--ssh-key-value={ssh_key_value} --node-vm-size={node_vm_size} --enable-managed-identity ' \
'--enable-azure-monitor-metrics --azure-monitor-workspace-resource-id={amw_id} ' \
'--enable-windows-recording-rules --output=json'
self.cmd(create_cmd, checks=[
self.check('provisioningState', 'Succeeded'),
])
self._cmd_with_retried_create_recovery(
create_cmd,
checks=[self.check('provisioningState', 'Succeeded')],
recovery_command='aks update --resource-group={resource_group} --name={name} '
'--enable-azure-monitor-metrics --azure-monitor-workspace-resource-id={amw_id} '
'--enable-windows-recording-rules',
)

# azuremonitor metrics will be set to false after initial creation command as its in the
# postprocessing step that we do an update to enable it. Adding a wait for the second put request
Expand Down Expand Up @@ -8915,6 +8936,9 @@ def test_aks_create_with_control_plane_metrics(self, resource_group, resource_gr
self._cmd_with_retried_create_recovery(
create_cmd,
checks=[self.check('provisioningState', 'Succeeded')],
recovery_command='aks update --resource-group={resource_group} --name={name} '
'--enable-azure-monitor-metrics --azure-monitor-workspace-resource-id={amw_id} '
'--enable-control-plane-metrics',
)
except Exception as ex: # pylint: disable=broad-except
message = str(ex).casefold()
Expand Down Expand Up @@ -8974,6 +8998,8 @@ def test_aks_update_with_control_plane_metrics(self, resource_group, resource_gr
self._cmd_with_retried_create_recovery(
create_cmd,
checks=[self.check('provisioningState', 'Succeeded')],
recovery_command='aks update --resource-group={resource_group} --name={name} '
'--enable-azure-monitor-metrics --azure-monitor-workspace-resource-id={amw_id}',
)

# wait for AMW background setup to complete before issuing update
Expand Down Expand Up @@ -10648,22 +10674,22 @@ def test_aks_azure_cni_overlay_migration(self, resource_group, resource_group_lo
preserve_default_location=True,
)
def test_aks_kubenet_to_cni_overlay_migration(self, resource_group, resource_group_location):
cluster_location = 'eastus' if self.is_live else resource_group_location
cluster_location = resource_group_location
_, create_version = self._get_versions(cluster_location)
aks_name = self.create_random_name('cliakstest', 16)
self.kwargs.update({
'resource_group': resource_group,
'name': aks_name,
'location': cluster_location,
'k8s_version': create_version,
'node_vm_size': 'Standard_D2s_v3',
'node_vm_size_args': '' if self.is_live else '--node-vm-size Standard_D2s_v3',
'ssh_key_value': self.generate_ssh_keys(),
})

# create
create_cmd = 'aks create --resource-group={resource_group} --name={name} --location={location} ' \
'--network-plugin kubenet --ssh-key-value={ssh_key_value} --kubernetes-version {k8s_version} ' \
'--node-vm-size {node_vm_size} ' \
'{node_vm_size_args} ' \
'--service-cidr 172.56.0.0/16 --dns-service-ip 172.56.0.10 --pod-cidr 100.112.0.0/12 ' \
'--aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayPreview'
self.cmd(create_cmd, checks=[
Expand Down Expand Up @@ -14915,6 +14941,7 @@ def test_aks_update_with_acns_transit_encryption(
self.cmd(create_cmd, checks=[self.check("provisioningState", "Succeeded")])

# update: enable WireGuard transit encryption
self._wait_for_cluster_update()
update_cmd = (
"aks update --resource-group={resource_group} --name={name} "
"--enable-acns --acns-transit-encryption-type WireGuard "
Expand Down Expand Up @@ -16163,6 +16190,7 @@ def test_aks_nodepool_update_with_localdns_config(self, resource_group, resource
self.cmd(create_cmd, checks=[self.check("provisioningState", "Succeeded")])

# Add nodepool without localdns config
self._wait_for_cluster_update()
add_cmd = (
"aks nodepool add --resource-group={resource_group} --cluster-name={name} "
"--name={nodepool_name} --node-count 1 "
Expand Down
Loading
Loading