From ecc4b14f1056e65dd393bd25b7a2226b3fd10e02 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Tue, 21 Apr 2026 20:24:36 +0000 Subject: [PATCH 01/14] aks-preview: Support BYO VNet for --enable-hosted-system automatic clusters + --disable-hosted-system Adds CLI surface for BYO VNet HOBO (hosted system pool) automatic clusters: * `--system-node-vnet-subnet-id` and `--node-vnet-subnet-id` on `az aks create` to bring your own VNet for the hosted system pool and user node pool. Must be used together with `--apiserver-subnet-id` and `--enable-hosted-system`. * `--disable-hosted-system` on `az aks create` to deterministically opt out of HOBO on automatic clusters (mutually exclusive with `--enable-hosted-system`, both gated to `--sku automatic`). Supported scenarios: 1. az aks create --sku automatic --enable-hosted-system 2. ... + --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id (NATGW) 3. ... + --outbound-type loadBalancer for BYO VNet with SLB outbound 4. az aks create --sku automatic --disable-hosted-system 5. az aks update --sku base to downgrade an automatic+HOBO cluster Validation (client-side, before PATCH): * --enable-hosted-system and --disable-hosted-system are mutually exclusive. * Both require --sku automatic. * If --enable-hosted-system is set with any of the 3 BYO subnet flags, all three must be provided; otherwise a clear error lists the missing ones. * BYO subnet flags cannot be used without --enable-hosted-system. Live-only E2E tests cover BYO+NATGW, BYO+SLB with downgrade to base SKU, and the disable opt-out path. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/HISTORY.rst | 1 + src/aks-preview/azext_aks_preview/_help.py | 28 ++ src/aks-preview/azext_aks_preview/_params.py | 15 + .../azext_aks_preview/_validators.py | 8 + src/aks-preview/azext_aks_preview/custom.py | 3 + .../managed_cluster_decorator.py | 304 ++++++++++++++++-- .../tests/latest/test_aks_commands.py | 223 ++++++++++++- 7 files changed, 552 insertions(+), 30 deletions(-) diff --git a/src/aks-preview/HISTORY.rst b/src/aks-preview/HISTORY.rst index 951a47639ec..d01efcaa6bf 100644 --- a/src/aks-preview/HISTORY.rst +++ b/src/aks-preview/HISTORY.rst @@ -11,6 +11,7 @@ To release a new version, please select a new version number (usually plus 1 to Pending +++++++ +* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-vnet-subnet-id` and `--node-vnet-subnet-id`; add `--disable-hosted-system` opt-out. 21.0.0b9 ++++++++ diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index 6d04154ca5d..f6cad3e492d 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -741,6 +741,28 @@ - name: --enable-hosted-system type: bool short-summary: Create a cluster with fully hosted system components. This applies only when creating a new automatic cluster. + long-summary: | + Deterministically opts the cluster into HOBO (Hosted Overlay System Pool). AKS hosts and manages the system node pool. + Can be combined with BYO VNet via `--system-node-vnet-subnet-id`, `--node-vnet-subnet-id`, and `--apiserver-subnet-id` + (all three must be provided together and must belong to the same VNet). Cannot be used with `--disable-hosted-system`. + - name: --disable-hosted-system + type: bool + short-summary: Opt the automatic cluster out of hosted system components. + long-summary: | + Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. + Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. + - name: --system-node-vnet-subnet-id + type: string + short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). + long-summary: | + Only valid with `--enable-hosted-system`. Must be provided together with `--node-vnet-subnet-id` + and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. + - name: --node-vnet-subnet-id + type: string + short-summary: Resource ID of the subnet joined by tenant worker nodes in BYO VNet HOBO clusters. + long-summary: | + Only valid with `--enable-hosted-system`. Must be provided together with `--system-node-vnet-subnet-id` + and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. - name: --control-plane-scaling-size --cp-scaling-size type: string short-summary: (PREVIEW) The control plane scaling size for the cluster. @@ -845,6 +867,12 @@ text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system - name: Create a kubernetes cluster with Azure Backup enabled (default Week strategy). Requires the 'dataprotection' extension. Implicitly waits for cluster creation. text: az aks create -g MyResourceGroup -n MyManagedCluster --generate-ssh-keys --enable-backup --yes + - name: Create a hosted-system automatic cluster in a BYO VNet (NAT gateway outbound, the default). + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id + - name: Create a hosted-system automatic cluster in a BYO VNet with Load Balancer outbound. + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id --outbound-type loadBalancer + - name: Create an automatic cluster and opt out of hosted system components. + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --disable-hosted-system """ diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index 3d9ee3b4f42..7e958fa0558 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -241,6 +241,8 @@ validate_duration_hours, validate_vm_set_type, validate_vnet_subnet_id, + validate_system_node_vnet_subnet_id, + validate_node_vnet_subnet_id, validate_force_upgrade_disable_and_enable_parameters, validate_azure_service_mesh_revision, validate_artifact_streaming, @@ -1308,6 +1310,19 @@ def load_arguments(self, _): help="Enable Gateway API based ingress on App Routing via Istio" ) c.argument("enable_hosted_system", action="store_true", is_preview=True) + c.argument("disable_hosted_system", action="store_true", is_preview=True) + c.argument( + "system_node_vnet_subnet_id", + options_list=["--system-node-vnet-subnet-id"], + validator=validate_system_node_vnet_subnet_id, + is_preview=True, + ) + c.argument( + "node_vnet_subnet_id", + options_list=["--node-vnet-subnet-id"], + validator=validate_node_vnet_subnet_id, + is_preview=True, + ) c.argument( "control_plane_scaling_size", options_list=["--control-plane-scaling-size", "--cp-scaling-size"], diff --git a/src/aks-preview/azext_aks_preview/_validators.py b/src/aks-preview/azext_aks_preview/_validators.py index 1031176ba8d..de20c9a6496 100644 --- a/src/aks-preview/azext_aks_preview/_validators.py +++ b/src/aks-preview/azext_aks_preview/_validators.py @@ -356,6 +356,14 @@ def validate_apiserver_subnet_id(namespace): _validate_subnet_id(namespace.apiserver_subnet_id, "--apiserver-subnet-id") +def validate_system_node_vnet_subnet_id(namespace): + _validate_subnet_id(namespace.system_node_vnet_subnet_id, "--system-node-vnet-subnet-id") + + +def validate_node_vnet_subnet_id(namespace): + _validate_subnet_id(namespace.node_vnet_subnet_id, "--node-vnet-subnet-id") + + def _validate_subnet_id(subnet_id, name): if subnet_id is None or subnet_id == '': return diff --git a/src/aks-preview/azext_aks_preview/custom.py b/src/aks-preview/azext_aks_preview/custom.py index 94e26ac3f56..516e3411671 100644 --- a/src/aks-preview/azext_aks_preview/custom.py +++ b/src/aks-preview/azext_aks_preview/custom.py @@ -1437,6 +1437,9 @@ def aks_create( # app routing istio enable_app_routing_istio=False, enable_hosted_system=False, + disable_hosted_system=False, + system_node_vnet_subnet_id=None, + node_vnet_subnet_id=None, control_plane_scaling_size=None, # health monitor enable_continuous_control_plane_and_addon_monitor=False, diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 4996e23a04d..ccaa306aa3f 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -605,12 +605,28 @@ def _get_outbound_type( # pylint: disable=too-many-branches CONST_OUTBOUND_TYPE_NONE, CONST_OUTBOUND_TYPE_BLOCK,] ): + # Preserve the user's explicit --outbound-type (if any) before default-completion + # overwrites it with CONST_OUTBOUND_TYPE_LOAD_BALANCER below. + user_supplied_outbound_type = self.raw_param.get("outbound_type") outbound_type = CONST_OUTBOUND_TYPE_LOAD_BALANCER skuName = self.get_sku_name() isVnetSubnetIdEmpty = self.get_vnet_subnet_id() in ["", None] - if skuName is not None and skuName == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and isVnetSubnetIdEmpty: - # outbound_type of Automatic SKU should be ManagedNATGateway if no subnet id provided. - outbound_type = CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY + # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-vnet-subnet-id / + # --node-vnet-subnet-id instead of --vnet-subnet-id. + hobo_byo_subnets = bool( + self.raw_param.get("system_node_vnet_subnet_id") or + self.raw_param.get("node_vnet_subnet_id") + ) + if ( + skuName is not None and skuName == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and + isVnetSubnetIdEmpty + ): + # Default outbound for Automatic SKU without a VNet is managedNATGateway. + # For BYO HOBO, only apply this default when the user did NOT explicitly pass + # --outbound-type; otherwise we would silently overwrite a user's explicit + # --outbound-type loadBalancer (regression seen in BYO SLB scenario). + if not (hobo_byo_subnets and user_supplied_outbound_type): + outbound_type = CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY # validation # Note: The parameters involved in the validation are not verified in their own getters. @@ -1991,8 +2007,18 @@ def _get_apiserver_subnet_id(self, enable_validation: bool = False) -> Union[str # validation if enable_validation: if self.decorator_mode == DecoratorMode.CREATE: + # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so misuse + # (e.g. passing --system-node-vnet-subnet-id without --enable-hosted-system) + # is rejected up front rather than silently dropped. + self._validate_byo_hobo_subnet_trio() vnet_subnet_id = self.get_vnet_subnet_id() - if apiserver_subnet_id and vnet_subnet_id is None: + # For BYO VNet HOBO automatic clusters, --system-node-vnet-subnet-id and + # --node-vnet-subnet-id replace --vnet-subnet-id. + hobo_byo_subnets = ( + self.raw_param.get("system_node_vnet_subnet_id") or + self.raw_param.get("node_vnet_subnet_id") + ) + if apiserver_subnet_id and vnet_subnet_id is None and not hobo_byo_subnets: raise RequiredArgumentMissingError( '"--apiserver-subnet-id" requires "--vnet-subnet-id".') @@ -4125,10 +4151,94 @@ def get_enable_hosted_system(self) -> bool: :return: bool """ enable_hosted_system = self.raw_param.get("enable_hosted_system") + disable_hosted_system = self.raw_param.get("disable_hosted_system") + if enable_hosted_system and disable_hosted_system: + raise MutuallyExclusiveArgumentError( + 'Cannot specify "--enable-hosted-system" and "--disable-hosted-system" at the same time.' + ) if enable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: raise RequiredArgumentMissingError('"--enable-hosted-system" requires "--sku automatic".') return enable_hosted_system + def get_disable_hosted_system(self) -> bool: + """Obtain the value of disable_hosted_system. + + :return: bool + """ + disable_hosted_system = self.raw_param.get("disable_hosted_system") + enable_hosted_system = self.raw_param.get("enable_hosted_system") + if disable_hosted_system and enable_hosted_system: + raise MutuallyExclusiveArgumentError( + 'Cannot specify "--enable-hosted-system" and "--disable-hosted-system" at the same time.' + ) + if disable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: + raise RequiredArgumentMissingError('"--disable-hosted-system" requires "--sku automatic".') + return disable_hosted_system + + def get_system_node_vnet_subnet_id(self) -> Union[str, None]: + """Obtain the value of system_node_vnet_subnet_id. + + Validates that BYO VNet subnet flags are used together with + --enable-hosted-system and that the full triple + (system-node / node / apiserver) is provided. + + :return: str or None + """ + system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") + self._validate_byo_hobo_subnet_trio() + return system_node_vnet_subnet_id + + def get_node_vnet_subnet_id(self) -> Union[str, None]: + """Obtain the value of node_vnet_subnet_id. + + :return: str or None + """ + node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") + self._validate_byo_hobo_subnet_trio() + return node_vnet_subnet_id + + def _validate_byo_hobo_subnet_trio(self) -> None: + """Cross-validate the BYO VNet HOBO subnet flags. + + Rule: if any of --system-node-vnet-subnet-id, --node-vnet-subnet-id, or + --apiserver-subnet-id is set together with --enable-hosted-system, then + all three must be set (SDK requires all three subnets to share a VNet). + Any of these flags set without --enable-hosted-system is flagged so + the user doesn't silently get a non-HOBO cluster with the wrong shape. + """ + system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") + node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") + apiserver_subnet_id = self.raw_param.get("apiserver_subnet_id") + enable_hosted_system = self.raw_param.get("enable_hosted_system") + + # The HOBO-specific flags only make sense with --enable-hosted-system. + # --apiserver-subnet-id is a general VNet-integration flag, so only + # require HOBO on it when one of the other two is also set. + hobo_byo_flags_set = bool(system_node_vnet_subnet_id) or bool(node_vnet_subnet_id) + if hobo_byo_flags_set and not enable_hosted_system: + raise RequiredArgumentMissingError( + '"--system-node-vnet-subnet-id" and "--node-vnet-subnet-id" ' + 'require "--enable-hosted-system".' + ) + + if enable_hosted_system and ( + system_node_vnet_subnet_id or node_vnet_subnet_id or apiserver_subnet_id + ): + missing = [] + if not system_node_vnet_subnet_id: + missing.append("--system-node-vnet-subnet-id") + if not node_vnet_subnet_id: + missing.append("--node-vnet-subnet-id") + if not apiserver_subnet_id: + missing.append("--apiserver-subnet-id") + if missing: + raise RequiredArgumentMissingError( + "BYO VNet for hosted-system clusters requires all of " + "--system-node-vnet-subnet-id, --node-vnet-subnet-id, and " + "--apiserver-subnet-id to be provided together. " + f"Missing: {', '.join(missing)}." + ) + def get_control_plane_scaling_size(self) -> Union[str, None]: """Obtain the value of control_plane_scaling_size. @@ -4355,29 +4465,43 @@ def set_up_run_command(self, mc: ManagedCluster) -> ManagedCluster: def set_up_api_server_access_profile(self, mc: ManagedCluster) -> ManagedCluster: """Set up apiserverAccessProfile enableVnetIntegration and subnetId for the ManagedCluster object. - Note: Inherited and extended in aks-preview to set vnet integration configs. + Note: This is a full override (not calling super()) because the base acs module writes + `enableVnetIntegration` / `subnetId` via msrest-style `additional_properties`, which the + vendored 2026-02-02-preview SDK (azure.core Model) does not expose and would raise + AttributeError. The logic below mirrors the base implementation (authorized IP ranges, + private cluster, public FQDN, private DNS zone, fqdn subdomain) but writes + `enable_vnet_integration` and `subnet_id` as typed fields. :return: the ManagedCluster object """ - mc = super().set_up_api_server_access_profile(mc) + self._ensure_mc(mc) + + api_server_access_profile = None + api_server_authorized_ip_ranges = self.context.get_api_server_authorized_ip_ranges() + enable_private_cluster = self.context.get_enable_private_cluster() + disable_public_fqdn = self.context.get_disable_public_fqdn() + private_dns_zone = self.context.get_private_dns_zone() + if api_server_authorized_ip_ranges or enable_private_cluster: + # pylint: disable=no-member + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile( + authorized_ip_ranges=api_server_authorized_ip_ranges, + enable_private_cluster=True if enable_private_cluster else None, + enable_private_cluster_public_fqdn=False if disable_public_fqdn else None, + private_dns_zone=private_dns_zone, + ) if self.context.get_enable_apiserver_vnet_integration(): - if mc.api_server_access_profile is None: + if api_server_access_profile is None: # pylint: disable=no-member - mc.api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() - mc.api_server_access_profile.enable_vnet_integration = True + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() + api_server_access_profile.enable_vnet_integration = True if self.context.get_apiserver_subnet_id(): - if mc.api_server_access_profile is None: + if api_server_access_profile is None: # pylint: disable=no-member - mc.api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() - mc.api_server_access_profile.subnet_id = self.context.get_apiserver_subnet_id() + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() + api_server_access_profile.subnet_id = self.context.get_apiserver_subnet_id() + mc.api_server_access_profile = api_server_access_profile - if ( - mc.api_server_access_profile is not None and - hasattr(mc.api_server_access_profile, 'additional_properties') and - mc.api_server_access_profile.additional_properties is not None - ): - # remove the additional properties that are set in official azure-cli/acs - mc.api_server_access_profile.additional_properties = {} + mc.fqdn_subdomain = self.context.get_fqdn_subdomain() return mc def build_gitops_addon_profile(self) -> ManagedClusterAddonProfile: @@ -5385,17 +5509,123 @@ def set_up_enable_hosted_components(self, mc: ManagedCluster) -> ManagedCluster: self._ensure_mc(mc) enable_hosted_components = self.context.get_enable_hosted_system() + disable_hosted_components = self.context.get_disable_hosted_system() if enable_hosted_components: if mc.hosted_system_profile is None: mc.hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() # pylint: disable=no-member mc.hosted_system_profile.enabled = True + # BYO VNet: plumb subnet IDs through to the SDK model. All three + # subnets (system-node / node / apiserver) must share a VNet, but + # the server enforces that check. + system_node_vnet_subnet_id = self.context.get_system_node_vnet_subnet_id() + node_vnet_subnet_id = self.context.get_node_vnet_subnet_id() + if system_node_vnet_subnet_id: + mc.hosted_system_profile.system_node_subnet_id = system_node_vnet_subnet_id + if node_vnet_subnet_id: + mc.hosted_system_profile.node_subnet_id = node_vnet_subnet_id + # Remove default agent pool profiles when hosted system profile is enabled if mc.agent_pool_profiles is not None: mc.agent_pool_profiles = None + elif disable_hosted_components: + # Deterministic opt-out: explicitly set enabled=False so customers + # don't get HOBO once the server flips the default in their region. + if mc.hosted_system_profile is None: + mc.hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() # pylint: disable=no-member + mc.hosted_system_profile.enabled = False return mc + def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> None: + """Extend base role assignment to also cover BYO VNet HOBO subnets. + + Base behavior: if ``--vnet-subnet-id`` is provided, grant Network Contributor on + that subnet to the cluster identity (SP or UAMI). BYO HOBO uses three separate + subnets (``--system-node-vnet-subnet-id``, ``--node-vnet-subnet-id``, + ``--apiserver-subnet-id``) instead of ``--vnet-subnet-id``; without this + override, cluster creation fails with ``ResourceMissingPermissionError`` on the + BYO subnets. + + Strategy: call super() so the original ``--vnet-subnet-id`` path still works, + then iterate over any HOBO BYO subnets and run the same role-assignment logic + for each. Skipping is honored via ``--skip-subnet-role-assignment``. + """ + # Preserve base behavior for the --vnet-subnet-id case. + super().process_add_role_assignment_for_vnet_subnet(mc) + + # Only extend for BYO VNet HOBO; outside that mode --apiserver-subnet-id keeps its + # generic apiserver-VNet-integration meaning and must NOT trigger an extra RBAC grant. + if not self.context.get_enable_hosted_system(): + return + + # Trio validation fires earlier via _get_apiserver_subnet_id on CREATE, so by the + # time we reach here we should have either all three HOBO subnets or none. Defend + # anyway — skip cleanly when no HOBO subnets are present. + hobo_subnets = [] + seen = set() + for raw_key in ( + "system_node_vnet_subnet_id", + "node_vnet_subnet_id", + "apiserver_subnet_id", + ): + subnet_id = self.context.raw_param.get(raw_key) + if subnet_id and subnet_id not in seen: + seen.add(subnet_id) + hobo_subnets.append(subnet_id) + + if not hobo_subnets: + return + + if self.context.get_skip_subnet_role_assignment(): + return + + service_principal_profile = mc.service_principal_profile + assign_identity = self.context.get_assign_identity() + + # For system-assigned identity clusters the SP does not exist yet and we can + # only grant after the cluster is created. Defer via the existing post-create + # flag AND stash the HOBO subnet list so the post-create handler can iterate it; + # base behavior only grants on --vnet-subnet-id, which is absent for BYO HOBO. + if service_principal_profile is None and not assign_identity: + self.context.set_intermediate( + "need_post_creation_vnet_permission_granting", + True, + overwrite_exists=True, + ) + self.context.set_intermediate( + "hobo_byo_subnets_pending_grant", + hobo_subnets, + overwrite_exists=True, + ) + return + + for subnet_id in hobo_subnets: + if self.context.external_functions.subnet_role_assignment_exists(self.cmd, subnet_id): + continue + if assign_identity: + identity_object_id = self.context.get_user_assigned_identity_object_id() + granted = self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + identity_object_id, + is_service_principal=False, + scope=subnet_id, + ) + else: + granted = self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + service_principal_profile.client_id, + scope=subnet_id, + ) + if not granted: + logger.warning( + "Could not create a role assignment for subnet %s. " + "Are you an Owner on this subscription?", + subnet_id, + ) + # pylint: disable=unused-argument def construct_mc_profile_preview(self, bypass_restore_defaults: bool = False) -> ManagedCluster: """The overall controller used to construct the default ManagedCluster profile. @@ -5570,16 +5800,32 @@ def immediate_processing_after_request(self, mc: ManagedCluster) -> None: # Grant vnet permission to system assigned identity RIGHT AFTER the cluster is put, this operation can # reduce latency for the role assignment take effect instant_cluster = self.client.get(self.context.get_resource_group_name(), self.context.get_name()) - if not self.context.external_functions.add_role_assignment( - self.cmd, - "Network Contributor", - instant_cluster.identity.principal_id, - scope=self.context.get_vnet_subnet_id(), - is_service_principal=False, - ): - logger.warning( - "Could not create a role assignment for subnet. Are you an Owner on this subscription?" - ) + # Determine the scopes to grant: base behavior uses --vnet-subnet-id only; BYO VNet HOBO + # uses the three HOBO subnets stashed by process_add_role_assignment_for_vnet_subnet. + # Iterate a list so both classic and HOBO cases share the same code path. + scopes = [] + vnet_subnet_id = self.context.get_vnet_subnet_id() + if vnet_subnet_id: + scopes.append(vnet_subnet_id) + hobo_subnets = self.context.get_intermediate( + "hobo_byo_subnets_pending_grant", default_value=[] + ) + for subnet in hobo_subnets or []: + if subnet and subnet not in scopes: + scopes.append(subnet) + for scope in scopes: + if not self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + instant_cluster.identity.principal_id, + scope=scope, + is_service_principal=False, + ): + logger.warning( + "Could not create a role assignment for subnet %s. " + "Are you an Owner on this subscription?", + scope, + ) # pylint: disable=too-many-locals,too-many-branches def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 1f8a1c86ac1..15e4cdcbd78 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -5984,7 +5984,7 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso # so we only need to test for cluster creation here create_cmd = ( "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --enable-hosted-system " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--aks-custom-header AKSHTTPCustomFeatures=Microsoft.ContainerService/AutomaticSKUPreview," "AKSHTTPCustomFeatures=Microsoft.ContainerService/AKS-AutomaticHostedSystemProfilePreview " "--ssh-key-value={ssh_key_value}" @@ -6016,6 +6016,227 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso checks=[self.is_empty()], ) + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_hosted_system_byovnet_natgw(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + vnet_name = self.create_random_name("clivnet", 16) + identity_name = self.create_random_name("cliakstest", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "vnet_name": vnet_name, + "identity_name": identity_name, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # user-assigned MSI is required for BYO VNet on automatic SKU + identity_id = self.cmd( + "identity create -g {resource_group} -n {identity_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"identity_id": identity_id}) + + # create a BYO VNet with 3 subnets: system-node, node, apiserver + self.cmd( + "network vnet create -g {resource_group} -n {vnet_name} " + "--address-prefix 10.0.0.0/8 " + "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name node --address-prefix 10.43.0.0/16" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name apiserver --address-prefix 10.44.0.0/28" + ) + system_node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " + "--query id -o tsv" + ).output.strip() + node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " + "--query id -o tsv" + ).output.strip() + apiserver_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " + "--query id -o tsv" + ).output.strip() + self.kwargs.update({ + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }) + + # BYO VNet HOBO automatic cluster (default outbound = NAT gateway) + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--enable-managed-identity --assign-identity {identity_id} " + "--system-node-vnet-subnet-id={system_node_subnet_id} " + "--node-vnet-subnet-id={node_subnet_id} " + "--apiserver-subnet-id={apiserver_subnet_id} " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", True), + self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), + self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), + self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), + ], + ) + + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + vnet_name = self.create_random_name("clivnet", 16) + identity_name = self.create_random_name("cliakstest", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "vnet_name": vnet_name, + "identity_name": identity_name, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # user-assigned MSI is required for BYO VNet on automatic SKU + identity_id = self.cmd( + "identity create -g {resource_group} -n {identity_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"identity_id": identity_id}) + + # create a BYO VNet with 3 subnets: system-node, node, apiserver + self.cmd( + "network vnet create -g {resource_group} -n {vnet_name} " + "--address-prefix 10.0.0.0/8 " + "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name node --address-prefix 10.43.0.0/16" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name apiserver --address-prefix 10.44.0.0/28" + ) + system_node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " + "--query id -o tsv" + ).output.strip() + node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " + "--query id -o tsv" + ).output.strip() + apiserver_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " + "--query id -o tsv" + ).output.strip() + self.kwargs.update({ + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }) + + # BYO VNet HOBO automatic cluster with loadBalancer outbound + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--enable-managed-identity --assign-identity {identity_id} " + "--system-node-vnet-subnet-id={system_node_subnet_id} " + "--node-vnet-subnet-id={node_subnet_id} " + "--apiserver-subnet-id={apiserver_subnet_id} " + "--outbound-type loadBalancer " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", True), + self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), + self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), + self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), + self.check("networkProfile.outboundType", "loadBalancer"), + ], + ) + + # convert to Base SKU; expect the transition to succeed for the BYO VNet HOBO case. + # Wait for the cluster to become idle (no in-progress RP reconciliation) before starting + # the update; without this the RP returns 409 OperationNotAllowed for a few minutes after + # create completes while it finishes post-create reconciliation. + self.cmd( + "aks wait -g {resource_group} -n {name} --created --timeout 900", + checks=[self.is_empty()], + ) + time.sleep(180) + self.cmd( + "aks update -g {resource_group} -n {name} --sku base", + checks=[ + self.check("sku.name", "Base"), + ], + ) + + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_with_hosted_system_disabled(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # opt out of HOBO even if the region defaults to it + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --disable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", False), + ], + ) + + # agent pool profiles should be present (non-HOBO automatic) + self.cmd( + "aks show --resource-group={resource_group} --name={name}", + checks=[self.greater_than("length(agentPoolProfiles)", 0)], + ) + @AllowLargeResponse() @AKSCustomResourceGroupPreparer( random_name_length=17, name_prefix="clitest", location="westus2" From 619eed81a78912f585c16db4912451ebcf74bfb6 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 00:59:25 +0000 Subject: [PATCH 02/14] aks-preview: Address CI linter/style + Copilot review feedback * Add short alias --sys-node-subnet-id for --system-node-vnet-subnet-id to satisfy option_length_too_long linter rule. * Rename skuName/isVnetSubnetIdEmpty to snake_case per PEP 8. * Disable too-many-branches pylint warning on _get_outbound_type (overridden from base azure-cli and the preview-specific branches are necessary). * Replace fixed 180s sleep before aks update --sku base with a retry loop that handles the RP's post-create 409 OperationNotAllowed window more robustly. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/azext_aks_preview/_params.py | 2 +- .../managed_cluster_decorator.py | 11 +++---- .../tests/latest/test_aks_commands.py | 29 ++++++++++++++----- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index 7e958fa0558..2774a61bbf6 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -1313,7 +1313,7 @@ def load_arguments(self, _): c.argument("disable_hosted_system", action="store_true", is_preview=True) c.argument( "system_node_vnet_subnet_id", - options_list=["--system-node-vnet-subnet-id"], + options_list=["--system-node-vnet-subnet-id", "--sys-node-subnet-id"], validator=validate_system_node_vnet_subnet_id, is_preview=True, ) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index ccaa306aa3f..93ec28bc0b3 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -548,7 +548,8 @@ def _get_disable_local_accounts(self, enable_validation: bool = False) -> bool: ) return disable_local_accounts - def _get_outbound_type( # pylint: disable=too-many-branches + # pylint: disable=too-many-branches + def _get_outbound_type( self, enable_validation: bool = False, read_only: bool = False, @@ -609,8 +610,8 @@ def _get_outbound_type( # pylint: disable=too-many-branches # overwrites it with CONST_OUTBOUND_TYPE_LOAD_BALANCER below. user_supplied_outbound_type = self.raw_param.get("outbound_type") outbound_type = CONST_OUTBOUND_TYPE_LOAD_BALANCER - skuName = self.get_sku_name() - isVnetSubnetIdEmpty = self.get_vnet_subnet_id() in ["", None] + sku_name = self.get_sku_name() + is_vnet_subnet_id_empty = self.get_vnet_subnet_id() in ["", None] # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-vnet-subnet-id / # --node-vnet-subnet-id instead of --vnet-subnet-id. hobo_byo_subnets = bool( @@ -618,8 +619,8 @@ def _get_outbound_type( # pylint: disable=too-many-branches self.raw_param.get("node_vnet_subnet_id") ) if ( - skuName is not None and skuName == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and - isVnetSubnetIdEmpty + sku_name is not None and sku_name == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and + is_vnet_subnet_id_empty ): # Default outbound for Automatic SKU without a VNet is managedNATGateway. # For BYO HOBO, only apply this default when the user did NOT explicitly pass diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 15e4cdcbd78..3887fd58438 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6190,13 +6190,28 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou "aks wait -g {resource_group} -n {name} --created --timeout 900", checks=[self.is_empty()], ) - time.sleep(180) - self.cmd( - "aks update -g {resource_group} -n {name} --sku base", - checks=[ - self.check("sku.name", "Base"), - ], - ) + # Poll with retries: `aks wait --created` returns once provisioningState hits Succeeded, + # but the RP can still have an internal in-progress reconciliation for several minutes + # afterwards which surfaces as 409 OperationNotAllowed on `aks update`. Back off and retry. + update_deadline = time.time() + 900 + last_error = None + while time.time() < update_deadline: + try: + self.cmd( + "aks update -g {resource_group} -n {name} --sku base", + checks=[self.check("sku.name", "Base")], + ) + last_error = None + break + except Exception as ex: # pylint: disable=broad-except + message = str(ex) + if "OperationNotAllowed" in message or "in progress" in message or "409" in message: + last_error = ex + time.sleep(60) + continue + raise + if last_error is not None: + raise last_error @live_only() @AllowLargeResponse() From 5e2f2fcb89ea180a8df77aee8c566bf3d1c70e4f Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 01:35:46 +0000 Subject: [PATCH 03/14] aks-preview: Fix help name for --system-node-vnet-subnet-id alias Include both --system-node-vnet-subnet-id and --sys-node-subnet-id in the help entry name so azdev linter recognizes all option aliases and does not report unrecognized_help_parameter_rule / missing_parameter_help. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/azext_aks_preview/_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index f6cad3e492d..46e6828b70d 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -751,7 +751,7 @@ long-summary: | Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. - - name: --system-node-vnet-subnet-id + - name: --system-node-vnet-subnet-id --sys-node-subnet-id type: string short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). long-summary: | From a33e4b11427d51cacc4ca25d959b44a0b6855160 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 01:43:24 +0000 Subject: [PATCH 04/14] aks-preview: Sort options in help name to match azdev linter expectation The azdev linter compares help parameter names against knack's HelpParameter.name which is built by sorting options alphabetically (knack/help.py line 349). Swap the order so --sys-node-subnet-id comes before --system-node-vnet-subnet-id. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/azext_aks_preview/_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index 46e6828b70d..2bdcc40fa53 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -751,7 +751,7 @@ long-summary: | Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. - - name: --system-node-vnet-subnet-id --sys-node-subnet-id + - name: --sys-node-subnet-id --system-node-vnet-subnet-id type: string short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). long-summary: | From e633cdc8e56b7e4be2183b779b35f614abf37556 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:52:33 +0000 Subject: [PATCH 05/14] aks-preview: Allow userAssignedNATGateway outbound for BYO HOBO The _get_outbound_type validation previously required --vnet-subnet-id when outbound_type is userAssignedNATGateway / userDefinedRouting. For BYO HOBO automatic clusters the VNet is provided via --system-node-vnet-subnet-id / --node-vnet-subnet-id instead, so treat those as satisfying the requirement. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../azext_aks_preview/managed_cluster_decorator.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 93ec28bc0b3..05bd6e66945 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -648,7 +648,17 @@ def _get_outbound_type( CONST_OUTBOUND_TYPE_USER_DEFINED_ROUTING, CONST_OUTBOUND_TYPE_USER_ASSIGNED_NAT_GATEWAY, ]: - if self.get_vnet_subnet_id() in ["", None]: + # BYO HOBO scenarios satisfy the VNet requirement via + # --system-node-vnet-subnet-id / --node-vnet-subnet-id + # instead of --vnet-subnet-id. + hobo_byo_has_subnet = bool( + self.raw_param.get("system_node_vnet_subnet_id") or + self.raw_param.get("node_vnet_subnet_id") + ) + if ( + self.get_vnet_subnet_id() in ["", None] and + not hobo_byo_has_subnet + ): if self.decorator_mode == DecoratorMode.CREATE: raise RequiredArgumentMissingError( "--vnet-subnet-id must be specified for userDefinedRouting and it must " From a28b1a195443c6b1b8188c294b752fec7b851ad9 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:42:52 +0000 Subject: [PATCH 06/14] test: add BYO HOBO userAssignedNATGateway live test Adds test_aks_automatic_sku_hosted_system_byovnet_user_natgw covering BYO VNet hosted-system automatic clusters with userAssignedNATGateway outbound type, exercising the _get_outbound_type fix that treats BYO HOBO subnets as satisfying the VNet requirement. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../tests/latest/test_aks_commands.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 3887fd58438..42da4fefbc4 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6213,6 +6213,119 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou if last_error is not None: raise last_error + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + vnet_name = self.create_random_name("clivnet", 16) + identity_name = self.create_random_name("cliakstest", 16) + natgw_name = self.create_random_name("clinatgw", 16) + pip_name = self.create_random_name("clinatpip", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "vnet_name": vnet_name, + "identity_name": identity_name, + "natgw_name": natgw_name, + "pip_name": pip_name, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # user-assigned MSI is required for BYO VNet on automatic SKU + identity_id = self.cmd( + "identity create -g {resource_group} -n {identity_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"identity_id": identity_id}) + + # create a BYO VNet with 3 subnets: system-node, node, apiserver + self.cmd( + "network vnet create -g {resource_group} -n {vnet_name} " + "--address-prefix 10.0.0.0/8 " + "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name node --address-prefix 10.43.0.0/16" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name apiserver --address-prefix 10.44.0.0/28" + ) + + # Create a user-assigned NAT gateway and attach it to the node subnet + self.cmd( + "network public-ip create -g {resource_group} -n {pip_name} " + "--sku Standard --allocation-method Static" + ) + self.cmd( + "network nat gateway create -g {resource_group} -n {natgw_name} " + "--public-ip-addresses {pip_name} --idle-timeout 10" + ) + natgw_id = self.cmd( + "network nat gateway show -g {resource_group} -n {natgw_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"natgw_id": natgw_id}) + + # Attach the NAT gateway to both the system-node and node subnets (egress paths) + self.cmd( + "network vnet subnet update -g {resource_group} --vnet-name {vnet_name} " + "--name systemnode --nat-gateway {natgw_id}" + ) + self.cmd( + "network vnet subnet update -g {resource_group} --vnet-name {vnet_name} " + "--name node --nat-gateway {natgw_id}" + ) + + system_node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " + "--query id -o tsv" + ).output.strip() + node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " + "--query id -o tsv" + ).output.strip() + apiserver_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " + "--query id -o tsv" + ).output.strip() + self.kwargs.update({ + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }) + + # BYO VNet HOBO automatic cluster with userAssignedNATGateway outbound + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--enable-managed-identity --assign-identity {identity_id} " + "--system-node-vnet-subnet-id={system_node_subnet_id} " + "--node-vnet-subnet-id={node_subnet_id} " + "--apiserver-subnet-id={apiserver_subnet_id} " + "--outbound-type userAssignedNATGateway " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", True), + self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), + self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), + self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), + self.check("networkProfile.outboundType", "userAssignedNATGateway"), + ], + ) + @live_only() @AllowLargeResponse() @AKSCustomResourceGroupPreparer( From 96743d6a030a8833f70eb1137c1f638dc575efe7 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:10:21 +0000 Subject: [PATCH 07/14] aks-preview: Fail-fast BYO HOBO trio validation before RBAC grants Previously _validate_byo_hobo_subnet_trio only ran when set_up_api_server_access_profile invoked get_apiserver_subnet_id, which in the base construct_mc_profile_default flow runs AFTER process_add_role_assignment_for_vnet_subnet. A malformed BYO HOBO create (partial subnet trio, or HOBO subnet flags without --enable-hosted-system) could therefore leave residual Network Contributor grants on customer subnets before the CLI surface-level validation fired. Move trio validation to the start of the role-assignment override so misuse fails before any RBAC mutation happens. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../azext_aks_preview/managed_cluster_decorator.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 05bd6e66945..0d094f33b2f 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -5562,6 +5562,14 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non then iterate over any HOBO BYO subnets and run the same role-assignment logic for each. Skipping is honored via ``--skip-subnet-role-assignment``. """ + # Fail-fast validation BEFORE any role assignment runs, so a malformed BYO HOBO + # create (e.g. partial subnet trio, --system-node-vnet-subnet-id without + # --enable-hosted-system) cannot leave residual Network Contributor grants on + # customer subnets. Trio validation is otherwise invoked later through + # set_up_api_server_access_profile, which executes AFTER this method in the base + # construct_mc_profile_default flow. + self.context._validate_byo_hobo_subnet_trio() # pylint: disable=protected-access + # Preserve base behavior for the --vnet-subnet-id case. super().process_add_role_assignment_for_vnet_subnet(mc) @@ -5570,9 +5578,8 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non if not self.context.get_enable_hosted_system(): return - # Trio validation fires earlier via _get_apiserver_subnet_id on CREATE, so by the - # time we reach here we should have either all three HOBO subnets or none. Defend - # anyway — skip cleanly when no HOBO subnets are present. + # By the time we reach here trio validation has already passed, so we have either + # all three HOBO subnets or none. Defend anyway — skip cleanly when absent. hobo_subnets = [] seen = set() for raw_key in ( From 38651f283e432ef82d686f314cd1523032332d74 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:16:52 +0000 Subject: [PATCH 08/14] aks-preview: Consolidate BYO HOBO subnet presence check into helper Refactor-only (no behavior change): introduce public AKSPreviewManagedClusterContext.has_byo_hobo_subnets() and replace three inline duplicate 'system_node_vnet_subnet_id or node_vnet_subnet_id' checks (in _get_outbound_type default-completion, _get_outbound_type validation, and get_api_server_access_profile validation) with calls to it. Also rename _validate_byo_hobo_subnet_trio to validate_byo_hobo_subnet_trio (drop the leading underscore) so the CreateDecorator override can call it directly without a pylint protected-access exception. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../managed_cluster_decorator.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 0d094f33b2f..86b5feb6d25 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -614,10 +614,7 @@ def _get_outbound_type( is_vnet_subnet_id_empty = self.get_vnet_subnet_id() in ["", None] # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-vnet-subnet-id / # --node-vnet-subnet-id instead of --vnet-subnet-id. - hobo_byo_subnets = bool( - self.raw_param.get("system_node_vnet_subnet_id") or - self.raw_param.get("node_vnet_subnet_id") - ) + hobo_byo_subnets = self.has_byo_hobo_subnets() if ( sku_name is not None and sku_name == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and is_vnet_subnet_id_empty @@ -651,13 +648,9 @@ def _get_outbound_type( # BYO HOBO scenarios satisfy the VNet requirement via # --system-node-vnet-subnet-id / --node-vnet-subnet-id # instead of --vnet-subnet-id. - hobo_byo_has_subnet = bool( - self.raw_param.get("system_node_vnet_subnet_id") or - self.raw_param.get("node_vnet_subnet_id") - ) if ( self.get_vnet_subnet_id() in ["", None] and - not hobo_byo_has_subnet + not self.has_byo_hobo_subnets() ): if self.decorator_mode == DecoratorMode.CREATE: raise RequiredArgumentMissingError( @@ -2021,15 +2014,15 @@ def _get_apiserver_subnet_id(self, enable_validation: bool = False) -> Union[str # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so misuse # (e.g. passing --system-node-vnet-subnet-id without --enable-hosted-system) # is rejected up front rather than silently dropped. - self._validate_byo_hobo_subnet_trio() + self.validate_byo_hobo_subnet_trio() vnet_subnet_id = self.get_vnet_subnet_id() # For BYO VNet HOBO automatic clusters, --system-node-vnet-subnet-id and # --node-vnet-subnet-id replace --vnet-subnet-id. - hobo_byo_subnets = ( - self.raw_param.get("system_node_vnet_subnet_id") or - self.raw_param.get("node_vnet_subnet_id") - ) - if apiserver_subnet_id and vnet_subnet_id is None and not hobo_byo_subnets: + if ( + apiserver_subnet_id and + vnet_subnet_id is None and + not self.has_byo_hobo_subnets() + ): raise RequiredArgumentMissingError( '"--apiserver-subnet-id" requires "--vnet-subnet-id".') @@ -4196,7 +4189,7 @@ def get_system_node_vnet_subnet_id(self) -> Union[str, None]: :return: str or None """ system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") - self._validate_byo_hobo_subnet_trio() + self.validate_byo_hobo_subnet_trio() return system_node_vnet_subnet_id def get_node_vnet_subnet_id(self) -> Union[str, None]: @@ -4205,10 +4198,22 @@ def get_node_vnet_subnet_id(self) -> Union[str, None]: :return: str or None """ node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") - self._validate_byo_hobo_subnet_trio() + self.validate_byo_hobo_subnet_trio() return node_vnet_subnet_id - def _validate_byo_hobo_subnet_trio(self) -> None: + def has_byo_hobo_subnets(self) -> bool: + """Return True when at least one of the BYO HOBO node subnet flags is set. + + The apiserver subnet flag is intentionally excluded here: it is a general + VNet-integration flag (not HOBO-specific) and including it would mis-classify + non-HOBO apiserver-vnet-integration clusters as BYO HOBO. + """ + return bool( + self.raw_param.get("system_node_vnet_subnet_id") or + self.raw_param.get("node_vnet_subnet_id") + ) + + def validate_byo_hobo_subnet_trio(self) -> None: """Cross-validate the BYO VNet HOBO subnet flags. Rule: if any of --system-node-vnet-subnet-id, --node-vnet-subnet-id, or @@ -4225,8 +4230,7 @@ def _validate_byo_hobo_subnet_trio(self) -> None: # The HOBO-specific flags only make sense with --enable-hosted-system. # --apiserver-subnet-id is a general VNet-integration flag, so only # require HOBO on it when one of the other two is also set. - hobo_byo_flags_set = bool(system_node_vnet_subnet_id) or bool(node_vnet_subnet_id) - if hobo_byo_flags_set and not enable_hosted_system: + if self.has_byo_hobo_subnets() and not enable_hosted_system: raise RequiredArgumentMissingError( '"--system-node-vnet-subnet-id" and "--node-vnet-subnet-id" ' 'require "--enable-hosted-system".' @@ -5568,7 +5572,7 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non # customer subnets. Trio validation is otherwise invoked later through # set_up_api_server_access_profile, which executes AFTER this method in the base # construct_mc_profile_default flow. - self.context._validate_byo_hobo_subnet_trio() # pylint: disable=protected-access + self.context.validate_byo_hobo_subnet_trio() # Preserve base behavior for the --vnet-subnet-id case. super().process_add_role_assignment_for_vnet_subnet(mc) From cca9cc6a7a7fd925844d56ea1f1a6f0a12af82f3 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:59:06 +0000 Subject: [PATCH 09/14] aks-preview: Rename BYO HOBO subnet flags per review feedback Rename --system-node-vnet-subnet-id -> --system-node-subnet-id and --node-vnet-subnet-id -> --node-subnet-id (with Python identifiers system_node_subnet_id / node_subnet_id) per @zqingqing1 review feedback on PR #9812. The --sys-node-subnet-id alias is retained. Also drop the BYO VNet combination paragraph from the --enable-hosted-system long-summary per PM guidance. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/HISTORY.rst | 2 +- src/aks-preview/azext_aks_preview/_help.py | 15 ++-- src/aks-preview/azext_aks_preview/_params.py | 16 ++--- .../azext_aks_preview/_validators.py | 8 +-- src/aks-preview/azext_aks_preview/custom.py | 4 +- .../managed_cluster_decorator.py | 72 +++++++++---------- .../tests/latest/test_aks_commands.py | 12 ++-- 7 files changed, 64 insertions(+), 65 deletions(-) diff --git a/src/aks-preview/HISTORY.rst b/src/aks-preview/HISTORY.rst index d01efcaa6bf..cc45766c096 100644 --- a/src/aks-preview/HISTORY.rst +++ b/src/aks-preview/HISTORY.rst @@ -11,7 +11,7 @@ To release a new version, please select a new version number (usually plus 1 to Pending +++++++ -* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-vnet-subnet-id` and `--node-vnet-subnet-id`; add `--disable-hosted-system` opt-out. +* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-subnet-id` and `--node-subnet-id`; add `--disable-hosted-system` opt-out. 21.0.0b9 ++++++++ diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index 2bdcc40fa53..be0577308d9 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -743,25 +743,24 @@ short-summary: Create a cluster with fully hosted system components. This applies only when creating a new automatic cluster. long-summary: | Deterministically opts the cluster into HOBO (Hosted Overlay System Pool). AKS hosts and manages the system node pool. - Can be combined with BYO VNet via `--system-node-vnet-subnet-id`, `--node-vnet-subnet-id`, and `--apiserver-subnet-id` - (all three must be provided together and must belong to the same VNet). Cannot be used with `--disable-hosted-system`. + Cannot be used with `--disable-hosted-system`. - name: --disable-hosted-system type: bool short-summary: Opt the automatic cluster out of hosted system components. long-summary: | Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. - - name: --sys-node-subnet-id --system-node-vnet-subnet-id + - name: --sys-node-subnet-id --system-node-subnet-id type: string short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). long-summary: | - Only valid with `--enable-hosted-system`. Must be provided together with `--node-vnet-subnet-id` + Only valid with `--enable-hosted-system`. Must be provided together with `--node-subnet-id` and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. - - name: --node-vnet-subnet-id + - name: --node-subnet-id type: string short-summary: Resource ID of the subnet joined by tenant worker nodes in BYO VNet HOBO clusters. long-summary: | - Only valid with `--enable-hosted-system`. Must be provided together with `--system-node-vnet-subnet-id` + Only valid with `--enable-hosted-system`. Must be provided together with `--system-node-subnet-id` and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. - name: --control-plane-scaling-size --cp-scaling-size type: string @@ -868,9 +867,9 @@ - name: Create a kubernetes cluster with Azure Backup enabled (default Week strategy). Requires the 'dataprotection' extension. Implicitly waits for cluster creation. text: az aks create -g MyResourceGroup -n MyManagedCluster --generate-ssh-keys --enable-backup --yes - name: Create a hosted-system automatic cluster in a BYO VNet (NAT gateway outbound, the default). - text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-subnet-id --node-subnet-id --apiserver-subnet-id - name: Create a hosted-system automatic cluster in a BYO VNet with Load Balancer outbound. - text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id --outbound-type loadBalancer + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-subnet-id --node-subnet-id --apiserver-subnet-id --outbound-type loadBalancer - name: Create an automatic cluster and opt out of hosted system components. text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --disable-hosted-system diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index 2774a61bbf6..568d7b740b5 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -241,8 +241,8 @@ validate_duration_hours, validate_vm_set_type, validate_vnet_subnet_id, - validate_system_node_vnet_subnet_id, - validate_node_vnet_subnet_id, + validate_system_node_subnet_id, + validate_node_subnet_id, validate_force_upgrade_disable_and_enable_parameters, validate_azure_service_mesh_revision, validate_artifact_streaming, @@ -1312,15 +1312,15 @@ def load_arguments(self, _): c.argument("enable_hosted_system", action="store_true", is_preview=True) c.argument("disable_hosted_system", action="store_true", is_preview=True) c.argument( - "system_node_vnet_subnet_id", - options_list=["--system-node-vnet-subnet-id", "--sys-node-subnet-id"], - validator=validate_system_node_vnet_subnet_id, + "system_node_subnet_id", + options_list=["--system-node-subnet-id", "--sys-node-subnet-id"], + validator=validate_system_node_subnet_id, is_preview=True, ) c.argument( - "node_vnet_subnet_id", - options_list=["--node-vnet-subnet-id"], - validator=validate_node_vnet_subnet_id, + "node_subnet_id", + options_list=["--node-subnet-id"], + validator=validate_node_subnet_id, is_preview=True, ) c.argument( diff --git a/src/aks-preview/azext_aks_preview/_validators.py b/src/aks-preview/azext_aks_preview/_validators.py index de20c9a6496..b84add3f8bb 100644 --- a/src/aks-preview/azext_aks_preview/_validators.py +++ b/src/aks-preview/azext_aks_preview/_validators.py @@ -356,12 +356,12 @@ def validate_apiserver_subnet_id(namespace): _validate_subnet_id(namespace.apiserver_subnet_id, "--apiserver-subnet-id") -def validate_system_node_vnet_subnet_id(namespace): - _validate_subnet_id(namespace.system_node_vnet_subnet_id, "--system-node-vnet-subnet-id") +def validate_system_node_subnet_id(namespace): + _validate_subnet_id(namespace.system_node_subnet_id, "--system-node-subnet-id") -def validate_node_vnet_subnet_id(namespace): - _validate_subnet_id(namespace.node_vnet_subnet_id, "--node-vnet-subnet-id") +def validate_node_subnet_id(namespace): + _validate_subnet_id(namespace.node_subnet_id, "--node-subnet-id") def _validate_subnet_id(subnet_id, name): diff --git a/src/aks-preview/azext_aks_preview/custom.py b/src/aks-preview/azext_aks_preview/custom.py index 516e3411671..fab5569d6c1 100644 --- a/src/aks-preview/azext_aks_preview/custom.py +++ b/src/aks-preview/azext_aks_preview/custom.py @@ -1438,8 +1438,8 @@ def aks_create( enable_app_routing_istio=False, enable_hosted_system=False, disable_hosted_system=False, - system_node_vnet_subnet_id=None, - node_vnet_subnet_id=None, + system_node_subnet_id=None, + node_subnet_id=None, control_plane_scaling_size=None, # health monitor enable_continuous_control_plane_and_addon_monitor=False, diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 86b5feb6d25..6cd1e0733f0 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -612,8 +612,8 @@ def _get_outbound_type( outbound_type = CONST_OUTBOUND_TYPE_LOAD_BALANCER sku_name = self.get_sku_name() is_vnet_subnet_id_empty = self.get_vnet_subnet_id() in ["", None] - # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-vnet-subnet-id / - # --node-vnet-subnet-id instead of --vnet-subnet-id. + # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-subnet-id / + # --node-subnet-id instead of --vnet-subnet-id. hobo_byo_subnets = self.has_byo_hobo_subnets() if ( sku_name is not None and sku_name == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and @@ -646,7 +646,7 @@ def _get_outbound_type( CONST_OUTBOUND_TYPE_USER_ASSIGNED_NAT_GATEWAY, ]: # BYO HOBO scenarios satisfy the VNet requirement via - # --system-node-vnet-subnet-id / --node-vnet-subnet-id + # --system-node-subnet-id / --node-subnet-id # instead of --vnet-subnet-id. if ( self.get_vnet_subnet_id() in ["", None] and @@ -2012,12 +2012,12 @@ def _get_apiserver_subnet_id(self, enable_validation: bool = False) -> Union[str if enable_validation: if self.decorator_mode == DecoratorMode.CREATE: # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so misuse - # (e.g. passing --system-node-vnet-subnet-id without --enable-hosted-system) + # (e.g. passing --system-node-subnet-id without --enable-hosted-system) # is rejected up front rather than silently dropped. self.validate_byo_hobo_subnet_trio() vnet_subnet_id = self.get_vnet_subnet_id() - # For BYO VNet HOBO automatic clusters, --system-node-vnet-subnet-id and - # --node-vnet-subnet-id replace --vnet-subnet-id. + # For BYO VNet HOBO automatic clusters, --system-node-subnet-id and + # --node-subnet-id replace --vnet-subnet-id. if ( apiserver_subnet_id and vnet_subnet_id is None and @@ -4179,8 +4179,8 @@ def get_disable_hosted_system(self) -> bool: raise RequiredArgumentMissingError('"--disable-hosted-system" requires "--sku automatic".') return disable_hosted_system - def get_system_node_vnet_subnet_id(self) -> Union[str, None]: - """Obtain the value of system_node_vnet_subnet_id. + def get_system_node_subnet_id(self) -> Union[str, None]: + """Obtain the value of system_node_subnet_id. Validates that BYO VNet subnet flags are used together with --enable-hosted-system and that the full triple @@ -4188,18 +4188,18 @@ def get_system_node_vnet_subnet_id(self) -> Union[str, None]: :return: str or None """ - system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") + system_node_subnet_id = self.raw_param.get("system_node_subnet_id") self.validate_byo_hobo_subnet_trio() - return system_node_vnet_subnet_id + return system_node_subnet_id - def get_node_vnet_subnet_id(self) -> Union[str, None]: - """Obtain the value of node_vnet_subnet_id. + def get_node_subnet_id(self) -> Union[str, None]: + """Obtain the value of node_subnet_id. :return: str or None """ - node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") + node_subnet_id = self.raw_param.get("node_subnet_id") self.validate_byo_hobo_subnet_trio() - return node_vnet_subnet_id + return node_subnet_id def has_byo_hobo_subnets(self) -> bool: """Return True when at least one of the BYO HOBO node subnet flags is set. @@ -4209,21 +4209,21 @@ def has_byo_hobo_subnets(self) -> bool: non-HOBO apiserver-vnet-integration clusters as BYO HOBO. """ return bool( - self.raw_param.get("system_node_vnet_subnet_id") or - self.raw_param.get("node_vnet_subnet_id") + self.raw_param.get("system_node_subnet_id") or + self.raw_param.get("node_subnet_id") ) def validate_byo_hobo_subnet_trio(self) -> None: """Cross-validate the BYO VNet HOBO subnet flags. - Rule: if any of --system-node-vnet-subnet-id, --node-vnet-subnet-id, or + Rule: if any of --system-node-subnet-id, --node-subnet-id, or --apiserver-subnet-id is set together with --enable-hosted-system, then all three must be set (SDK requires all three subnets to share a VNet). Any of these flags set without --enable-hosted-system is flagged so the user doesn't silently get a non-HOBO cluster with the wrong shape. """ - system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") - node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") + system_node_subnet_id = self.raw_param.get("system_node_subnet_id") + node_subnet_id = self.raw_param.get("node_subnet_id") apiserver_subnet_id = self.raw_param.get("apiserver_subnet_id") enable_hosted_system = self.raw_param.get("enable_hosted_system") @@ -4232,24 +4232,24 @@ def validate_byo_hobo_subnet_trio(self) -> None: # require HOBO on it when one of the other two is also set. if self.has_byo_hobo_subnets() and not enable_hosted_system: raise RequiredArgumentMissingError( - '"--system-node-vnet-subnet-id" and "--node-vnet-subnet-id" ' + '"--system-node-subnet-id" and "--node-subnet-id" ' 'require "--enable-hosted-system".' ) if enable_hosted_system and ( - system_node_vnet_subnet_id or node_vnet_subnet_id or apiserver_subnet_id + system_node_subnet_id or node_subnet_id or apiserver_subnet_id ): missing = [] - if not system_node_vnet_subnet_id: - missing.append("--system-node-vnet-subnet-id") - if not node_vnet_subnet_id: - missing.append("--node-vnet-subnet-id") + if not system_node_subnet_id: + missing.append("--system-node-subnet-id") + if not node_subnet_id: + missing.append("--node-subnet-id") if not apiserver_subnet_id: missing.append("--apiserver-subnet-id") if missing: raise RequiredArgumentMissingError( "BYO VNet for hosted-system clusters requires all of " - "--system-node-vnet-subnet-id, --node-vnet-subnet-id, and " + "--system-node-subnet-id, --node-subnet-id, and " "--apiserver-subnet-id to be provided together. " f"Missing: {', '.join(missing)}." ) @@ -5533,12 +5533,12 @@ def set_up_enable_hosted_components(self, mc: ManagedCluster) -> ManagedCluster: # BYO VNet: plumb subnet IDs through to the SDK model. All three # subnets (system-node / node / apiserver) must share a VNet, but # the server enforces that check. - system_node_vnet_subnet_id = self.context.get_system_node_vnet_subnet_id() - node_vnet_subnet_id = self.context.get_node_vnet_subnet_id() - if system_node_vnet_subnet_id: - mc.hosted_system_profile.system_node_subnet_id = system_node_vnet_subnet_id - if node_vnet_subnet_id: - mc.hosted_system_profile.node_subnet_id = node_vnet_subnet_id + system_node_subnet_id = self.context.get_system_node_subnet_id() + node_subnet_id = self.context.get_node_subnet_id() + if system_node_subnet_id: + mc.hosted_system_profile.system_node_subnet_id = system_node_subnet_id + if node_subnet_id: + mc.hosted_system_profile.node_subnet_id = node_subnet_id # Remove default agent pool profiles when hosted system profile is enabled if mc.agent_pool_profiles is not None: @@ -5557,7 +5557,7 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non Base behavior: if ``--vnet-subnet-id`` is provided, grant Network Contributor on that subnet to the cluster identity (SP or UAMI). BYO HOBO uses three separate - subnets (``--system-node-vnet-subnet-id``, ``--node-vnet-subnet-id``, + subnets (``--system-node-subnet-id``, ``--node-subnet-id``, ``--apiserver-subnet-id``) instead of ``--vnet-subnet-id``; without this override, cluster creation fails with ``ResourceMissingPermissionError`` on the BYO subnets. @@ -5567,7 +5567,7 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non for each. Skipping is honored via ``--skip-subnet-role-assignment``. """ # Fail-fast validation BEFORE any role assignment runs, so a malformed BYO HOBO - # create (e.g. partial subnet trio, --system-node-vnet-subnet-id without + # create (e.g. partial subnet trio, --system-node-subnet-id without # --enable-hosted-system) cannot leave residual Network Contributor grants on # customer subnets. Trio validation is otherwise invoked later through # set_up_api_server_access_profile, which executes AFTER this method in the base @@ -5587,8 +5587,8 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non hobo_subnets = [] seen = set() for raw_key in ( - "system_node_vnet_subnet_id", - "node_vnet_subnet_id", + "system_node_subnet_id", + "node_subnet_id", "apiserver_subnet_id", ): subnet_id = self.context.raw_param.get(raw_key) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 42da4fefbc4..55808463161 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6081,8 +6081,8 @@ def test_aks_automatic_sku_hosted_system_byovnet_natgw(self, resource_group, res "aks create --resource-group={resource_group} --name={name} --location={location} " "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--enable-managed-identity --assign-identity {identity_id} " - "--system-node-vnet-subnet-id={system_node_subnet_id} " - "--node-vnet-subnet-id={node_subnet_id} " + "--system-node-subnet-id={system_node_subnet_id} " + "--node-subnet-id={node_subnet_id} " "--apiserver-subnet-id={apiserver_subnet_id} " "--ssh-key-value={ssh_key_value}" ) @@ -6163,8 +6163,8 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou "aks create --resource-group={resource_group} --name={name} --location={location} " "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--enable-managed-identity --assign-identity {identity_id} " - "--system-node-vnet-subnet-id={system_node_subnet_id} " - "--node-vnet-subnet-id={node_subnet_id} " + "--system-node-subnet-id={system_node_subnet_id} " + "--node-subnet-id={node_subnet_id} " "--apiserver-subnet-id={apiserver_subnet_id} " "--outbound-type loadBalancer " "--ssh-key-value={ssh_key_value}" @@ -6307,8 +6307,8 @@ def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group "aks create --resource-group={resource_group} --name={name} --location={location} " "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--enable-managed-identity --assign-identity {identity_id} " - "--system-node-vnet-subnet-id={system_node_subnet_id} " - "--node-vnet-subnet-id={node_subnet_id} " + "--system-node-subnet-id={system_node_subnet_id} " + "--node-subnet-id={node_subnet_id} " "--apiserver-subnet-id={apiserver_subnet_id} " "--outbound-type userAssignedNATGateway " "--ssh-key-value={ssh_key_value}" From 5bd621f4de05a370562e9eeaae93672936f1ce81 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Thu, 23 Apr 2026 00:07:29 +0000 Subject: [PATCH 10/14] test: drop BYO HOBO managedNATGateway live test RP rejects the combination of BYO VNet + managedNATGateway with "Outbound type is managedNATGateway but agent pool 'hostedpool' is using custom VNet, which is not allowed" (by design, enforced in natgatewayv2.go). For BYO VNet the RP auto-defaults outboundType to loadBalancer and only accepts loadBalancer or userAssignedNATGateway. The byovnet_slb and byovnet_user_natgw tests already cover the two supported BYO HOBO outbound modes, so this test was attempting an unsupported scenario and is removed. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../tests/latest/test_aks_commands.py | 82 ------------------- 1 file changed, 82 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 55808463161..9ac85d14da1 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6016,88 +6016,6 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso checks=[self.is_empty()], ) - @live_only() - @AllowLargeResponse() - @AKSCustomResourceGroupPreparer( - random_name_length=17, name_prefix="clitest", location="eastus2euap" - ) - def test_aks_automatic_sku_hosted_system_byovnet_natgw(self, resource_group, resource_group_location): - # reset the count so in replay mode the random names will start with 0 - self.test_resources_count = 0 - aks_name = self.create_random_name("cliakstest", 16) - vnet_name = self.create_random_name("clivnet", 16) - identity_name = self.create_random_name("cliakstest", 16) - self.kwargs.update( - { - "resource_group": resource_group, - "name": aks_name, - "location": resource_group_location, - "vnet_name": vnet_name, - "identity_name": identity_name, - "ssh_key_value": self.generate_ssh_keys(), - } - ) - - # user-assigned MSI is required for BYO VNet on automatic SKU - identity_id = self.cmd( - "identity create -g {resource_group} -n {identity_name} --query id -o tsv" - ).output.strip() - self.kwargs.update({"identity_id": identity_id}) - - # create a BYO VNet with 3 subnets: system-node, node, apiserver - self.cmd( - "network vnet create -g {resource_group} -n {vnet_name} " - "--address-prefix 10.0.0.0/8 " - "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" - ) - self.cmd( - "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " - "--name node --address-prefix 10.43.0.0/16" - ) - self.cmd( - "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " - "--name apiserver --address-prefix 10.44.0.0/28" - ) - system_node_subnet_id = self.cmd( - "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " - "--query id -o tsv" - ).output.strip() - node_subnet_id = self.cmd( - "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " - "--query id -o tsv" - ).output.strip() - apiserver_subnet_id = self.cmd( - "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " - "--query id -o tsv" - ).output.strip() - self.kwargs.update({ - "system_node_subnet_id": system_node_subnet_id, - "node_subnet_id": node_subnet_id, - "apiserver_subnet_id": apiserver_subnet_id, - }) - - # BYO VNet HOBO automatic cluster (default outbound = NAT gateway) - create_cmd = ( - "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " - "--enable-managed-identity --assign-identity {identity_id} " - "--system-node-subnet-id={system_node_subnet_id} " - "--node-subnet-id={node_subnet_id} " - "--apiserver-subnet-id={apiserver_subnet_id} " - "--ssh-key-value={ssh_key_value}" - ) - self.cmd( - create_cmd, - checks=[ - self.check("provisioningState", "Succeeded"), - self.check("sku.name", "Automatic"), - self.check("hostedSystemProfile.enabled", True), - self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), - self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), - self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), - ], - ) - @live_only() @AllowLargeResponse() @AKSCustomResourceGroupPreparer( From 4bc6d630358ad19ac45c382656bef07fbb4b54c3 Mon Sep 17 00:00:00 2001 From: Wen Huang Date: Mon, 1 Jun 2026 22:50:51 +0000 Subject: [PATCH 11/14] aks-preview: Align hosted-system BYO VNet logic with CLI --- src/aks-preview/HISTORY.rst | 2 +- src/aks-preview/azext_aks_preview/_help.py | 26 +-- src/aks-preview/azext_aks_preview/_params.py | 1 - src/aks-preview/azext_aks_preview/custom.py | 1 - .../managed_cluster_decorator.py | 166 ++++++++------- .../tests/latest/test_aks_commands.py | 39 ---- .../latest/test_managed_cluster_decorator.py | 197 ++++++++++++++++++ 7 files changed, 300 insertions(+), 132 deletions(-) diff --git a/src/aks-preview/HISTORY.rst b/src/aks-preview/HISTORY.rst index cc45766c096..e5975b244f1 100644 --- a/src/aks-preview/HISTORY.rst +++ b/src/aks-preview/HISTORY.rst @@ -11,7 +11,7 @@ To release a new version, please select a new version number (usually plus 1 to Pending +++++++ -* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-subnet-id` and `--node-subnet-id`; add `--disable-hosted-system` opt-out. +* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-subnet-id` and `--node-subnet-id`. 21.0.0b9 ++++++++ diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index be0577308d9..13e1b7c3c60 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -743,25 +743,19 @@ short-summary: Create a cluster with fully hosted system components. This applies only when creating a new automatic cluster. long-summary: | Deterministically opts the cluster into HOBO (Hosted Overlay System Pool). AKS hosts and manages the system node pool. - Cannot be used with `--disable-hosted-system`. - - name: --disable-hosted-system - type: bool - short-summary: Opt the automatic cluster out of hosted system components. - long-summary: | - Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. - Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. + This is also implied when you provide the BYO VNet subnet trio. - name: --sys-node-subnet-id --system-node-subnet-id type: string short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). long-summary: | - Only valid with `--enable-hosted-system`. Must be provided together with `--node-subnet-id` - and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. + Must be provided together with `--node-subnet-id` and `--apiserver-subnet-id`, + and all three subnets must belong to the same VNet. Only valid with `--sku automatic`. - name: --node-subnet-id type: string short-summary: Resource ID of the subnet joined by tenant worker nodes in BYO VNet HOBO clusters. long-summary: | - Only valid with `--enable-hosted-system`. Must be provided together with `--system-node-subnet-id` - and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. + Must be provided together with `--system-node-subnet-id` and `--apiserver-subnet-id`, + and all three subnets must belong to the same VNet. Only valid with `--sku automatic`. - name: --control-plane-scaling-size --cp-scaling-size type: string short-summary: (PREVIEW) The control plane scaling size for the cluster. @@ -864,14 +858,12 @@ text: az aks create -g MyResourceGroup -n MyManagedCluster --control-plane-scaling-size H4 - name: Create an automatic cluster with hosted system components enabled. text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system - - name: Create a kubernetes cluster with Azure Backup enabled (default Week strategy). Requires the 'dataprotection' extension. Implicitly waits for cluster creation. - text: az aks create -g MyResourceGroup -n MyManagedCluster --generate-ssh-keys --enable-backup --yes - - name: Create a hosted-system automatic cluster in a BYO VNet (NAT gateway outbound, the default). - text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-subnet-id --node-subnet-id --apiserver-subnet-id + - name: Create a hosted-system automatic cluster in a BYO VNet. + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --system-node-subnet-id --node-subnet-id --apiserver-subnet-id - name: Create a hosted-system automatic cluster in a BYO VNet with Load Balancer outbound. text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-subnet-id --node-subnet-id --apiserver-subnet-id --outbound-type loadBalancer - - name: Create an automatic cluster and opt out of hosted system components. - text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --disable-hosted-system + - name: Create a kubernetes cluster with Azure Backup enabled (default Week strategy). Requires the 'dataprotection' extension. Implicitly waits for cluster creation. + text: az aks create -g MyResourceGroup -n MyManagedCluster --generate-ssh-keys --enable-backup --yes """ diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index 568d7b740b5..a77d62e9a2c 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -1310,7 +1310,6 @@ def load_arguments(self, _): help="Enable Gateway API based ingress on App Routing via Istio" ) c.argument("enable_hosted_system", action="store_true", is_preview=True) - c.argument("disable_hosted_system", action="store_true", is_preview=True) c.argument( "system_node_subnet_id", options_list=["--system-node-subnet-id", "--sys-node-subnet-id"], diff --git a/src/aks-preview/azext_aks_preview/custom.py b/src/aks-preview/azext_aks_preview/custom.py index fab5569d6c1..ffffd8cbbc2 100644 --- a/src/aks-preview/azext_aks_preview/custom.py +++ b/src/aks-preview/azext_aks_preview/custom.py @@ -1437,7 +1437,6 @@ def aks_create( # app routing istio enable_app_routing_istio=False, enable_hosted_system=False, - disable_hosted_system=False, system_node_subnet_id=None, node_subnet_id=None, control_plane_scaling_size=None, diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 6cd1e0733f0..3fb8cdc4ef7 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -548,6 +548,31 @@ def _get_disable_local_accounts(self, enable_validation: bool = False) -> bool: ) return disable_local_accounts + @staticmethod + def _raise_missing_vnet_subnet_for_outbound_type(outbound_type: str, sku_name: str) -> None: + if outbound_type == CONST_OUTBOUND_TYPE_USER_DEFINED_ROUTING: + subnet_requirement = "a route table with egress rules" + else: + subnet_requirement = "a NAT gateway with outbound ips" + + if sku_name == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: + raise RequiredArgumentMissingError( + "For an Automatic cluster using Managed System Pool BYO VNet, --system-node-subnet-id, " + "--node-subnet-id and --apiserver-subnet-id must be specified for {outbound_type}. " + "For other BYO VNet clusters, specify --vnet-subnet-id. The subnet must be " + "pre-configured with {requirement}".format( + outbound_type=outbound_type, + requirement=subnet_requirement, + ) + ) + raise RequiredArgumentMissingError( + "--vnet-subnet-id must be specified for {outbound_type} and it must " + "be pre-configured with {requirement}".format( + outbound_type=outbound_type, + requirement=subnet_requirement, + ) + ) + # pylint: disable=too-many-branches def _get_outbound_type( self, @@ -606,9 +631,6 @@ def _get_outbound_type( CONST_OUTBOUND_TYPE_NONE, CONST_OUTBOUND_TYPE_BLOCK,] ): - # Preserve the user's explicit --outbound-type (if any) before default-completion - # overwrites it with CONST_OUTBOUND_TYPE_LOAD_BALANCER below. - user_supplied_outbound_type = self.raw_param.get("outbound_type") outbound_type = CONST_OUTBOUND_TYPE_LOAD_BALANCER sku_name = self.get_sku_name() is_vnet_subnet_id_empty = self.get_vnet_subnet_id() in ["", None] @@ -620,10 +642,9 @@ def _get_outbound_type( is_vnet_subnet_id_empty ): # Default outbound for Automatic SKU without a VNet is managedNATGateway. - # For BYO HOBO, only apply this default when the user did NOT explicitly pass - # --outbound-type; otherwise we would silently overwrite a user's explicit - # --outbound-type loadBalancer (regression seen in BYO SLB scenario). - if not (hobo_byo_subnets and user_supplied_outbound_type): + # For BYO HOBO, keep the loadBalancer default because managedNATGateway + # is not supported with custom VNet subnets. + if not hobo_byo_subnets: outbound_type = CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY # validation @@ -649,20 +670,27 @@ def _get_outbound_type( # --system-node-subnet-id / --node-subnet-id # instead of --vnet-subnet-id. if ( + not read_from_mc and self.get_vnet_subnet_id() in ["", None] and - not self.has_byo_hobo_subnets() + not self.has_byo_hobo_subnets_configured() ): - if self.decorator_mode == DecoratorMode.CREATE: - raise RequiredArgumentMissingError( - "--vnet-subnet-id must be specified for userDefinedRouting and it must " - "be pre-configured with a route table with egress rules" + if self.decorator_mode == DecoratorMode.UPDATE: + raise InvalidArgumentValueError( + f"Updating outbound type to {outbound_type} is only supported for " + "clusters using a custom (BYO) virtual network. Managed VNet clusters " + f"cannot be updated to {outbound_type}. Please refer to " + "https://learn.microsoft.com/en-us/azure/aks/egress-outboundtype" + "#updating-outboundtype-after-cluster-creation for supported migration paths." ) + self._raise_missing_vnet_subnet_for_outbound_type( + outbound_type, + self.get_sku_name(), + ) + if outbound_type == CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY: + if self.get_vnet_subnet_id() not in ["", None] or self.has_byo_hobo_subnets(): raise InvalidArgumentValueError( - f"Updating outbound type to {outbound_type} is only supported for " - "clusters using a custom (BYO) virtual network. Managed VNet clusters " - f"cannot be updated to {outbound_type}. Please refer to " - "https://learn.microsoft.com/en-us/azure/aks/egress-outboundtype" - "#updating-outboundtype-after-cluster-creation for supported migration paths." + "--vnet-subnet-id, --system-node-subnet-id and --node-subnet-id " + "cannot be specified for managedNATGateway" ) if ( @@ -2011,9 +2039,8 @@ def _get_apiserver_subnet_id(self, enable_validation: bool = False) -> Union[str # validation if enable_validation: if self.decorator_mode == DecoratorMode.CREATE: - # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so misuse - # (e.g. passing --system-node-subnet-id without --enable-hosted-system) - # is rejected up front rather than silently dropped. + # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so partial + # subnet trios are rejected up front rather than silently dropped. self.validate_byo_hobo_subnet_trio() vnet_subnet_id = self.get_vnet_subnet_id() # For BYO VNet HOBO automatic clusters, --system-node-subnet-id and @@ -4154,37 +4181,25 @@ def get_enable_hosted_system(self) -> bool: :return: bool """ - enable_hosted_system = self.raw_param.get("enable_hosted_system") - disable_hosted_system = self.raw_param.get("disable_hosted_system") - if enable_hosted_system and disable_hosted_system: - raise MutuallyExclusiveArgumentError( - 'Cannot specify "--enable-hosted-system" and "--disable-hosted-system" at the same time.' - ) - if enable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: + if self.decorator_mode != DecoratorMode.CREATE: + return False + explicit = bool(self.raw_param.get("enable_hosted_system")) + implicit = all( + [ + self.raw_param.get("system_node_subnet_id"), + self.raw_param.get("node_subnet_id"), + self.raw_param.get("apiserver_subnet_id"), + ] + ) + if (explicit or implicit) and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: raise RequiredArgumentMissingError('"--enable-hosted-system" requires "--sku automatic".') - return enable_hosted_system - - def get_disable_hosted_system(self) -> bool: - """Obtain the value of disable_hosted_system. - - :return: bool - """ - disable_hosted_system = self.raw_param.get("disable_hosted_system") - enable_hosted_system = self.raw_param.get("enable_hosted_system") - if disable_hosted_system and enable_hosted_system: - raise MutuallyExclusiveArgumentError( - 'Cannot specify "--enable-hosted-system" and "--disable-hosted-system" at the same time.' - ) - if disable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: - raise RequiredArgumentMissingError('"--disable-hosted-system" requires "--sku automatic".') - return disable_hosted_system + return explicit or implicit def get_system_node_subnet_id(self) -> Union[str, None]: """Obtain the value of system_node_subnet_id. - Validates that BYO VNet subnet flags are used together with - --enable-hosted-system and that the full triple - (system-node / node / apiserver) is provided. + Validates that BYO VNet subnet flags are used as a full triple + (system-node / node / apiserver). :return: str or None """ @@ -4213,32 +4228,38 @@ def has_byo_hobo_subnets(self) -> bool: self.raw_param.get("node_subnet_id") ) + def has_existing_byo_hobo_subnets(self) -> bool: + """Return True when an update target already has BYO HOBO node subnets.""" + hosted_system_profile = getattr(self.mc, "hosted_system_profile", None) if self.mc else None + return bool( + self.decorator_mode == DecoratorMode.UPDATE and + hosted_system_profile and + ( + getattr(hosted_system_profile, "system_node_subnet_id", None) or + getattr(hosted_system_profile, "node_subnet_id", None) + ) + ) + + def has_byo_hobo_subnets_configured(self) -> bool: + """Return True for BYO HOBO subnets set in this request or already on the cluster.""" + return self.has_byo_hobo_subnets() or self.has_existing_byo_hobo_subnets() + def validate_byo_hobo_subnet_trio(self) -> None: """Cross-validate the BYO VNet HOBO subnet flags. - Rule: if any of --system-node-subnet-id, --node-subnet-id, or - --apiserver-subnet-id is set together with --enable-hosted-system, then - all three must be set (SDK requires all three subnets to share a VNet). - Any of these flags set without --enable-hosted-system is flagged so - the user doesn't silently get a non-HOBO cluster with the wrong shape. + Rule: if either --system-node-subnet-id or --node-subnet-id is set, the + full BYO trio must be set and --sku must be automatic. A complete trio + implies hosted-system enablement. """ system_node_subnet_id = self.raw_param.get("system_node_subnet_id") node_subnet_id = self.raw_param.get("node_subnet_id") apiserver_subnet_id = self.raw_param.get("apiserver_subnet_id") - enable_hosted_system = self.raw_param.get("enable_hosted_system") + enable_hosted_system = bool(self.raw_param.get("enable_hosted_system")) - # The HOBO-specific flags only make sense with --enable-hosted-system. - # --apiserver-subnet-id is a general VNet-integration flag, so only - # require HOBO on it when one of the other two is also set. - if self.has_byo_hobo_subnets() and not enable_hosted_system: - raise RequiredArgumentMissingError( - '"--system-node-subnet-id" and "--node-subnet-id" ' - 'require "--enable-hosted-system".' - ) + if enable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: + raise RequiredArgumentMissingError('"--enable-hosted-system" requires "--sku automatic".') - if enable_hosted_system and ( - system_node_subnet_id or node_subnet_id or apiserver_subnet_id - ): + if self.has_byo_hobo_subnets(): missing = [] if not system_node_subnet_id: missing.append("--system-node-subnet-id") @@ -4253,6 +4274,10 @@ def validate_byo_hobo_subnet_trio(self) -> None: "--apiserver-subnet-id to be provided together. " f"Missing: {', '.join(missing)}." ) + if self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: + raise RequiredArgumentMissingError( + '"--system-node-subnet-id" and "--node-subnet-id" require "--sku automatic".' + ) def get_control_plane_scaling_size(self) -> Union[str, None]: """Obtain the value of control_plane_scaling_size. @@ -4514,6 +4539,8 @@ def set_up_api_server_access_profile(self, mc: ManagedCluster) -> ManagedCluster # pylint: disable=no-member api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() api_server_access_profile.subnet_id = self.context.get_apiserver_subnet_id() + if self.context.has_byo_hobo_subnets(): + api_server_access_profile.enable_vnet_integration = True mc.api_server_access_profile = api_server_access_profile mc.fqdn_subdomain = self.context.get_fqdn_subdomain() @@ -5523,8 +5550,8 @@ def set_up_upstream_kubescheduler_user_configuration(self, mc: ManagedCluster) - def set_up_enable_hosted_components(self, mc: ManagedCluster) -> ManagedCluster: self._ensure_mc(mc) + self.context.validate_byo_hobo_subnet_trio() enable_hosted_components = self.context.get_enable_hosted_system() - disable_hosted_components = self.context.get_disable_hosted_system() if enable_hosted_components: if mc.hosted_system_profile is None: mc.hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() # pylint: disable=no-member @@ -5543,12 +5570,6 @@ def set_up_enable_hosted_components(self, mc: ManagedCluster) -> ManagedCluster: # Remove default agent pool profiles when hosted system profile is enabled if mc.agent_pool_profiles is not None: mc.agent_pool_profiles = None - elif disable_hosted_components: - # Deterministic opt-out: explicitly set enabled=False so customers - # don't get HOBO once the server flips the default in their region. - if mc.hosted_system_profile is None: - mc.hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() # pylint: disable=no-member - mc.hosted_system_profile.enabled = False return mc @@ -5567,9 +5588,8 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non for each. Skipping is honored via ``--skip-subnet-role-assignment``. """ # Fail-fast validation BEFORE any role assignment runs, so a malformed BYO HOBO - # create (e.g. partial subnet trio, --system-node-subnet-id without - # --enable-hosted-system) cannot leave residual Network Contributor grants on - # customer subnets. Trio validation is otherwise invoked later through + # create (e.g. partial subnet trio) cannot leave residual Network Contributor + # grants on customer subnets. Trio validation is otherwise invoked later through # set_up_api_server_access_profile, which executes AFTER this method in the base # construct_mc_profile_default flow. self.context.validate_byo_hobo_subnet_trio() diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 9ac85d14da1..1a0656936d9 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6244,45 +6244,6 @@ def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group ], ) - @live_only() - @AllowLargeResponse() - @AKSCustomResourceGroupPreparer( - random_name_length=17, name_prefix="clitest", location="eastus2euap" - ) - def test_aks_automatic_sku_with_hosted_system_disabled(self, resource_group, resource_group_location): - # reset the count so in replay mode the random names will start with 0 - self.test_resources_count = 0 - aks_name = self.create_random_name("cliakstest", 16) - self.kwargs.update( - { - "resource_group": resource_group, - "name": aks_name, - "location": resource_group_location, - "ssh_key_value": self.generate_ssh_keys(), - } - ) - - # opt out of HOBO even if the region defaults to it - create_cmd = ( - "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --disable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " - "--ssh-key-value={ssh_key_value}" - ) - self.cmd( - create_cmd, - checks=[ - self.check("provisioningState", "Succeeded"), - self.check("sku.name", "Automatic"), - self.check("hostedSystemProfile.enabled", False), - ], - ) - - # agent pool profiles should be present (non-HOBO automatic) - self.cmd( - "aks show --resource-group={resource_group} --name={name}", - checks=[self.greater_than("length(agentPoolProfiles)", 0)], - ) - @AllowLargeResponse() @AKSCustomResourceGroupPreparer( random_name_length=17, name_prefix="clitest", location="westus2" diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py index 2f62cd453d9..0173ff0a2b7 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py @@ -4894,6 +4894,148 @@ def test_get_outbound_type(self): CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY_V2, ) + system_node_subnet_id = "/subscriptions/testid/resourceGroups/MockedResourceGroup/providers/Microsoft.Network/virtualNetworks/MockedNetworkId/subnets/SystemNode" + node_subnet_id = "/subscriptions/testid/resourceGroups/MockedResourceGroup/providers/Microsoft.Network/virtualNetworks/MockedNetworkId/subnets/Node" + apiserver_subnet_id = "/subscriptions/testid/resourceGroups/MockedResourceGroup/providers/Microsoft.Network/virtualNetworks/MockedNetworkId/subnets/APIServer" + ctx6 = AKSPreviewManagedClusterContext( + self.cmd, + AKSManagedClusterParamDict({ + "sku": "automatic", + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }), + self.models, + decorator_mode=DecoratorMode.CREATE, + ) + self.create_attach_agentpool_context(ctx6) + outbound_type_6 = ctx6._get_outbound_type(False, False, None) + self.assertEqual(outbound_type_6, CONST_OUTBOUND_TYPE_LOAD_BALANCER) + + ctx7 = AKSPreviewManagedClusterContext( + self.cmd, + AKSManagedClusterParamDict({ + "sku": "automatic", + "outbound_type": "managedNATGateway", + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }), + self.models, + decorator_mode=DecoratorMode.CREATE, + ) + self.create_attach_agentpool_context(ctx7) + with self.assertRaises(InvalidArgumentValueError): + ctx7._get_outbound_type(True, False, None) + + ctx8 = AKSPreviewManagedClusterContext( + self.cmd, + AKSManagedClusterParamDict({ + "sku": "automatic", + "outbound_type": "userAssignedNATGateway", + }), + self.models, + decorator_mode=DecoratorMode.CREATE, + ) + self.create_attach_agentpool_context(ctx8) + with self.assertRaisesRegex(RequiredArgumentMissingError, "NAT gateway with outbound ips"): + ctx8._get_outbound_type(True, False, None) + + hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() + hosted_system_profile.system_node_subnet_id = system_node_subnet_id + hosted_system_profile.node_subnet_id = node_subnet_id + mc_existing_byo = self.models.ManagedCluster( + location="test_location", + network_profile=self.models.ContainerServiceNetworkProfile( + outbound_type=CONST_OUTBOUND_TYPE_LOAD_BALANCER + ), + sku=self.models.ManagedClusterSKU(name="Automatic"), + hosted_system_profile=hosted_system_profile, + ) + for outbound_type in ["userDefinedRouting", "userAssignedNATGateway"]: + ctx_existing_byo = AKSPreviewManagedClusterContext( + self.cmd, + AKSManagedClusterParamDict({"outbound_type": outbound_type}), + self.models, + decorator_mode=DecoratorMode.UPDATE, + ) + self.create_attach_agentpool_context(ctx_existing_byo) + ctx_existing_byo.attach_mc(mc_existing_byo) + self.assertEqual( + ctx_existing_byo._get_outbound_type(True, False, None), + outbound_type, + ) + + def test_get_outbound_type_update_preserves_existing_value(self): + network_profile = self.models.ContainerServiceNetworkProfile( + outbound_type=CONST_OUTBOUND_TYPE_USER_ASSIGNED_NAT_GATEWAY, + ) + mc = self.models.ManagedCluster( + location="test_location", + network_profile=network_profile, + sku=self.models.ManagedClusterSKU(name="Base"), + ) + ctx = AKSPreviewManagedClusterContext( + self.cmd, + AKSManagedClusterParamDict({"sku": "base"}), + self.models, + decorator_mode=DecoratorMode.UPDATE, + ) + ctx.agentpool_context = mock.MagicMock() + ctx.agentpool_context.get_vnet_subnet_id.return_value = None + ctx.attach_mc(mc) + + self.assertEqual( + ctx.get_outbound_type(), + CONST_OUTBOUND_TYPE_USER_ASSIGNED_NAT_GATEWAY, + ) + + def test_byo_hobo_subnet_trio_implies_enable_hosted_system(self): + system_node_subnet_id = "/subscriptions/testid/resourceGroups/MockedResourceGroup/providers/Microsoft.Network/virtualNetworks/MockedNetworkId/subnets/SystemNode" + node_subnet_id = "/subscriptions/testid/resourceGroups/MockedResourceGroup/providers/Microsoft.Network/virtualNetworks/MockedNetworkId/subnets/Node" + apiserver_subnet_id = "/subscriptions/testid/resourceGroups/MockedResourceGroup/providers/Microsoft.Network/virtualNetworks/MockedNetworkId/subnets/APIServer" + + ctx1 = AKSPreviewManagedClusterContext( + self.cmd, + AKSManagedClusterParamDict({ + "sku": "automatic", + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }), + self.models, + decorator_mode=DecoratorMode.CREATE, + ) + ctx1.validate_byo_hobo_subnet_trio() + self.assertTrue(ctx1.get_enable_hosted_system()) + + ctx2 = AKSPreviewManagedClusterContext( + self.cmd, + AKSManagedClusterParamDict({ + "sku": "automatic", + "system_node_subnet_id": system_node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }), + self.models, + decorator_mode=DecoratorMode.CREATE, + ) + with self.assertRaises(RequiredArgumentMissingError): + ctx2.validate_byo_hobo_subnet_trio() + + ctx3 = AKSPreviewManagedClusterContext( + self.cmd, + AKSManagedClusterParamDict({ + "sku": "base", + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }), + self.models, + decorator_mode=DecoratorMode.CREATE, + ) + with self.assertRaises(RequiredArgumentMissingError): + ctx3.validate_byo_hobo_subnet_trio() + def test_get_outbound_type_update_udr_byo_vnet(self): """Test that updating to UDR succeeds when the cluster has a BYO VNet (vnet_subnet_id is set on agentpool).""" ctx = AKSPreviewManagedClusterContext( @@ -4984,6 +5126,7 @@ def test_get_outbound_type_create_udr_no_subnet(self): with self.assertRaises(RequiredArgumentMissingError): ctx._get_outbound_type(enable_validation=True) + def test_get_enable_gateway_api(self): # default value ctx_1 = AKSPreviewManagedClusterContext( @@ -6329,6 +6472,60 @@ def test_set_up_api_server_access_profile(self): ) self.assertEqual(dec_mc_4, ground_truth_mc_4) + system_node_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/systemnode" + node_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/node" + dec_5 = AKSPreviewManagedClusterCreateDecorator( + self.cmd, + self.client, + { + "apiserver_subnet_id": apiserver_subnet_id, + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "sku": "automatic", + }, + CUSTOM_MGMT_AKS_PREVIEW, + ) + mc_5 = self.models.ManagedCluster(location="test_location") + dec_5.context.attach_mc(mc_5) + dec_mc_5 = dec_5.set_up_api_server_access_profile(mc_5) + ground_truth_api_server_access_profile_5 = ( + self.models.ManagedClusterAPIServerAccessProfile( + enable_vnet_integration=True, + subnet_id=apiserver_subnet_id, + ) + ) + ground_truth_mc_5 = self.models.ManagedCluster( + location="test_location", + api_server_access_profile=ground_truth_api_server_access_profile_5, + ) + self.assertEqual(dec_mc_5, ground_truth_mc_5) + + def test_set_up_enable_hosted_components_byo_trio_implies_enable(self): + system_node_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/systemnode" + node_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/node" + apiserver_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/apiserver" + dec_1 = AKSPreviewManagedClusterCreateDecorator( + self.cmd, + self.client, + { + "sku": "automatic", + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }, + CUSTOM_MGMT_AKS_PREVIEW, + ) + mc_1 = self.models.ManagedCluster( + location="test_location", + agent_pool_profiles=[self.models.ManagedClusterAgentPoolProfile(name="nodepool1")], + ) + dec_1.context.attach_mc(mc_1) + dec_mc_1 = dec_1.set_up_enable_hosted_components(mc_1) + self.assertTrue(dec_mc_1.hosted_system_profile.enabled) + self.assertEqual(dec_mc_1.hosted_system_profile.system_node_subnet_id, system_node_subnet_id) + self.assertEqual(dec_mc_1.hosted_system_profile.node_subnet_id, node_subnet_id) + self.assertIsNone(dec_mc_1.agent_pool_profiles) + def test_build_gitops_addon_profile(self): # default dec_1 = AKSPreviewManagedClusterCreateDecorator( From de49af2224b3055bbaa12c6a9f17c72f587b4242 Mon Sep 17 00:00:00 2001 From: Wen Huang Date: Tue, 2 Jun 2026 00:39:40 +0000 Subject: [PATCH 12/14] test: create Log Analytics workspace for hosted-system live tests --- .../tests/latest/test_aks_commands.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 1a0656936d9..c1e81ac6179 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -43,6 +43,22 @@ def __init__(self, method_name): method_name, recording_processors=[KeyReplacer()] ) + def _create_log_analytics_workspace(self, resource_group_location): + workspace_name = self.create_random_name("clilaw", 16) + workspace_location = ( + "eastus2" if resource_group_location.lower().endswith("euap") else resource_group_location + ) + self.kwargs.update( + { + "workspace_name": workspace_name, + "workspace_location": workspace_location, + } + ) + return self.cmd( + "monitor log-analytics workspace create -g {resource_group} -n {workspace_name} " + "--location {workspace_location} --query id -o tsv" + ).output.strip() + def _get_versions(self, location): """Return the previous and current Kubernetes minor release versions, such as ("1.11.6", "1.12.4").""" supported_versions = self.cmd( @@ -5978,13 +5994,16 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso "ssh_key_value": self.generate_ssh_keys(), } ) + self.kwargs.update({ + "workspace_resource_id": self._create_log_analytics_workspace(resource_group_location), + }) # create an Automatic cluster with hosted system enabled # hobo pool is fully managed by AKS, not accessible to users, and can only be configured during creation, # so we only need to test for cluster creation here create_cmd = ( "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--sku automatic --enable-hosted-system --workspace-resource-id={workspace_resource_id} " "--aks-custom-header AKSHTTPCustomFeatures=Microsoft.ContainerService/AutomaticSKUPreview," "AKSHTTPCustomFeatures=Microsoft.ContainerService/AKS-AutomaticHostedSystemProfilePreview " "--ssh-key-value={ssh_key_value}" @@ -6074,12 +6093,13 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou "system_node_subnet_id": system_node_subnet_id, "node_subnet_id": node_subnet_id, "apiserver_subnet_id": apiserver_subnet_id, + "workspace_resource_id": self._create_log_analytics_workspace(resource_group_location), }) # BYO VNet HOBO automatic cluster with loadBalancer outbound create_cmd = ( "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--sku automatic --enable-hosted-system --workspace-resource-id={workspace_resource_id} " "--enable-managed-identity --assign-identity {identity_id} " "--system-node-subnet-id={system_node_subnet_id} " "--node-subnet-id={node_subnet_id} " @@ -6218,12 +6238,13 @@ def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group "system_node_subnet_id": system_node_subnet_id, "node_subnet_id": node_subnet_id, "apiserver_subnet_id": apiserver_subnet_id, + "workspace_resource_id": self._create_log_analytics_workspace(resource_group_location), }) # BYO VNet HOBO automatic cluster with userAssignedNATGateway outbound create_cmd = ( "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--sku automatic --enable-hosted-system --workspace-resource-id={workspace_resource_id} " "--enable-managed-identity --assign-identity {identity_id} " "--system-node-subnet-id={system_node_subnet_id} " "--node-subnet-id={node_subnet_id} " From 48b267d8f49384534bd439dafa75009fb049ad45 Mon Sep 17 00:00:00 2001 From: Wen Huang Date: Wed, 15 Jul 2026 00:09:53 +0000 Subject: [PATCH 13/14] aks-preview: Fix managed system pool live create --- .../managed_cluster_decorator.py | 9 +++ .../prepared_image_specification.py | 2 +- .../tests/latest/test_aks_commands.py | 39 ++----------- .../latest/test_managed_cluster_decorator.py | 57 +++++++++++++++++++ .../test_prepared_image_specification.py | 16 ++++++ 5 files changed, 89 insertions(+), 34 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 3fb8cdc4ef7..0447bbbc349 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -4367,6 +4367,15 @@ def set_up_agentpool_profile(self, mc: ManagedCluster) -> ManagedCluster: mc.agent_pool_profiles = [agentpool_profile] return mc + def set_up_linux_profile(self, mc: ManagedCluster) -> ManagedCluster: + """Skip VM SSH configuration when the system pool is fully managed by AKS.""" + self._ensure_mc(mc) + + if self.context.get_enable_hosted_system(): + return mc + + return super().set_up_linux_profile(mc) + def set_up_network_profile(self, mc: ManagedCluster) -> ManagedCluster: """Set up network profile for the ManagedCluster object. diff --git a/src/aks-preview/azext_aks_preview/prepared_image_specification.py b/src/aks-preview/azext_aks_preview/prepared_image_specification.py index 23910d961e0..f639c5d1f56 100644 --- a/src/aks-preview/azext_aks_preview/prepared_image_specification.py +++ b/src/aks-preview/azext_aks_preview/prepared_image_specification.py @@ -37,7 +37,7 @@ def pis_identity_resource_id(cmd, agentpool): def ensure_pis_managed_identity_permission_for_cluster(cmd, cluster): - for agentpool in cluster.agent_pool_profiles: + for agentpool in (cluster.agent_pool_profiles or []): pis_identity_id = pis_identity_resource_id(cmd, agentpool) if not pis_identity_id: continue diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index c1e81ac6179..ab701e80d9a 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6038,7 +6038,7 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso @live_only() @AllowLargeResponse() @AKSCustomResourceGroupPreparer( - random_name_length=17, name_prefix="clitest", location="eastus2euap" + random_name_length=17, name_prefix="clitest", location="westus3" ) def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 @@ -6113,6 +6113,8 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou self.check("provisioningState", "Succeeded"), self.check("sku.name", "Automatic"), self.check("hostedSystemProfile.enabled", True), + self.check("agentPoolProfiles", None), + self.check("linuxProfile", None), self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), @@ -6120,41 +6122,10 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou ], ) - # convert to Base SKU; expect the transition to succeed for the BYO VNet HOBO case. - # Wait for the cluster to become idle (no in-progress RP reconciliation) before starting - # the update; without this the RP returns 409 OperationNotAllowed for a few minutes after - # create completes while it finishes post-create reconciliation. - self.cmd( - "aks wait -g {resource_group} -n {name} --created --timeout 900", - checks=[self.is_empty()], - ) - # Poll with retries: `aks wait --created` returns once provisioningState hits Succeeded, - # but the RP can still have an internal in-progress reconciliation for several minutes - # afterwards which surfaces as 409 OperationNotAllowed on `aks update`. Back off and retry. - update_deadline = time.time() + 900 - last_error = None - while time.time() < update_deadline: - try: - self.cmd( - "aks update -g {resource_group} -n {name} --sku base", - checks=[self.check("sku.name", "Base")], - ) - last_error = None - break - except Exception as ex: # pylint: disable=broad-except - message = str(ex) - if "OperationNotAllowed" in message or "in progress" in message or "409" in message: - last_error = ex - time.sleep(60) - continue - raise - if last_error is not None: - raise last_error - @live_only() @AllowLargeResponse() @AKSCustomResourceGroupPreparer( - random_name_length=17, name_prefix="clitest", location="eastus2euap" + random_name_length=17, name_prefix="clitest", location="westus3" ) def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 @@ -6258,6 +6229,8 @@ def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group self.check("provisioningState", "Succeeded"), self.check("sku.name", "Automatic"), self.check("hostedSystemProfile.enabled", True), + self.check("agentPoolProfiles", None), + self.check("linuxProfile", None), self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py index 0173ff0a2b7..065e6824054 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py @@ -6309,6 +6309,63 @@ def test_set_up_agentpool_profile(self): ground_truth_mc_1.agent_pool_profiles = [ground_truth_agentpool_profile_1] self.assertEqual(dec_mc_1, ground_truth_mc_1) + def test_set_up_agentpool_profile_hosted_system_skips_default_pool(self): + dec_1 = AKSPreviewManagedClusterCreateDecorator( + self.cmd, + self.client, + { + "enable_hosted_system": True, + "sku": "automatic", + }, + CUSTOM_MGMT_AKS_PREVIEW, + ) + mc_1 = self.models.ManagedCluster(location="test_location") + dec_1.context.attach_mc(mc_1) + + with patch.object( + dec_1.agentpool_decorator, + "construct_agentpool_profile_preview", + ) as construct_agentpool_profile: + dec_mc_1 = dec_1.set_up_agentpool_profile(mc_1) + + self.assertIs(dec_mc_1, mc_1) + self.assertIsNone(dec_mc_1.agent_pool_profiles) + construct_agentpool_profile.assert_not_called() + + def test_set_up_agentpool_profile_ssh_access_allows_no_agent_pools(self): + dec_1 = AKSPreviewManagedClusterCreateDecorator( + self.cmd, + self.client, + {"ssh_access": CONST_SSH_ACCESS_LOCALUSER}, + CUSTOM_MGMT_AKS_PREVIEW, + ) + mc_1 = self.models.ManagedCluster(location="test_location", agent_pool_profiles=None) + dec_1.context.attach_mc(mc_1) + + dec_mc_1 = dec_1.set_up_agentpool_profile_ssh_access(mc_1) + + self.assertIs(dec_mc_1, mc_1) + self.assertIsNone(dec_mc_1.agent_pool_profiles) + + def test_set_up_linux_profile_hosted_system_skips_ssh_key(self): + dec_1 = AKSPreviewManagedClusterCreateDecorator( + self.cmd, + self.client, + { + "enable_hosted_system": True, + "sku": "automatic", + "ssh_key_value": "unused-for-managed-system-pool", + }, + CUSTOM_MGMT_AKS_PREVIEW, + ) + mc_1 = self.models.ManagedCluster(location="test_location") + dec_1.context.attach_mc(mc_1) + + dec_mc_1 = dec_1.set_up_linux_profile(mc_1) + + self.assertIs(dec_mc_1, mc_1) + self.assertIsNone(dec_mc_1.linux_profile) + def test_set_up_network_profile_preview(self): # custom value dec_1 = AKSPreviewManagedClusterCreateDecorator( diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_prepared_image_specification.py b/src/aks-preview/azext_aks_preview/tests/latest/test_prepared_image_specification.py index 7b9fa8f63d3..2d2c8573f33 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_prepared_image_specification.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_prepared_image_specification.py @@ -4,15 +4,31 @@ # -------------------------------------------------------------------------------------------- import unittest +from unittest.mock import Mock, patch from azure.cli.testsdk import ScenarioTest, live_only +from azext_aks_preview.prepared_image_specification import ( + ensure_pis_managed_identity_permission_for_cluster, +) from azext_aks_preview.tests.latest.recording_processors import KeyReplacer from azext_aks_preview.tests.latest.custom_preparers import ( AKSCustomResourceGroupPreparer, ) +class PreparedImageSpecificationUnitTestCases(unittest.TestCase): + def test_ensure_cluster_permission_allows_no_agent_pools(self): + cluster = Mock(agent_pool_profiles=None) + + with patch( + "azext_aks_preview.prepared_image_specification.pis_identity_resource_id" + ) as pis_identity_resource_id: + ensure_pis_managed_identity_permission_for_cluster(Mock(), cluster) + + pis_identity_resource_id.assert_not_called() + + class PreparedImageSpecificationTestCases(ScenarioTest): def __init__(self, method_name): super(PreparedImageSpecificationTestCases, self).__init__( From 35317000d808c4e9d8d6f4eb3a43b45038705ce6 Mon Sep 17 00:00:00 2001 From: Wen Huang Date: Wed, 15 Jul 2026 06:08:23 +0000 Subject: [PATCH 14/14] aks-preview: Preserve CLI alignment after rebase --- src/aks-preview/HISTORY.rst | 2 +- src/aks-preview/azext_aks_preview/_help.py | 26 ++-- src/aks-preview/azext_aks_preview/_params.py | 1 - src/aks-preview/azext_aks_preview/custom.py | 13 +- .../managed_cluster_decorator.py | 72 +++++----- .../tests/latest/test_custom.py | 16 +-- .../latest/test_managed_cluster_decorator.py | 123 +++++++++++++++++- 7 files changed, 189 insertions(+), 64 deletions(-) diff --git a/src/aks-preview/HISTORY.rst b/src/aks-preview/HISTORY.rst index e5975b244f1..5530f5d68b5 100644 --- a/src/aks-preview/HISTORY.rst +++ b/src/aks-preview/HISTORY.rst @@ -11,7 +11,7 @@ To release a new version, please select a new version number (usually plus 1 to Pending +++++++ -* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-subnet-id` and `--node-subnet-id`. +* `az aks create`: Add parameters `--system-node-subnet-id`, `--node-subnet-id` and `--enable-hosted-system` to support BYO VNet for Automatic Managed System Pool clusters. 21.0.0b9 ++++++++ diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index 13e1b7c3c60..45cd68ef61b 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -740,22 +740,28 @@ Cannot be used simultaneously with the Istio service mesh add-on (--enable-azure-service-mesh). - name: --enable-hosted-system type: bool - short-summary: Create a cluster with fully hosted system components. This applies only when creating a new automatic cluster. + short-summary: (Automatic SKU) Explicitly opt in to a Managed System Pool for the Automatic cluster. long-summary: | - Deterministically opts the cluster into HOBO (Hosted Overlay System Pool). AKS hosts and manages the system node pool. - This is also implied when you provide the BYO VNet subnet trio. - - name: --sys-node-subnet-id --system-node-subnet-id + Only valid with `--sku automatic`. Use this flag when you want to deterministically + request a Managed System Pool regardless of region defaults. It is also implied when + you supply the bring-your-own VNet subnet trio (`--system-node-subnet-id`, + `--node-subnet-id`, `--apiserver-subnet-id`). + - name: --system-node-subnet-id type: string - short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). + short-summary: (Automatic SKU) The ID of a subnet in an existing VNet to be used by the Managed System Pool in an Automatic cluster. long-summary: | - Must be provided together with `--node-subnet-id` and `--apiserver-subnet-id`, - and all three subnets must belong to the same VNet. Only valid with `--sku automatic`. + Bring-your-own VNet for an Automatic cluster requires three subnets supplied together: + `--system-node-subnet-id` (this flag, for the Managed System Pool), `--node-subnet-id` + (for user node pools), and `--apiserver-subnet-id` (for the control plane API server). + All three subnets must belong to the same VNet and can only be used with `--sku automatic`. - name: --node-subnet-id type: string - short-summary: Resource ID of the subnet joined by tenant worker nodes in BYO VNet HOBO clusters. + short-summary: (Automatic SKU) The ID of a subnet in an existing VNet to be used by user node pools in an Automatic cluster. long-summary: | - Must be provided together with `--system-node-subnet-id` and `--apiserver-subnet-id`, - and all three subnets must belong to the same VNet. Only valid with `--sku automatic`. + Bring-your-own VNet for an Automatic cluster requires three subnets supplied together: + `--system-node-subnet-id` (for the Managed System Pool), `--node-subnet-id` (this flag, + for user node pools), and `--apiserver-subnet-id` (for the control plane API server). + All three subnets must belong to the same VNet and can only be used with `--sku automatic`. - name: --control-plane-scaling-size --cp-scaling-size type: string short-summary: (PREVIEW) The control plane scaling size for the cluster. diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index a77d62e9a2c..5db313112ca 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -1312,7 +1312,6 @@ def load_arguments(self, _): c.argument("enable_hosted_system", action="store_true", is_preview=True) c.argument( "system_node_subnet_id", - options_list=["--system-node-subnet-id", "--sys-node-subnet-id"], validator=validate_system_node_subnet_id, is_preview=True, ) diff --git a/src/aks-preview/azext_aks_preview/custom.py b/src/aks-preview/azext_aks_preview/custom.py index ffffd8cbbc2..89db7e706d4 100644 --- a/src/aks-preview/azext_aks_preview/custom.py +++ b/src/aks-preview/azext_aks_preview/custom.py @@ -1921,14 +1921,21 @@ def aks_scale(cmd, # pylint: disable=unused-argument instance = client.get(resource_group_name, name) _fill_defaults_for_pod_identity_profile(instance.pod_identity_profile) - if len(instance.agent_pool_profiles) > 1 and nodepool_name == "": + agent_pool_profiles = instance.agent_pool_profiles or [] + if not agent_pool_profiles: + raise CLIError( + "The cluster has no scalable node pools (this may be a Managed System Pool for " + "an Automatic cluster). Use az aks nodepool add/scale against a user node pool instead." + ) + + if len(agent_pool_profiles) > 1 and nodepool_name == "": raise CLIError( "There are more than one node pool in the cluster. " "Please specify nodepool name or use az aks nodepool command to scale node pool" ) - for agent_profile in (instance.agent_pool_profiles or []): - if agent_profile.name == nodepool_name or (nodepool_name == "" and instance.agent_pool_profiles and len(instance.agent_pool_profiles) == 1): + for agent_profile in agent_pool_profiles: + if agent_profile.name == nodepool_name or (nodepool_name == "" and len(agent_pool_profiles) == 1): if agent_profile.enable_auto_scaling: raise CLIError( "Cannot scale cluster autoscaler enabled node pool.") diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 0447bbbc349..cbf374a13e2 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -573,8 +573,7 @@ def _raise_missing_vnet_subnet_for_outbound_type(outbound_type: str, sku_name: s ) ) - # pylint: disable=too-many-branches - def _get_outbound_type( + def _get_outbound_type( # pylint: disable=too-many-branches self, enable_validation: bool = False, read_only: bool = False, @@ -604,9 +603,9 @@ def _get_outbound_type( """ # read the original value passed by the command outbound_type = self.raw_param.get("outbound_type") - # In create mode, try to read the property value corresponding to the parameter from the `mc` object. + # Preserve the existing value when the user did not explicitly provide one. read_from_mc = False - if self.decorator_mode == DecoratorMode.CREATE: + if outbound_type is None: if ( self.mc and self.mc.network_profile and @@ -717,7 +716,7 @@ def _get_outbound_type( "userDefinedRouting doesn't support customizing \ a standard load balancer with IP addresses" ) - if self.decorator_mode == DecoratorMode.UPDATE: + if self.decorator_mode == DecoratorMode.UPDATE and not read_from_mc: if outbound_type in [ CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY, CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY_V2, @@ -4363,6 +4362,9 @@ def set_up_agentpool_profile(self, mc: ManagedCluster) -> ManagedCluster: """ self._ensure_mc(mc) + if self.context.get_enable_hosted_system(): + return mc + agentpool_profile = self.agentpool_decorator.construct_agentpool_profile_preview() mc.agent_pool_profiles = [agentpool_profile] return mc @@ -5448,7 +5450,7 @@ def set_up_agentpool_profile_ssh_access(self, mc: ManagedCluster) -> ManagedClus ssh_access = self.context.get_ssh_access() if ssh_access is not None: - for agent_pool_profile in mc.agent_pool_profiles: + for agent_pool_profile in (mc.agent_pool_profiles or []): if agent_pool_profile.security_profile is None: agent_pool_profile.security_profile = self.models.AgentPoolSecurityProfile() # pylint: disable=no-member agent_pool_profile.security_profile.ssh_access = ssh_access @@ -5576,10 +5578,6 @@ def set_up_enable_hosted_components(self, mc: ManagedCluster) -> ManagedCluster: if node_subnet_id: mc.hosted_system_profile.node_subnet_id = node_subnet_id - # Remove default agent pool profiles when hosted system profile is enabled - if mc.agent_pool_profiles is not None: - mc.agent_pool_profiles = None - return mc def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> None: @@ -5606,6 +5604,12 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non # Preserve base behavior for the --vnet-subnet-id case. super().process_add_role_assignment_for_vnet_subnet(mc) + # Azure CLI 2.86+ handles Managed System Pool BYO subnets in the base + # decorator. Keep the fallback below only for the extension's minimum + # supported CLI (2.85), avoiding duplicate role-assignment requests. + if callable(getattr(AKSManagedClusterCreateDecorator, "_get_byo_hosted_system_subnet_ids", None)): + return + # Only extend for BYO VNet HOBO; outside that mode --apiserver-subnet-id keeps its # generic apiserver-VNet-integration meaning and must NOT trigger an extra RBAC grant. if not self.context.get_enable_hosted_system(): @@ -5639,16 +5643,22 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non # flag AND stash the HOBO subnet list so the post-create handler can iterate it; # base behavior only grants on --vnet-subnet-id, which is absent for BYO HOBO. if service_principal_profile is None and not assign_identity: - self.context.set_intermediate( - "need_post_creation_vnet_permission_granting", - True, - overwrite_exists=True, - ) - self.context.set_intermediate( - "hobo_byo_subnets_pending_grant", - hobo_subnets, - overwrite_exists=True, - ) + pending_post_creation_subnets = [ + subnet_id + for subnet_id in hobo_subnets + if not self.context.external_functions.subnet_role_assignment_exists(self.cmd, subnet_id) + ] + if pending_post_creation_subnets: + self.context.set_intermediate( + "need_post_creation_vnet_permission_granting", + True, + overwrite_exists=True, + ) + self.context.set_intermediate( + "byo_hosted_system_subnets_pending_grant", + pending_post_creation_subnets, + overwrite_exists=True, + ) return for subnet_id in hobo_subnets: @@ -5757,8 +5767,8 @@ def construct_mc_profile_preview(self, bypass_restore_defaults: bool = False) -> mc = self.set_up_imds_restriction(mc) # set up user-defined scheduler configuration for kube-scheduler upstream mc = self.set_up_upstream_kubescheduler_user_configuration(mc) - # set up enable hosted components - # enabling hosted components will remove the default agent pool profiles from the mc object + # Set up hosted components. Managed System Pool creation already skipped + # synthesizing the default agent pool in set_up_agentpool_profile. mc = self.set_up_enable_hosted_components(mc) # validate the azure cli core version @@ -5858,10 +5868,10 @@ def immediate_processing_after_request(self, mc: ManagedCluster) -> None: vnet_subnet_id = self.context.get_vnet_subnet_id() if vnet_subnet_id: scopes.append(vnet_subnet_id) - hobo_subnets = self.context.get_intermediate( - "hobo_byo_subnets_pending_grant", default_value=[] + hosted_system_subnets = self.context.get_intermediate( + "byo_hosted_system_subnets_pending_grant", default_value=[] ) - for subnet in hobo_subnets or []: + for subnet in hosted_system_subnets or []: if subnet and subnet not in scopes: scopes.append(subnet) for scope in scopes: @@ -6310,14 +6320,12 @@ def update_agentpool_profile(self, mc: ManagedCluster) -> ManagedCluster: """ self._ensure_mc(mc) - # Preview-specific change: an AKS ManagedCluster of automatic - # cluster with hosted system components may not have agent pools - # When transitioning from hosted to non-hosted automatic clusters, - # customers must first add a system node pool before disabling - # the hosted system profile. + # Managed System Pool clusters manage the system pool server-side. Do not + # treat a user pool as the CLI-managed default pool during a generic update. + if mc.hosted_system_profile and mc.hosted_system_profile.enabled: + return mc + if not mc.agent_pool_profiles: - if mc.hosted_system_profile and mc.hosted_system_profile.enabled: - return mc raise UnknownError( "Encounter an unexpected error while getting agent pool profiles from the cluster in the process of " "updating agentpool profile." diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_custom.py b/src/aks-preview/azext_aks_preview/tests/latest/test_custom.py index 783eddf8be6..d1211a000da 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_custom.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_custom.py @@ -19,6 +19,7 @@ ) from azext_aks_preview.tests.latest.mocks import MockCLI, MockClient, MockCmd from azext_aks_preview.tests.latest.test_vm_skus import _make_sku, _make_restriction +from knack.util import CLIError class TestCustomCommand(unittest.TestCase): @@ -55,21 +56,14 @@ def test_aks_stop(self): self.assertEqual(aks_stop(self.cmd, self.client, "rg", "name", False), None) def test_aks_scale_with_none_agent_pool_profiles(self): - """Test aks_scale handles None agent_pool_profiles gracefully""" - # Test case: automatic cluster with hosted system components, no agent pools + """Managed System Pool clusters return a useful error instead of len(None).""" mc = self.models.ManagedCluster(location="test_location") - mc.agent_pool_profiles = None # This is the key scenario + mc.agent_pool_profiles = None mc.pod_identity_profile = None - self.client.get = Mock(return_value=mc) - # Should not raise NoneType error and should return without crashing - try: - result = aks_scale(self.cmd, self.client, "rg", "name", 3, "nodepool1") - # We expect this to complete without NoneType errors - except Exception as e: - # Should not be a NoneType error - self.assertNotIn("NoneType", str(type(e))) + with self.assertRaisesRegex(CLIError, "no scalable node pools"): + aks_scale(self.cmd, self.client, "rg", "name", 3, "nodepool1") def test_aks_upgrade_node_image_only_skips_machines_mode_pool(self): """Machines mode pools must be skipped during --node-image-only to avoid a known client-side error.""" diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py index 065e6824054..203e266488e 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_managed_cluster_decorator.py @@ -81,6 +81,7 @@ DecoratorMode, ) from azure.cli.command_modules.acs.managed_cluster_decorator import ( + AKSManagedClusterCreateDecorator, AKSManagedClusterParamDict, ) from azext_aks_preview.tests.latest.mocks import ( @@ -5126,7 +5127,6 @@ def test_get_outbound_type_create_udr_no_subnet(self): with self.assertRaises(RequiredArgumentMissingError): ctx._get_outbound_type(enable_validation=True) - def test_get_enable_gateway_api(self): # default value ctx_1 = AKSPreviewManagedClusterContext( @@ -6572,16 +6572,100 @@ def test_set_up_enable_hosted_components_byo_trio_implies_enable(self): }, CUSTOM_MGMT_AKS_PREVIEW, ) - mc_1 = self.models.ManagedCluster( - location="test_location", - agent_pool_profiles=[self.models.ManagedClusterAgentPoolProfile(name="nodepool1")], - ) + agent_pool_profiles = [self.models.ManagedClusterAgentPoolProfile(name="nodepool1")] + mc_1 = self.models.ManagedCluster(location="test_location", agent_pool_profiles=agent_pool_profiles) dec_1.context.attach_mc(mc_1) dec_mc_1 = dec_1.set_up_enable_hosted_components(mc_1) self.assertTrue(dec_mc_1.hosted_system_profile.enabled) self.assertEqual(dec_mc_1.hosted_system_profile.system_node_subnet_id, system_node_subnet_id) self.assertEqual(dec_mc_1.hosted_system_profile.node_subnet_id, node_subnet_id) - self.assertIsNone(dec_mc_1.agent_pool_profiles) + self.assertEqual(dec_mc_1.agent_pool_profiles, agent_pool_profiles) + + def test_process_add_role_assignment_for_byo_hobo_system_identity_cli_285_fallback(self): + system_node_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/systemnode" + node_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/node" + apiserver_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/apiserver" + dec_1 = AKSPreviewManagedClusterCreateDecorator( + self.cmd, + self.client, + { + "enable_managed_identity": True, + "sku": "automatic", + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }, + CUSTOM_MGMT_AKS_PREVIEW, + ) + mc_1 = self.models.ManagedCluster(location="test_location") + dec_1.context.attach_mc(mc_1) + + with patch.object( + AKSManagedClusterCreateDecorator, + "_get_byo_hosted_system_subnet_ids", + None, + create=True, + ), patch.object( + AKSManagedClusterCreateDecorator, + "process_add_role_assignment_for_vnet_subnet", + return_value=None, + ), patch.object( + dec_1.context.external_functions, + "subnet_role_assignment_exists", + return_value=False, + ): + dec_1.process_add_role_assignment_for_vnet_subnet(mc_1) + + self.assertTrue( + dec_1.context.get_intermediate("need_post_creation_vnet_permission_granting") + ) + self.assertEqual( + dec_1.context.get_intermediate("byo_hosted_system_subnets_pending_grant"), + [system_node_subnet_id, node_subnet_id, apiserver_subnet_id], + ) + + def test_process_add_role_assignment_for_byo_hobo_cli_285_fallback_only_defers_missing_subnets(self): + system_node_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/systemnode" + node_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/node" + apiserver_subnet_id = "/subscriptions/fakesub/resourceGroups/fakerg/providers/Microsoft.Network/virtualNetworks/fakevnet/subnets/apiserver" + dec_1 = AKSPreviewManagedClusterCreateDecorator( + self.cmd, + self.client, + { + "enable_managed_identity": True, + "sku": "automatic", + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }, + CUSTOM_MGMT_AKS_PREVIEW, + ) + mc_1 = self.models.ManagedCluster(location="test_location") + dec_1.context.attach_mc(mc_1) + + with patch.object( + AKSManagedClusterCreateDecorator, + "_get_byo_hosted_system_subnet_ids", + None, + create=True, + ), patch.object( + AKSManagedClusterCreateDecorator, + "process_add_role_assignment_for_vnet_subnet", + return_value=None, + ), patch.object( + dec_1.context.external_functions, + "subnet_role_assignment_exists", + side_effect=[True, False, True], + ): + dec_1.process_add_role_assignment_for_vnet_subnet(mc_1) + + self.assertTrue( + dec_1.context.get_intermediate("need_post_creation_vnet_permission_granting") + ) + self.assertEqual( + dec_1.context.get_intermediate("byo_hosted_system_subnets_pending_grant"), + [node_subnet_id], + ) def test_build_gitops_addon_profile(self): # default @@ -17782,6 +17866,33 @@ def test_update_agentpool_profile_with_none_agent_pool_profiles(self): self.assertIsNone(result_1.agent_pool_profiles) self.assertTrue(result_1.hosted_system_profile.enabled) + def test_update_agentpool_profile_hosted_system_with_user_pool_skips_default_pool_update(self): + dec_1 = AKSPreviewManagedClusterUpdateDecorator( + self.cmd, + self.client, + {}, + CUSTOM_MGMT_AKS_PREVIEW, + ) + agent_pool_profiles = [ + self.models.ManagedClusterAgentPoolProfile(name="userpool", mode="User") + ] + mc_1 = self.models.ManagedCluster( + location="test_location", + agent_pool_profiles=agent_pool_profiles, + hosted_system_profile=self.models.ManagedClusterHostedSystemProfile(enabled=True), + ) + dec_1.context.attach_mc(mc_1) + + with patch.object( + dec_1.agentpool_decorator, + "update_agentpool_profile_preview", + ) as update_agentpool_profile: + result_1 = dec_1.update_agentpool_profile(mc_1) + + self.assertIs(result_1, mc_1) + self.assertEqual(result_1.agent_pool_profiles, agent_pool_profiles) + update_agentpool_profile.assert_not_called() + def test_update_agentpool_profile_with_none_agent_pool_profiles_no_hosted_system( self, ):