From a2f8fde87defa3306d72dc7a38425ba154897547 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 1 Sep 2026 19:53:22 -0400 Subject: [PATCH 01/12] Use canonical locations for compute placement --- Taskfile.yaml | 8 +- api/v1alpha/instance_types.go | 11 +- api/v1alpha/labels.go | 6 +- api/v1alpha/workload_types.go | 6 +- api/v1alpha/workloaddeployment_types.go | 16 +- api/v1alpha/zz_generated.deepcopy.go | 15 +- cmd/main.go | 2 + ...ute.datumapis.com_workloaddeployments.yaml | 34 +- .../compute.datumapis.com_workloads.yaml | 17 +- config/components/controller_rbac/role.yaml | 10 +- .../compute_v1alpha_workload-sandbox.yaml | 4 +- .../samples/compute_v1alpha_workload-vm.yaml | 4 +- config/samples/kong-gateway.yaml | 10 +- config/samples/multi-placement-sandbox.yaml | 8 +- docs/api/instances.md | 410 +++++++++++++++++- docs/api/workloaddeployments.md | 205 ++++++--- docs/api/workloads.md | 246 ++++++++++- docs/enhancements/datumctl-compute-dx.md | 37 +- .../federated-deployment-scheduling.md | 16 +- go.mod | 1 + go.sum | 2 + internal/cmd/compute/deploy/deploy.go | 91 +++- internal/cmd/compute/destroy/destroy.go | 10 +- internal/cmd/compute/instances/instances.go | 72 ++- internal/cmd/compute/restart/restart.go | 21 +- internal/cmd/compute/scale/scale.go | 2 +- internal/cmd/compute/util/client.go | 4 + internal/cmd/compute/util/completion.go | 49 ++- internal/cmd/compute/watch/watch.go | 12 +- internal/cmd/compute/watch/watch_test.go | 2 +- internal/cmd/compute/workloads/workloads.go | 52 +-- internal/controller/indexers.go | 4 +- internal/controller/instance_controller.go | 4 +- .../controller/instance_controller_test.go | 2 + .../controller/instance_projector_test.go | 3 +- internal/controller/instance_setup_test.go | 18 + .../controller/instance_writeback_test.go | 16 +- .../stateful/stateful_control.go | 7 +- .../stateful/stateful_control_test.go | 65 +-- internal/controller/networkinterfaceclaim.go | 2 +- .../networkinterfaceclaim_controller_test.go | 13 +- .../controller/referenceddata_controller.go | 2 +- .../referenceddata_controller_test.go | 5 +- internal/controller/testing_helpers_test.go | 3 + internal/controller/workload_controller.go | 30 +- .../controller/workload_controller_test.go | 15 +- .../workloaddeployment_controller.go | 74 ++-- .../workloaddeployment_controller_test.go | 9 +- .../workloaddeployment_federator.go | 69 ++- .../workloaddeployment_federator_test.go | 65 +-- .../workloaddeployment_hpa_controller.go | 2 +- .../workloaddeployment_hpa_controller_test.go | 5 +- .../workloaddeployment_location_test.go | 56 ++- .../workloaddeployment_network_binding.go | 8 +- ...workloaddeployment_network_binding_test.go | 13 +- .../workloaddeployment_setup_test.go | 2 + .../validation/instance_validation_test.go | 10 +- internal/validation/workload_validation.go | 19 +- .../validation/workload_validation_test.go | 21 +- internal/webhook/v1alpha/workload_webhook.go | 29 +- test/e2e/chainsaw-config.yaml | 4 +- .../deletion-cascade/workload-deployment.yaml | 3 +- .../chainsaw-test.yaml | 3 +- .../federation-hub-reclaim/chainsaw-test.yaml | 3 +- .../instance-pop-dfw.yaml | 2 +- .../workload-deployment.yaml | 3 +- .../workload-deployment.yaml | 3 +- .../full-federation-ord/chainsaw-test.yaml | 22 +- .../workload-deployment.yaml | 3 +- .../full-federation/workload-deployment.yaml | 3 +- .../workload-deployment.yaml | 3 +- .../e2e/instance-writeback/chainsaw-test.yaml | 3 +- .../instance-writeback/instance-pop-dfw.yaml | 2 +- .../chainsaw-test.yaml | 10 +- .../workload-deployment-alpha.yaml | 3 +- .../workload-deployment-beta.yaml | 3 +- .../workload-deployment.yaml | 3 +- .../referenced-data-mounts/chainsaw-test.yaml | 8 +- .../workload-deployment.yaml | 3 +- .../chainsaw-test.yaml | 2 +- .../workload-deployment-no-pull-secret.yaml | 3 +- .../workload-deployment.yaml | 3 +- .../workload-deployment-suspended.yaml | 3 +- test/e2e/suspension/workload-deployment.yaml | 3 +- .../chainsaw-test.yaml | 20 +- .../workload-deployment.yaml | 3 +- .../workloaddeployment-retain-desired.yaml | 3 +- .../workloaddeployment-retain-observed.yaml | 3 +- ui/consumer/src/adapter.ts | 10 +- .../src/components/instance-page-chrome.tsx | 4 +- ui/consumer/src/pages/workload-detail.tsx | 20 +- ui/consumer/src/pages/workload-list.tsx | 6 +- ui/consumer/src/schema.ts | 6 +- ui/provider/src/adapter.ts | 8 +- ui/provider/src/pages/workload-detail.tsx | 12 +- ui/provider/src/pages/workload-list.tsx | 4 +- ui/provider/src/schema.ts | 6 +- 97 files changed, 1465 insertions(+), 695 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index f2af01e0..fbb25262 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -431,7 +431,7 @@ tasks: # ════════════════════════════════════════════════════════════════════════ e2e:karmada:join-clusters: - desc: "Register POP cell clusters with Karmada and apply city-code labels" + desc: "Register POP cell clusters with Karmada and apply canonical location labels" cmds: # cluster.karmada.io is served by the karmada-aggregated-apiserver through # API aggregation, and can briefly return ServiceUnavailable after the pod @@ -507,7 +507,7 @@ tasks: {{.CLUSTER_NAME}} \ {{.INTERNAL_KUBECONFIG}} # ── Apply cluster labels ─────────────────────────────────────────── - # city-code is what compute's federator places deployments by. + # location is the canonical identity compute's federator places deployments by. # # infra.datum.net/gateways=enabled is what NSO's propagation policy selects # on, and every cell in infra carries it. Without it that policy matches no @@ -516,10 +516,10 @@ tasks: - | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ label cluster {{.CLUSTER_NAME}} \ - topology.datum.net/city-code={{.CITY_CODE}} \ + topology.datum.net/location={{.CITY_CODE}} \ infra.datum.net/gateways=enabled \ --overwrite - echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}} and infra.datum.net/gateways=enabled" + echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/location={{.CITY_CODE}} and infra.datum.net/gateways=enabled" # ════════════════════════════════════════════════════════════════════════ # CRD installation diff --git a/api/v1alpha/instance_types.go b/api/v1alpha/instance_types.go index b24f16ab..b0a1b0d2 100644 --- a/api/v1alpha/instance_types.go +++ b/api/v1alpha/instance_types.go @@ -5,6 +5,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) // InstanceSpec defines the desired state of Instance @@ -40,7 +41,7 @@ type InstanceSpec struct { // The location which the instance has been scheduled to // // +kubebuilder:validation:Optional - Location *networkingv1alpha.LocationReference `json:"location,omitempty"` + Location *locationsv1alpha1.LocationReference `json:"location,omitempty"` // Controller contains settings driven by the controller managing the instance. // @@ -871,11 +872,11 @@ const ( // waits until the platform resolves the conflict. WorkloadDeploymentReasonAmbiguousServingLocation = "AmbiguousServingLocation" - // WorkloadDeploymentReasonCityCodeMismatch is set on - // WorkloadDeployment.Available when the deployment asks for one city and the - // cell serves another. It means the deployment was placed on the wrong cell, + // WorkloadDeploymentReasonLocationMismatch is set on + // WorkloadDeployment.Available when the deployment asks for one location and + // the cell serves another. It means the deployment was placed on the wrong cell, // which is a platform fault rather than anything the user can correct. - WorkloadDeploymentReasonCityCodeMismatch = "CityCodeMismatch" + WorkloadDeploymentReasonLocationMismatch = "LocationMismatch" // WorkloadDeploymentReasonNetworkProvisioning is set on WorkloadDeployment.Available // while the network binding or subnet is still being provisioned. diff --git a/api/v1alpha/labels.go b/api/v1alpha/labels.go index f894aa7e..ce5a997f 100644 --- a/api/v1alpha/labels.go +++ b/api/v1alpha/labels.go @@ -12,9 +12,9 @@ const ( // that owns an Instance. Stamped at creation and kept current on updates. WorkloadDeploymentNameLabel = LabelNamespace + "/workload-deployment-name" - // CityCodeLabel carries the city code of the WorkloadDeployment that owns - // an Instance, matching WorkloadDeploymentSpec.CityCode. - CityCodeLabel = LabelNamespace + "/city-code" + // LocationLabel carries the canonical location name of the + // WorkloadDeployment that owns an Instance. + LocationLabel = LabelNamespace + "/location" // WorkloadNameLabel carries the name of the Workload that an Instance // ultimately belongs to, sourced from WorkloadDeploymentSpec.WorkloadRef.Name. diff --git a/api/v1alpha/workload_types.go b/api/v1alpha/workload_types.go index e3e9b04e..0ed67b15 100644 --- a/api/v1alpha/workload_types.go +++ b/api/v1alpha/workload_types.go @@ -1,6 +1,7 @@ package v1alpha import ( + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" k8scorev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -136,10 +137,11 @@ type WorkloadPlacement struct { // +kubebuilder:validation:Required Name string `json:"name"` - // A list of city codes that define where the instances should be deployed. + // A list of locations where the instances should be deployed. // // +kubebuilder:validation:Required - CityCodes []string `json:"cityCodes,omitempty"` + // +kubebuilder:validation:MinItems=1 + Locations []locationsv1alpha1.LocationReference `json:"locations,omitempty"` // Scale settings such as minimum and maximum replica counts. // diff --git a/api/v1alpha/workloaddeployment_types.go b/api/v1alpha/workloaddeployment_types.go index 0c6c6380..f6dc1c29 100644 --- a/api/v1alpha/workloaddeployment_types.go +++ b/api/v1alpha/workloaddeployment_types.go @@ -1,9 +1,8 @@ package v1alpha import ( + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" ) // WorkloadDeploymentSpec defines the desired state of WorkloadDeployment @@ -18,11 +17,10 @@ type WorkloadDeploymentSpec struct { // +kubebuilder:validation:Required PlacementName string `json:"placementName"` - // TODO(jreese) think through how to structure this a bit better for when - // deployments can be scheduled in ways other than just a city code. + // The location where this deployment runs. // // +kubebuilder:validation:Required - CityCode string `json:"cityCode"` + LocationRef locationsv1alpha1.LocationReference `json:"locationRef"` // Defines settings for each instance. // @@ -43,11 +41,6 @@ type WorkloadDeploymentSpec struct { // WorkloadDeploymentStatus defines the observed state of WorkloadDeployment type WorkloadDeploymentStatus struct { - // The location which the deployment has been scheduled to - // - // +kubebuilder:validation:Optional - Location *networkingv1alpha.LocationReference `json:"location,omitempty"` - // Represents the observations of a deployment's current state. // Known condition types are: "Available", "Progressing" Conditions []metav1.Condition `json:"conditions,omitempty"` @@ -112,8 +105,7 @@ const ( // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas` // +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas` // +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.updatedReplicas` -// +kubebuilder:printcolumn:name="Location Namespace",type=string,JSONPath=`.status.location.namespace`,priority=1 -// +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name`,priority=1 +// +kubebuilder:printcolumn:name="Location",type=string,JSONPath=`.spec.locationRef.name` type WorkloadDeployment struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index f5002879..4ac235b8 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -8,6 +8,7 @@ package v1alpha import ( apiv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.miloapis.com/locations/api/v1alpha1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -582,7 +583,7 @@ func (in *InstanceSpec) DeepCopyInto(out *InstanceSpec) { } if in.Location != nil { in, out := &in.Location, &out.Location - *out = new(apiv1alpha.LocationReference) + *out = new(v1alpha1.LocationReference) **out = **in } if in.Controller != nil { @@ -1059,6 +1060,7 @@ func (in *WorkloadDeploymentList) DeepCopyObject() runtime.Object { func (in *WorkloadDeploymentSpec) DeepCopyInto(out *WorkloadDeploymentSpec) { *out = *in out.WorkloadRef = in.WorkloadRef + out.LocationRef = in.LocationRef in.Template.DeepCopyInto(&out.Template) in.ScaleSettings.DeepCopyInto(&out.ScaleSettings) if in.Replicas != nil { @@ -1081,11 +1083,6 @@ func (in *WorkloadDeploymentSpec) DeepCopy() *WorkloadDeploymentSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadDeploymentStatus) DeepCopyInto(out *WorkloadDeploymentStatus) { *out = *in - if in.Location != nil { - in, out := &in.Location, &out.Location - *out = new(apiv1alpha.LocationReference) - **out = **in - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) @@ -1195,9 +1192,9 @@ func (in *WorkloadList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadPlacement) DeepCopyInto(out *WorkloadPlacement) { *out = *in - if in.CityCodes != nil { - in, out := &in.CityCodes, &out.CityCodes - *out = make([]string, len(*in)) + if in.Locations != nil { + in, out := &in.Locations, &out.Locations + *out = make([]v1alpha1.LocationReference, len(*in)) copy(*out, *in) } in.ScaleSettings.DeepCopyInto(&out.ScaleSettings) diff --git a/cmd/main.go b/cmd/main.go index cb59e59e..64710aa5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -53,6 +53,7 @@ import ( computewebhook "go.datum.net/compute/internal/webhook" computev1alphawebhooks "go.datum.net/compute/internal/webhook/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" infrastructurev1alpha1 "go.miloapis.com/milo/pkg/apis/infrastructure/v1alpha1" quotav1alpha1 "go.miloapis.com/milo/pkg/apis/quota/v1alpha1" resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" @@ -88,6 +89,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(config.AddToScheme(scheme)) + utilruntime.Must(locationsv1alpha1.AddToScheme(scheme)) utilruntime.Must(config.RegisterDefaults(scheme)) utilruntime.Must(computev1alpha.AddToScheme(scheme)) utilruntime.Must(networkingv1alpha.AddToScheme(scheme)) diff --git a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml index 48fa5d2e..b7fb3ead 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml @@ -37,13 +37,8 @@ spec: - jsonPath: .status.updatedReplicas name: Up-to-date type: string - - jsonPath: .status.location.namespace - name: Location Namespace - priority: 1 - type: string - - jsonPath: .status.location.name - name: Location Name - priority: 1 + - jsonPath: .spec.locationRef.name + name: Location type: string name: v1alpha schema: @@ -71,10 +66,15 @@ spec: spec: description: WorkloadDeploymentSpec defines the desired state of WorkloadDeployment properties: - cityCode: - description: deployments can be scheduled in ways other than just - a city code. - type: string + locationRef: + description: The location where this deployment runs. + properties: + name: + description: Name of a datum location + type: string + required: + - name + type: object placementName: description: The placement in the workload which is driving a deployment type: string @@ -1192,7 +1192,7 @@ spec: - uid type: object required: - - cityCode + - locationRef - placementName - scaleSettings - template @@ -1270,16 +1270,6 @@ spec: description: The desired number of instances format: int32 type: integer - location: - description: The location which the deployment has been scheduled - to - properties: - name: - description: Name of a datum location - type: string - required: - - name - type: object observedGeneration: description: |- The most recent generation observed by the deployment controller. When diff --git a/config/base/crd/bases/compute.datumapis.com_workloads.yaml b/config/base/crd/bases/compute.datumapis.com_workloads.yaml index 41dfd418..654e86b7 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloads.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloads.yaml @@ -71,11 +71,18 @@ spec: will live in, such as in a city, or region. items: properties: - cityCodes: - description: A list of city codes that define where the instances - should be deployed. + locations: + description: A list of locations where the instances should + be deployed. items: - type: string + properties: + name: + description: Name of a datum location + type: string + required: + - name + type: object + minItems: 1 type: array name: description: The name of the placement @@ -148,7 +155,7 @@ spec: - minReplicas type: object required: - - cityCodes + - locations - name - scaleSettings type: object diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index acedcc22..f619cd36 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -76,6 +76,15 @@ rules: verbs: - create - patch +- apiGroups: + - locations.miloapis.com + resources: + - locations + - servinglocations + verbs: + - get + - list + - watch - apiGroups: - networking.datumapis.com resources: @@ -93,7 +102,6 @@ rules: resources: - networkinterfaces - networks - - servinglocations verbs: - get - list diff --git a/config/samples/compute_v1alpha_workload-sandbox.yaml b/config/samples/compute_v1alpha_workload-sandbox.yaml index ca909c04..4aa5adc8 100644 --- a/config/samples/compute_v1alpha_workload-sandbox.yaml +++ b/config/samples/compute_v1alpha_workload-sandbox.yaml @@ -57,7 +57,7 @@ spec: name: workload-sandbox-sample-configmap placements: - name: us - cityCodes: - - DFW + locations: + - name: gcp-us-south1-a scaleSettings: minReplicas: 1 diff --git a/config/samples/compute_v1alpha_workload-vm.yaml b/config/samples/compute_v1alpha_workload-vm.yaml index b3d6df97..a3c87543 100644 --- a/config/samples/compute_v1alpha_workload-vm.yaml +++ b/config/samples/compute_v1alpha_workload-vm.yaml @@ -65,7 +65,7 @@ spec: name: workload-vm-sample-configmap placements: - name: us-south - cityCodes: - - DFW + locations: + - name: gcp-us-south1-a scaleSettings: minReplicas: 1 diff --git a/config/samples/kong-gateway.yaml b/config/samples/kong-gateway.yaml index 547dfe8d..72c05e81 100644 --- a/config/samples/kong-gateway.yaml +++ b/config/samples/kong-gateway.yaml @@ -99,14 +99,14 @@ spec: name: kong-gateway-config placements: - name: us - cityCodes: - - DFW - - DLS + locations: + - name: gcp-us-south1-a + - name: gcp-us-west1-a scaleSettings: minReplicas: 1 - name: eu - cityCodes: - - LHR + locations: + - name: gcp-europe-west2-a scaleSettings: minReplicas: 1 # Think about multiple gateways per workload? diff --git a/config/samples/multi-placement-sandbox.yaml b/config/samples/multi-placement-sandbox.yaml index 4769fcea..966393fd 100644 --- a/config/samples/multi-placement-sandbox.yaml +++ b/config/samples/multi-placement-sandbox.yaml @@ -29,12 +29,12 @@ spec: cidr: 0.0.0.0/0 placements: - name: us-south - cityCodes: - - DFW + locations: + - name: gcp-us-south1-a scaleSettings: minReplicas: 1 - name: us-south2 - cityCodes: - - DFW + locations: + - name: gcp-us-south1-a scaleSettings: minReplicas: 1 diff --git a/docs/api/instances.md b/docs/api/instances.md index b603ad99..15f03b65 100644 --- a/docs/api/instances.md +++ b/docs/api/instances.md @@ -89,7 +89,13 @@ Spec defines the desired state of an Instance. networkInterfaces []object - Network interface configuration.
+ Network interface configuration. + +Keyed by interface name so an interface keeps its identity, and therefore +its addresses, across updates to the rest of the list. + +Limited to a single interface until the data plane can attach more than +one to an instance.
true @@ -130,7 +136,13 @@ Virtual Machine.
+InstanceNetworkInterface describes one interface an instance needs. The +fields beyond `network` and `networkPolicy` are copied verbatim onto the +NetworkInterfaceClaim created for each instance slot, so they carry the same +meaning, defaults, and immutability the claim API defines. +The location an interface is claimed in is implicit: the claim is created in +the control plane serving the instance, which is already location scoped. @@ -148,6 +160,53 @@ Virtual Machine.
The network to attach the network interface to.
+ + + + + + + + + + + + + + + @@ -160,6 +219,29 @@ will be of the lowest priority, and can effectively be prohibited from influencing network connectivity.
+ + + + +
true
addresses[]object + Requests for addresses beyond the ones the interface holds inside its +network, such as a public IPv4 address in front of a private one. Each is +reported in the interface's `externalAddresses` status. + +Omit this field for ordinary private addressing, which is the common case.
+
+ Validations:
  • self.all(a, self.exists_one(b, b.class == a.class)): Each address class may be requested at most once
    • +     		false
    ipFamilies[]enum + The address families the interface must carry, in priority order. List +[IPv6, IPv4] for a dual-stack interface. The first family listed holds the +interface's primary address, which is the one reported as the instance's +network IP. + +Every family listed must be satisfiable or the interface is never +published, so asking for a family the network does not carry fails rather +than yielding a partially addressed interface.
    +
    + Validations:
  • self.all(f, self.exists_one(g, g == f)): Each address family may be requested at most once
  • self == oldSelf: ipFamilies is immutable and cannot be changed after creation
    • + Enum: IPv4, IPv6
    + Default: [IPv6]
    +     		false
    namestring + The name of the interface, such as eth0 or eth1. It is both the device +name the guest operating system sees and the suffix of the interface +claim's name, which is what keeps an interface's addresses with the +instance slot across replacement. + +Immutable, because the guest is configured against it and the claim is +named after it.
    +
    + Validations:
  • self == oldSelf: name is immutable and cannot be changed after creation
    • + Default: eth0
    +     		false
    networkPolicy object false
    reclaimPolicyenum + What becomes of the interface, and its addresses, when the instance slot +it serves goes away. + +Delete returns the addresses to IPAM, so an instance recreated later comes +back on different addresses. Retain keeps them reserved, and billable, so a +later instance filling the same slot returns to the same addresses. Choose +Retain when an address is published in DNS, allowed through a firewall, or +otherwise depended on from outside. + +Both policies keep the addresses for as long as the slot exists, including +across instance replacement. They differ only on scale-down and deletion. + +Immutable. An address keeps the policy it was allocated under.
    +
    + Validations:
  • self == oldSelf: reclaimPolicy is immutable and cannot be changed after creation
    • + Enum: Delete, Retain
    + Default: Delete
    +     		false
    @@ -200,6 +282,38 @@ Defaults to the namespace for the type the reference is embedded in.
    +### Instance.spec.networkInterfaces[index].addresses[index] +[↩ Parent](#instancespecnetworkinterfacesindex) + + + +InstanceNetworkInterfaceAddressRequest asks for one address beyond the ones +the interface holds inside its network. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    classstring + The IPAM class to allocate from, such as public-ipv4. + +A class names a kind of address, and the platform decides which pool and +prefix length serve it. A class never names a pool, a prefix length, or a +CIDR, so a class cannot be used to ask for a particular address.
    +     		true
    + + ### Instance.spec.networkInterfaces[index].networkPolicy [↩ Parent](#instancespecnetworkinterfacesindex) @@ -1426,13 +1540,6 @@ The location which the instance has been scheduled to Name of a datum location
    true - - namespace - string - - Namespace for the datum location
    - - true @@ -2052,6 +2159,15 @@ Known condition types are: "Available", "Progressing"
    Network interface information
    false + + suspended + boolean + + Suspended, when true, indicates that the instance's process should be stopped +without releasing its placement, disk attachments, or quota allocation. +The provider controller stops the running container/VM.
    + + false @@ -2177,10 +2293,123 @@ Controller contains status information about the controller managing the instanc + addresses + []object + + The addresses the interface holds inside its network, each with its prefix +length and, once the location has a subnet, its gateway.
    + + false + assignments object + Single address projections of the fields above, kept for clients that read +one address per interface.
    + + false + + conditions + []object + + The observations of this interface's current state. Known condition types +are "Allocated" and "Programmed".
    + + false + + externalAddresses + []object + + The addresses the interface is reachable at from outside its network, one +per class requested in the spec. Each is a bare address with no prefix +length.
    + + false + + name + string + + The name of the interface this entry reports on, matching the name in the +instance's spec.
    + + false + + networkInterfaceRef + object + + The NetworkInterface bound to this entry, in the instance's namespace. An +infrastructure provider follows it to configure the NIC, so it never has to +derive the name of the claim that produced it.
    + + false + + + + +### Instance.status.networkInterfaces[index].addresses[index] +[↩ Parent](#instancestatusnetworkinterfacesindex) + + + +InstanceNetworkInterfaceAddress is an address the interface holds inside its +network. These are configured on the NIC itself, and always carry a prefix +length. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2192,7 +2421,8 @@ Controller contains status information about the controller managing the instanc - +Single address projections of the fields above, kept for clients that read +one address per interface.
    NameTypeDescriptionRequired
    addressstring + The address the interface holds, in CIDR notation, such as 10.128.0.2/32 +or 2001:db8:a001::1/128. + +For IPv6 this may be a block delegated to the interface rather than a +single address, such as 2001:db8:a001::/96. The interface owns the whole +block and assigns within it.
    +     		true
    familyenum + The address family of this entry.

    + Enum: IPv4, IPv6
    +     		true
    classstring + The IPAM class this address was allocated from, such as private-ipv6. It +is empty for addresses requested by family rather than by class.
    +     		false
    gatewaystring + The next hop the interface routes through for this family, such as +10.128.0.1. It is empty until the subnet backing the network in this +location exists.
    +     		false
    primaryboolean + Marks the address projected into `assignments.networkIP`. + +Exactly one address is primary for the interface as a whole, not one per +family. It is the address of the first family listed in `ipFamilies`.
    		 false
    @@ -2208,15 +2438,173 @@ Controller contains status information about the controller managing the instanc
    string The external IP address used for the interface. A one to one NAT will be -performed for this address with the interface's network IP.
    +performed for this address with the interface's network IP. It is a +projection of the first entry in the interface's `externalAddresses`.
    		 false
    networkIP string - The IP address assigned as the primary IP from the attached network.
    + The IP address assigned as the primary IP from the attached network. It is +a projection of the primary entry in the interface's `addresses`.
    		 false
    + + +### Instance.status.networkInterfaces[index].conditions[index] +[↩ Parent](#instancestatusnetworkinterfacesindex) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### Instance.status.networkInterfaces[index].externalAddresses[index] +[↩ Parent](#instancestatusnetworkinterfacesindex) + + + +InstanceNetworkInterfaceExternalAddress is an address reachable from outside +the network, mapped onto an address the interface holds inside it. A public +IPv4 address in front of a private address is the usual case. + +Unlike an interface address, it is a bare address with no prefix length, +because nothing configures it on the NIC. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + The externally reachable address, such as 203.0.113.10. It carries no +prefix length.
    +     		true
    classstring + The IPAM class this address was allocated from, such as public-ipv4. It +matches a class requested in the interface's `addresses`.
    +     		true
    familyenum + The address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    + + +### Instance.status.networkInterfaces[index].networkInterfaceRef +[↩ Parent](#instancestatusnetworkinterfacesindex) + + + +The NetworkInterface bound to this entry, in the instance's namespace. An +infrastructure provider follows it to configure the NIC, so it never has to +derive the name of the claim that produced it. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + name is the network interface name.
    +     		true
    diff --git a/docs/api/workloaddeployments.md b/docs/api/workloaddeployments.md index e3e5b079..2ff3863b 100644 --- a/docs/api/workloaddeployments.md +++ b/docs/api/workloaddeployments.md @@ -84,10 +84,10 @@ WorkloadDeploymentSpec defines the desired state of WorkloadDeployment - cityCode - string + locationRef + object - deployments can be scheduled in ways other than just a city code.
    + The location where this deployment runs.
    true @@ -132,6 +132,33 @@ unset, the deployment reconciles to scaleSettings.minReplicas.
    +### WorkloadDeployment.spec.locationRef +[↩ Parent](#workloaddeploymentspec) + + + +The location where this deployment runs. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of a datum location
    +     		true
    + + ### WorkloadDeployment.spec.scaleSettings [↩ Parent](#workloaddeploymentspec) @@ -348,7 +375,13 @@ Describes the desired configuration of an instance networkInterfaces []object - Network interface configuration.
    + Network interface configuration. + +Keyed by interface name so an interface keeps its identity, and therefore +its addresses, across updates to the rest of the list. + +Limited to a single interface until the data plane can attach more than +one to an instance.
    true @@ -389,7 +422,13 @@ Virtual Machine.
    +InstanceNetworkInterface describes one interface an instance needs. The +fields beyond `network` and `networkPolicy` are copied verbatim onto the +NetworkInterfaceClaim created for each instance slot, so they carry the same +meaning, defaults, and immutability the claim API defines. +The location an interface is claimed in is implicit: the claim is created in +the control plane serving the instance, which is already location scoped. @@ -407,6 +446,53 @@ Virtual Machine.
    The network to attach the network interface to.
    + + + + + + + + + + + + + + + @@ -419,6 +505,29 @@ will be of the lowest priority, and can effectively be prohibited from influencing network connectivity.
    + + + + +
    true
    addresses[]object + Requests for addresses beyond the ones the interface holds inside its +network, such as a public IPv4 address in front of a private one. Each is +reported in the interface's `externalAddresses` status. + +Omit this field for ordinary private addressing, which is the common case.
    +
    + Validations:
  • self.all(a, self.exists_one(b, b.class == a.class)): Each address class may be requested at most once
    • +     		false
    ipFamilies[]enum + The address families the interface must carry, in priority order. List +[IPv6, IPv4] for a dual-stack interface. The first family listed holds the +interface's primary address, which is the one reported as the instance's +network IP. + +Every family listed must be satisfiable or the interface is never +published, so asking for a family the network does not carry fails rather +than yielding a partially addressed interface.
    +
    + Validations:
  • self.all(f, self.exists_one(g, g == f)): Each address family may be requested at most once
  • self == oldSelf: ipFamilies is immutable and cannot be changed after creation
    • + Enum: IPv4, IPv6
    + Default: [IPv6]
    +     		false
    namestring + The name of the interface, such as eth0 or eth1. It is both the device +name the guest operating system sees and the suffix of the interface +claim's name, which is what keeps an interface's addresses with the +instance slot across replacement. + +Immutable, because the guest is configured against it and the claim is +named after it.
    +
    + Validations:
  • self == oldSelf: name is immutable and cannot be changed after creation
    • + Default: eth0
    +     		false
    networkPolicy object false
    reclaimPolicyenum + What becomes of the interface, and its addresses, when the instance slot +it serves goes away. + +Delete returns the addresses to IPAM, so an instance recreated later comes +back on different addresses. Retain keeps them reserved, and billable, so a +later instance filling the same slot returns to the same addresses. Choose +Retain when an address is published in DNS, allowed through a firewall, or +otherwise depended on from outside. + +Both policies keep the addresses for as long as the slot exists, including +across instance replacement. They differ only on scale-down and deletion. + +Immutable. An address keeps the policy it was allocated under.
    +
    + Validations:
  • self == oldSelf: reclaimPolicy is immutable and cannot be changed after creation
    • + Enum: Delete, Retain
    + Default: Delete
    +     		false
    @@ -459,6 +568,38 @@ Defaults to the namespace for the type the reference is embedded in.
    +### WorkloadDeployment.spec.template.spec.networkInterfaces[index].addresses[index] +[↩ Parent](#workloaddeploymentspectemplatespecnetworkinterfacesindex) + + + +InstanceNetworkInterfaceAddressRequest asks for one address beyond the ones +the interface holds inside its network. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    classstring + The IPAM class to allocate from, such as public-ipv4. + +A class names a kind of address, and the platform decides which pool and +prefix length serve it. A class never names a pool, a prefix length, or a +CIDR, so a class cannot be used to ask for a particular address.
    +     		true
    + + ### WorkloadDeployment.spec.template.spec.networkInterfaces[index].networkPolicy [↩ Parent](#workloaddeploymentspectemplatespecnetworkinterfacesindex) @@ -1685,13 +1826,6 @@ The location which the instance has been scheduled to Name of a datum location
    true - - namespace - string - - Namespace for the datum location
    - - true @@ -2435,13 +2569,6 @@ back up — making an in-progress roll observable.
    Known condition types are: "Available", "Progressing"
    false - - location - object - - The location which the deployment has been scheduled to
    - - false observedGeneration integer @@ -2460,6 +2587,14 @@ latest spec (e.g. a restart request).
    Selector is the label selector that identifies Pods backing this deployment.
    false + + suspended + boolean + + Suspended, when true, requests that all instances managed by this deployment +be stopped without releasing their placement, disk attachments, or quota allocation.
    + + false @@ -2539,37 +2674,3 @@ with respect to the current state of the instance.
    false - - -### WorkloadDeployment.status.location -[↩ Parent](#workloaddeploymentstatus) - - - -The location which the deployment has been scheduled to - - - - - - - - - - - - - - - - - - - - - -
    NameTypeDescriptionRequired
    namestring - Name of a datum location
    -     		true
    namespacestring - Namespace for the datum location
    -     		true
    diff --git a/docs/api/workloads.md b/docs/api/workloads.md index df10dcc0..a681cf31 100644 --- a/docs/api/workloads.md +++ b/docs/api/workloads.md @@ -119,10 +119,10 @@ will live in, such as in a city, or region.
    - cityCodes - []string + locations + []object - A list of city codes that define where the instances should be deployed.
    + A list of locations where the instances should be deployed.
    true @@ -143,6 +143,33 @@ will live in, such as in a city, or region.
    +### Workload.spec.placements[index].locations[index] +[↩ Parent](#workloadspecplacementsindex) + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of a datum location
    +     		true
    + + ### Workload.spec.placements[index].scaleSettings [↩ Parent](#workloadspecplacementsindex) @@ -359,7 +386,13 @@ Describes the desired configuration of an instance networkInterfaces []object - Network interface configuration.
    + Network interface configuration. + +Keyed by interface name so an interface keeps its identity, and therefore +its addresses, across updates to the rest of the list. + +Limited to a single interface until the data plane can attach more than +one to an instance.
    true @@ -400,7 +433,13 @@ Virtual Machine.
    +InstanceNetworkInterface describes one interface an instance needs. The +fields beyond `network` and `networkPolicy` are copied verbatim onto the +NetworkInterfaceClaim created for each instance slot, so they carry the same +meaning, defaults, and immutability the claim API defines. +The location an interface is claimed in is implicit: the claim is created in +the control plane serving the instance, which is already location scoped. @@ -418,6 +457,53 @@ Virtual Machine.
    The network to attach the network interface to.
    + + + + + + + + + + + + + + + @@ -430,6 +516,29 @@ will be of the lowest priority, and can effectively be prohibited from influencing network connectivity.
    + + + + +
    true
    addresses[]object + Requests for addresses beyond the ones the interface holds inside its +network, such as a public IPv4 address in front of a private one. Each is +reported in the interface's `externalAddresses` status. + +Omit this field for ordinary private addressing, which is the common case.
    +
    + Validations:
  • self.all(a, self.exists_one(b, b.class == a.class)): Each address class may be requested at most once
    • +     		false
    ipFamilies[]enum + The address families the interface must carry, in priority order. List +[IPv6, IPv4] for a dual-stack interface. The first family listed holds the +interface's primary address, which is the one reported as the instance's +network IP. + +Every family listed must be satisfiable or the interface is never +published, so asking for a family the network does not carry fails rather +than yielding a partially addressed interface.
    +
    + Validations:
  • self.all(f, self.exists_one(g, g == f)): Each address family may be requested at most once
  • self == oldSelf: ipFamilies is immutable and cannot be changed after creation
    • + Enum: IPv4, IPv6
    + Default: [IPv6]
    +     		false
    namestring + The name of the interface, such as eth0 or eth1. It is both the device +name the guest operating system sees and the suffix of the interface +claim's name, which is what keeps an interface's addresses with the +instance slot across replacement. + +Immutable, because the guest is configured against it and the claim is +named after it.
    +
    + Validations:
  • self == oldSelf: name is immutable and cannot be changed after creation
    • + Default: eth0
    +     		false
    networkPolicy object false
    reclaimPolicyenum + What becomes of the interface, and its addresses, when the instance slot +it serves goes away. + +Delete returns the addresses to IPAM, so an instance recreated later comes +back on different addresses. Retain keeps them reserved, and billable, so a +later instance filling the same slot returns to the same addresses. Choose +Retain when an address is published in DNS, allowed through a firewall, or +otherwise depended on from outside. + +Both policies keep the addresses for as long as the slot exists, including +across instance replacement. They differ only on scale-down and deletion. + +Immutable. An address keeps the policy it was allocated under.
    +
    + Validations:
  • self == oldSelf: reclaimPolicy is immutable and cannot be changed after creation
    • + Enum: Delete, Retain
    + Default: Delete
    +     		false
    @@ -470,6 +579,38 @@ Defaults to the namespace for the type the reference is embedded in.
    +### Workload.spec.template.spec.networkInterfaces[index].addresses[index] +[↩ Parent](#workloadspectemplatespecnetworkinterfacesindex) + + + +InstanceNetworkInterfaceAddressRequest asks for one address beyond the ones +the interface holds inside its network. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    classstring + The IPAM class to allocate from, such as public-ipv4. + +A class names a kind of address, and the platform decides which pool and +prefix length serve it. A class never names a pool, a prefix length, or a +CIDR, so a class cannot be used to ask for a particular address.
    +     		true
    + + ### Workload.spec.template.spec.networkInterfaces[index].networkPolicy [↩ Parent](#workloadspectemplatespecnetworkinterfacesindex) @@ -1696,13 +1837,6 @@ The location which the instance has been scheduled to Name of a datum location
    true - - namespace - string - - Namespace for the datum location
    - - true @@ -2558,6 +2692,24 @@ conditions:
    false + + attachedListenerSets + integer + + AttachedListenerSets represents the total number of ListenerSets that have been +successfully attached to this Gateway. + +A ListenerSet is successfully attached to a Gateway when all the following conditions are met: +- The ListenerSet is selected by the Gateway's AllowedListeners field +- The ListenerSet has a valid ParentRef selecting the Gateway +- The ListenerSet's status has the condition "Accepted: true" + +Uses for this field include troubleshooting AttachedListenerSets attachment and +measuring blast radius/impact of changes to a Gateway.
    +
    + Format: int32
    + + false conditions []object @@ -2573,7 +2725,35 @@ Known condition types are: * "Accepted" * "Programmed" -* "Ready"
    +* "Ready" + + +Notes for implementors: + +Conditions are a listType `map`, which means that they function like a +map with a key of the `type` field _in the k8s apiserver_. + +This means that implementations must obey some rules when updating this +section. + +* Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. +* Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. +* Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. +* Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. +* If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) +

    Default: [map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Accepted] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Programmed]]
    @@ -2737,8 +2917,11 @@ resource or a specific Listener as a parent resource (more detail on attachment semantics can be found in the documentation on the various Route kinds ParentRefs fields). Listener or Route status does not impact successful attachment, i.e. the AttachedRoutes field count MUST be set -for Listeners with condition Accepted: false and MUST count successfully -attached Routes that may themselves have Accepted: false conditions. +for Listeners, even if the Accepted condition of an individual Listener is set +to "False". The AttachedRoutes number represents the number of Routes with +the Accepted condition set to "True" that have been attached to this Listener. +Routes with any other value for the Accepted condition MUST NOT be included +in this count. Uses for this field include troubleshooting Route attachment and measuring blast radius/impact of changes to a Listener.
    @@ -2750,7 +2933,36 @@ measuring blast radius/impact of changes to a Listener.
    conditions []object - Conditions describe the current condition of this listener.
    + Conditions describe the current condition of this listener. + + +Notes for implementors: + +Conditions are a listType `map`, which means that they function like a +map with a key of the `type` field _in the k8s apiserver_. + +This means that implementations must obey some rules when updating this +section. + +* Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. +* Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. +* Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. +* Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. +* If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + +
    true @@ -2765,7 +2977,7 @@ measuring blast radius/impact of changes to a Listener.
    []object SupportedKinds is the list indicating the Kinds supported by this -listener. This MUST represent the kinds an implementation supports for +listener. This MUST represent the kinds supported by an implementation for that Listener configuration. If kinds are specified in Spec that are not supported, they MUST NOT @@ -2774,7 +2986,7 @@ condition to "False" with the "InvalidRouteKinds" reason. If both valid and invalid Route kinds are specified, the implementation MUST reference the valid Route kinds that have been specified.
    - true + false diff --git a/docs/enhancements/datumctl-compute-dx.md b/docs/enhancements/datumctl-compute-dx.md index 66ab3029..a999febd 100644 --- a/docs/enhancements/datumctl-compute-dx.md +++ b/docs/enhancements/datumctl-compute-dx.md @@ -49,24 +49,24 @@ The fastest path requires no YAML: $ datumctl compute deploy api \ --image=ghcr.io/acme/api:1.4.2 \ --instance-type=d1-standard-2 \ - --city=DFW,IAD \ + --location=us-east-1,eu-west-1 \ --min=2 \ --port=8080 Resolving workload "api" in project acme-prod... Workload does not exist — creating. - Placement "default": cities=[DFW, IAD], min=2 + Placement "default": locations=[us-east-1, eu-west-1], min=2 Applying... workload/api created Waiting for rollout. Ctrl-C to detach (rollout continues in background). - PLACEMENT CITY DESIRED READY PHASE - default DFW 2 0 Starting - default IAD 2 0 Starting - default DFW 2 2 Running - default IAD 2 2 Running + PLACEMENT LOCATION DESIRED READY PHASE + default us-east-1 2 0 Starting + default eu-west-1 2 0 Starting + default us-east-1 2 2 Running + default eu-west-1 2 2 Running Rollout complete in 47s. @@ -86,14 +86,14 @@ $ datumctl compute deploy ? Workload name: api ? Container image: ghcr.io/acme/api:1.4.2 ? Instance type [d1-standard-2]: -? Cities (comma-separated) [DFW]: DFW,IAD -? Min replicas per city [1]: 2 +? Locations [us-east-1]: us-east-1,eu-west-1 +? Min replicas per location [1]: 2 ? Expose port (optional): 8080 workload: api image: ghcr.io/acme/api:1.4.2 instance type: d1-standard-2 - cities: DFW, IAD + locations: us-east-1, eu-west-1 replicas: min=2 ports: 8080/tcp @@ -130,9 +130,9 @@ Updated 47s ago Revision #7 Health Available — all placements at desired replicas - CITY READY DESIRED TYPE - default DFW 2/2 2 d1-standard-2 - IAD 2/2 2 d1-standard-2 + LOCATION READY DESIRED TYPE + default us-east-1 2/2 2 d1-standard-2 + eu-west-1 2/2 2 d1-standard-2 ``` When something is wrong, the status view explains it in plain terms and tells the developer what to do next: @@ -170,7 +170,7 @@ $ datumctl compute rollout api Rolling workload "api" rev #7 → #8 - PLACEMENT CITY UPDATED READY OLD PHASE + PLACEMENT LOCATION UPDATED READY OLD PHASE default DFW 0 2 2 Pending default IAD 0 2 2 Pending default DFW 1 1 1 Updating @@ -232,7 +232,7 @@ Tailing logs for workload "api" in DFW, IAD. Ctrl-C to stop. Common filters reduce the output without requiring instance name lookup: ``` -$ datumctl compute logs api --city=IAD --follow +$ datumctl compute logs api --location=us-east-1 --follow $ datumctl compute logs api --since=15m $ datumctl compute logs api -c worker --follow ``` @@ -246,7 +246,7 @@ When something is wrong with a specific instance, `datumctl compute instances` g ``` $ datumctl compute instances - NAME WORKLOAD CITY INTERNAL IP TYPE AGE STATUS + NAME WORKLOAD LOCATION INTERNAL IP TYPE AGE STATUS api-dfw-0 api DFW 10.4.1.5 d1-standard-2 2d Running api-dfw-1 api DFW 10.4.1.6 d1-standard-2 2d Running api-iad-0 api IAD 10.5.1.7 d1-standard-2 2d Running @@ -303,8 +303,8 @@ Next steps ``` datumctl compute deploy Deploy or update a workload datumctl compute status Show health across all cities -datumctl compute instances List all instances (--workload, --city to filter) -datumctl compute logs Stream logs (--workload, --city, --instance, -c/--container) +datumctl compute instances List all instances (--workload, --location to filter) +datumctl compute logs Stream logs (--workload, --location, --instance, -c/--container) datumctl compute rollout Watch a rollout in progress datumctl compute rollout history List recent revisions datumctl compute rollout undo Roll back to a previous revision @@ -327,4 +327,3 @@ datumctl compute cities [list | describe] datumctl compute instance-types [list | describe] datumctl compute quota [--breakdown | --constrained | --city=CITY] ``` - diff --git a/docs/enhancements/federated-deployment-scheduling.md b/docs/enhancements/federated-deployment-scheduling.md index be2e0dde..e95d6804 100644 --- a/docs/enhancements/federated-deployment-scheduling.md +++ b/docs/enhancements/federated-deployment-scheduling.md @@ -23,7 +23,7 @@ From a user perspective, nothing changes — you still specify city codes, and y - **Control Plane Cell** — The central compute operator that coordinates between Projects and the Karmada federation layer. - **Karmada** — An open-source multi-cluster orchestration system that distributes workloads across registered member clusters (POP Cells) and aggregates their status. - **Karmada API Server** — The central federation API server managed by Karmada. WorkloadDeployments are written here so Karmada can propagate them to the correct POP Cell. -- **PropagationPolicy** — A Karmada resource that defines which clusters a resource should be sent to, based on label selectors. One is created per city code per project namespace. +- **PropagationPolicy** — A Karmada resource that defines which clusters a resource should be sent to, based on label selectors. One is created per canonical location per project namespace. - **Management Cluster** — The central Kubernetes cluster that hosts shared platform infrastructure. - **NSO** — Network Services Operator — runs in each POP Cell to provision networking resources (NetworkBinding, SubnetClaim, Subnet) needed by Instances. - **Milo** — Datum's shared platform library. Provides utilities like namespace mapping and multi-tenant client strategies used across services. @@ -78,7 +78,7 @@ that replaces the single-platform-API-server MVP architecture. This document def │ Karmada Federation API Server │ │ │ │ WorkloadDeployment (propagated to POP cells) │ -│ PropagationPolicy (one per city code per namespace) │ +│ PropagationPolicy (one per location per namespace) │ │ Instance (written back by POP cell for visibility) │ │ Cluster objects (one per POP cell, labeled by city) │ └───────────────────┬─────────────────────────────────────┘ @@ -111,7 +111,7 @@ that replaces the single-platform-API-server MVP architecture. This document def | `Workload` | Project | Consumer | | `WorkloadDeployment` (consumer-facing) | Project | `WorkloadReconciler` (spec), `WorkloadDeploymentFederator` (status) | | `WorkloadDeployment` (federation intent) | Karmada API Server | `WorkloadDeploymentFederator` | -| `PropagationPolicy` | Karmada API Server | `WorkloadDeploymentFederator` (one per city code per namespace, lazy) | +| `PropagationPolicy` | Karmada API Server | `WorkloadDeploymentFederator` (one per location per namespace, lazy) | | `Instance` (write-back) | Karmada API Server | `InstanceReconciler` (POP cell) | | `Instance` (local execution) | POP Cell | `WorkloadDeploymentReconciler` (POP cell) | | `Instance` (projection) | Project | `InstanceProjector` | @@ -138,11 +138,11 @@ sequenceDiagram Project->>CPC: WorkloadReconciler watches Workload CPC->>Project: query Locations for city codes - CPC->>Project: create WorkloadDeployment (spec only, per city) + CPC->>Project: create WorkloadDeployment (spec only, per location) Project->>CPC: WorkloadDeploymentFederator watches WorkloadDeployment CPC->>Karmada: create WorkloadDeployment (labeled with city code) - CPC->>Karmada: create PropagationPolicy (once per city code, lazy) + CPC->>Karmada: create PropagationPolicy (once per location, lazy) Karmada->>POP: propagate WorkloadDeployment @@ -230,7 +230,7 @@ A new controller in the Control Plane Cell: - Watches `WorkloadDeployment` in every project (via multicluster-runtime). - On create/update: upserts a corresponding `WorkloadDeployment` (labeled with city code) in the Karmada API Server. -- Creates a `PropagationPolicy` per city code per project namespace lazily on first use. +- Creates a `PropagationPolicy` per location per project namespace lazily on first use. - Reads aggregated `WorkloadDeployment.status` from the Karmada API Server and writes it to the project. - On delete: removes the Karmada-side `WorkloadDeployment`. Removes the `PropagationPolicy` when no remaining deployment in the namespace targets that city code. @@ -240,7 +240,7 @@ A new controller in the Control Plane Cell: - Unchanged behavior: creates `Instance`, `NetworkBinding`, `SubnetClaim` using existing stateful control logic. - Manages `network` scheduling gate removal once NSO signals networks are ready. - Updates local `WorkloadDeployment.status` with aggregate replica counts (Karmada aggregates this back natively). -- **Remove**: `WorkloadDeployment.status.location` (location is now implicit in `spec.cityCode`). +- **Remove**: `WorkloadDeployment.status.location` (location is explicit in `spec.locationRef`). ### `InstanceReconciler` @@ -352,7 +352,7 @@ The `MappedNamespaceResourceStrategy` pattern will be promoted from NSO's `inter ### PropagationPolicy Scope -One `PropagationPolicy` per city code per project namespace, using a `labelSelector` to match all `WorkloadDeployment` objects labeled with `topology.datum.net/city-code: `. Created lazily on first use, deleted when no deployment in the namespace targets that city. +One `PropagationPolicy` per canonical location per project namespace, using a `labelSelector` to match all `WorkloadDeployment` objects labeled with `topology.datum.net/location: `. Created lazily on first use, deleted when no deployment in the namespace targets that location. ### NSO in POP Cells diff --git a/go.mod b/go.mod index 9247c2b6..98ee6d80 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( // predates the Prepared condition this gate reads. Re-pin to a tagged // release once one carries it. go.datum.net/network-services-operator v0.26.1-0.20260821014231-aceb24b1b569 + go.miloapis.com/locations v0.0.0-20260825185141-507ac2cbd48c go.miloapis.com/milo v0.32.0 golang.org/x/crypto v0.54.0 golang.org/x/sync v0.22.0 diff --git a/go.sum b/go.sum index 7c7406eb..292b7aa6 100644 --- a/go.sum +++ b/go.sum @@ -338,6 +338,8 @@ go.datum.net/datumctl v0.17.1-0.20260710003126-296c2fcbbd67 h1:Mhgt688CeTh2hX7ql go.datum.net/datumctl v0.17.1-0.20260710003126-296c2fcbbd67/go.mod h1:6skEjcE7aT8VPf/HVamA/BB6Dc9IISA6c/DdYKhqWNc= go.datum.net/network-services-operator v0.26.1-0.20260821014231-aceb24b1b569 h1:14vajo15fGGEAmdystQEE261rFH4AqER9s1Vg7POUKo= go.datum.net/network-services-operator v0.26.1-0.20260821014231-aceb24b1b569/go.mod h1:A7JNOuc+e6j/KkUVCcZ7Z2odvf6JFMQlX0Zo1Awj2TY= +go.miloapis.com/locations v0.0.0-20260825185141-507ac2cbd48c h1:+BQirT3wYCgv7H2lEZtAH+dpMiWu9j/Wa+c8s4jJUIA= +go.miloapis.com/locations v0.0.0-20260825185141-507ac2cbd48c/go.mod h1:gzfAfHhSMwl/N68k/uSNYXOKK3IOBJCdXYaBgCE3gdE= go.miloapis.com/milo v0.32.0 h1:TkNIQu/37d+SEquLJ5+GmdisSl+K2RT7eEC4idg6RIs= go.miloapis.com/milo v0.32.0/go.mod h1:GKK3afjCwshfZfvhjNe1wp/H45z4m7x5oG/8xbSgU1M= go.miloapis.com/service-catalog v0.4.0 h1:LvO1WCHMCoFokpS5igWMP8kyqly9gUFQmQj5IGhwuKs= diff --git a/internal/cmd/compute/deploy/deploy.go b/internal/cmd/compute/deploy/deploy.go index 2e84851a..3da3fbe5 100644 --- a/internal/cmd/compute/deploy/deploy.go +++ b/internal/cmd/compute/deploy/deploy.go @@ -13,6 +13,7 @@ import ( "golang.org/x/term" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilyaml "k8s.io/apimachinery/pkg/util/yaml" @@ -22,17 +23,20 @@ import ( "go.datum.net/compute/internal/cmd/compute/util" "go.datum.net/compute/internal/cmd/compute/watch" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" ) type options struct { - image string - instanceType string - cities []string - min int32 - port int32 - file string - yes bool + image string + instanceType string + locations []string + cities []string + allMatchingLocations bool + min int32 + port int32 + file string + yes bool } func Command() *cobra.Command { @@ -41,13 +45,13 @@ func Command() *cobra.Command { cmd := &cobra.Command{ Use: "deploy [workload-name]", Short: "Deploy or update a workload", - Long: `Deploy a container image as a workload across one or more cities. + Long: `Deploy a container image as a workload across one or more locations. If no arguments are given, an interactive prompt guides you through the deployment. Use -f to apply a workload manifest file instead of flags.`, Args: cobra.MaximumNArgs(1), Example: ` # Deploy with flags - datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --city=DFW,IAD --min=2 --port=8080 + datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --location=us-east-1,eu-west-1 --min=2 --port=8080 # Interactive mode datumctl compute deploy @@ -62,11 +66,15 @@ Use -f to apply a workload manifest file instead of flags.`, cmd.Flags().StringVar(&opts.image, "image", "", "Container image to deploy (e.g. ghcr.io/acme/api:1.4.2)") cmd.Flags().StringVar(&opts.instanceType, "instance-type", "datumcloud/d1-standard-2", "Instance type (e.g. datumcloud/d1-standard-2)") - cmd.Flags().StringSliceVar(&opts.cities, "city", nil, "One or more city codes to deploy to (e.g. DFW,IAD)") - cmd.Flags().Int32Var(&opts.min, "min", 1, "Minimum number of instances per city") + cmd.Flags().StringSliceVar(&opts.locations, "location", nil, "One or more locations to deploy to (e.g. us-east-1,eu-west-1)") + cmd.Flags().StringSliceVar(&opts.cities, "city", nil, "Resolve a city code to an available location (e.g. IAD)") + cmd.Flags().BoolVar(&opts.allMatchingLocations, "all-matching-locations", false, "Deploy to every location matching --city") + cmd.Flags().Int32Var(&opts.min, "min", 1, "Minimum number of instances per location") cmd.Flags().Int32Var(&opts.port, "port", 0, "Port to expose on the workload (optional)") cmd.Flags().StringVarP(&opts.file, "file", "f", "", "Path to a workload manifest file") cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false, "Skip confirmation prompts") + _ = cmd.RegisterFlagCompletionFunc("location", util.CompleteLocations) + _ = cmd.RegisterFlagCompletionFunc("city", util.CompleteCityCodes) return cmd } @@ -93,8 +101,11 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err if opts.image == "" { return fmt.Errorf("--image is required") } - if len(opts.cities) == 0 { - return fmt.Errorf("--city is required (e.g. --city=DFW,IAD)") + if len(opts.locations) == 0 && len(opts.cities) == 0 { + return fmt.Errorf("--location is required (e.g. --location=us-east-1,eu-west-1); use --city to resolve a city code") + } + if len(opts.locations) > 0 && len(opts.cities) > 0 { + return fmt.Errorf("--location and --city are mutually exclusive") } instanceType := opts.instanceType if instanceType == "" { @@ -108,6 +119,14 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err ctx := context.Background() out := cmd.OutOrStdout() + locations := opts.locations + if len(opts.cities) > 0 { + locations, err = resolveCities(ctx, c, opts.cities, opts.allMatchingLocations) + if err != nil { + return err + } + fmt.Fprintf(out, "Resolved --city to locations: %s\n", strings.Join(locations, ", ")) + } if err := ensureNetwork(ctx, cmd, c, "default", project, opts); err != nil { return err @@ -143,10 +162,14 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err } } - // All cities go into one "default" placement. + locationRefs := make([]locationsv1alpha1.LocationReference, 0, len(locations)) + for _, name := range locations { + locationRefs = append(locationRefs, locationsv1alpha1.LocationReference{Name: name}) + } + // All locations go into one "default" placement. placement := computev1alpha.WorkloadPlacement{ Name: "default", - CityCodes: opts.cities, + Locations: locationRefs, ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: opts.min, InstanceManagementPolicy: computev1alpha.OrderedReadyInstanceManagementPolicyType, @@ -175,8 +198,8 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err Placements: []computev1alpha.WorkloadPlacement{placement}, } - fmt.Fprintf(out, " Placement \"default\": cities=[%s], min=%d\n", - strings.Join(opts.cities, ", "), opts.min) + fmt.Fprintf(out, " Placement \"default\": locations=[%s], min=%d\n", + strings.Join(locations, ", "), opts.min) // Prompt unless --yes or non-interactive. if !opts.yes && term.IsTerminal(int(os.Stdin.Fd())) { @@ -411,8 +434,12 @@ func manifestDiff(existing, desired computev1alpha.Workload) []string { name, op.ScaleSettings.MinReplicas, np.ScaleSettings.MinReplicas)) } } else { - lines = append(lines, fmt.Sprintf(" + new placement %q: cities=[%s]", - name, strings.Join(np.CityCodes, ", "))) + locationNames := make([]string, 0, len(np.Locations)) + for _, ref := range np.Locations { + locationNames = append(locationNames, ref.Name) + } + lines = append(lines, fmt.Sprintf(" + new placement %q: locations=[%s]", + name, strings.Join(locationNames, ", "))) } } for name := range oldPlacements { @@ -423,3 +450,29 @@ func manifestDiff(existing, desired computev1alpha.Workload) []string { return lines } + +func resolveCities(ctx context.Context, c client.Client, cities []string, all bool) ([]string, error) { + var list locationsv1alpha1.LocationList + if err := c.List(ctx, &list); err != nil { + return nil, fmt.Errorf("listing project locations: %w", err) + } + var resolved []string + for _, city := range cities { + var matches []string + for _, location := range list.Items { + ready := apimeta.FindStatusCondition(location.Status.Conditions, locationsv1alpha1.LocationConditionReady) + if location.Spec.Topology[locationsv1alpha1.TopologyCityCodeKey] != city || ready == nil || ready.Status != metav1.ConditionTrue { + continue + } + matches = append(matches, location.Name) + } + if len(matches) == 0 { + return nil, fmt.Errorf("city %q has no Ready locations; run 'datumctl get locations' to see available locations", city) + } + if len(matches) > 1 && !all { + return nil, fmt.Errorf("city %q matches multiple Ready locations (%s); use --location to choose one or --all-matching-locations", city, strings.Join(matches, ", ")) + } + resolved = append(resolved, matches...) + } + return resolved, nil +} diff --git a/internal/cmd/compute/destroy/destroy.go b/internal/cmd/compute/destroy/destroy.go index e95e02c3..bc5b1b00 100644 --- a/internal/cmd/compute/destroy/destroy.go +++ b/internal/cmd/compute/destroy/destroy.go @@ -54,18 +54,20 @@ func runDestroy(cmd *cobra.Command, args []string, yes bool) error { } // Summarize placements. - var allCityCodes []string + var allLocations []string var totalMin int32 for _, p := range workload.Spec.Placements { - allCityCodes = append(allCityCodes, p.CityCodes...) + for _, ref := range p.Locations { + allLocations = append(allLocations, ref.Name) + } totalMin += p.ScaleSettings.MinReplicas } out := cmd.OutOrStdout() - fmt.Fprintf(out, "Workload: %s\nPlacements: %d Cities: %s\nMin replicas: %d\n\n", + fmt.Fprintf(out, "Workload: %s\nPlacements: %d Locations: %s\nMin replicas: %d\n\n", workloadName, len(workload.Spec.Placements), - strings.Join(allCityCodes, ", "), + strings.Join(allLocations, ", "), totalMin, ) diff --git a/internal/cmd/compute/instances/instances.go b/internal/cmd/compute/instances/instances.go index 6b65de00..0faecdc9 100644 --- a/internal/cmd/compute/instances/instances.go +++ b/internal/cmd/compute/instances/instances.go @@ -20,7 +20,7 @@ import ( type listOptions struct { workload string - city string + location string } func Command() *cobra.Command { @@ -37,8 +37,8 @@ Use the describe subcommand for full details on a single instance.`, # Filter by workload datumctl compute instances --workload=api - # Filter by city - datumctl compute instances --city=DFW + # Filter by location + datumctl compute instances --location=us-east-1 # Machine-readable output datumctl compute instances -o json @@ -51,12 +51,12 @@ Use the describe subcommand for full details on a single instance.`, } cmd.Flags().StringVar(&opts.workload, "workload", "", "Filter instances to a specific workload") - cmd.Flags().StringVar(&opts.city, "city", "", "Filter instances to a specific city") + cmd.Flags().StringVar(&opts.location, "location", "", "Filter instances to a specific location") cmd.Flags().StringP("output", "o", "table", "Output format: table, wide, json, yaml") cmd.Flags().Bool("no-headers", false, "Omit the table header row (table and wide only)") _ = cmd.RegisterFlagCompletionFunc("workload", util.CompleteWorkloadNames) - _ = cmd.RegisterFlagCompletionFunc("city", util.CompleteCityCodes) + _ = cmd.RegisterFlagCompletionFunc("location", util.CompleteLocations) _ = cmd.RegisterFlagCompletionFunc("output", util.CompleteOutputFormats("table", "wide", "json", "yaml")) cmd.AddCommand(describeCommand()) @@ -67,7 +67,7 @@ Use the describe subcommand for full details on a single instance.`, type instanceRow struct { name string workload string - city string + location string internalIP string runtimeKind string // "sandbox" or "vm" instType string @@ -111,7 +111,7 @@ func runList(cmd *cobra.Command, opts *listOptions) error { return fmt.Errorf("listing instances: %w", err) } - // JSON/YAML: emit raw API resource and return early (before city filter). + // JSON/YAML: emit raw API resource and return early (before location filter). switch util.OutputFormat(outputFlag) { case util.OutputJSON: return util.PrintJSON(cmd.OutOrStdout(), &instList) @@ -148,7 +148,7 @@ func runList(cmd *cobra.Command, opts *listOptions) error { for _, inst := range instList.Items { wlUID := inst.Labels[computev1alpha.WorkloadUIDLabel] - city := "unknown" + location := "unknown" wlName := workloadMap[wlUID] if wlName == "" { wlName = "orphaned" @@ -157,12 +157,11 @@ func runList(cmd *cobra.Command, opts *listOptions) error { // Prefer self-describing labels stamped at creation time (fast path — // no join needed). Fall back to the WorkloadDeployment join for older // instances that predate the labels. - labelCity := inst.Labels[computev1alpha.CityCodeLabel] labelWLName := inst.Labels[computev1alpha.WorkloadNameLabel] - if labelCity != "" && labelWLName != "" { + if inst.Spec.Location != nil && labelWLName != "" { // Both labels present: no join needed. - city = labelCity + location = inst.Spec.Location.Name wlName = labelWLName } else { // At least one label absent — fall back to WorkloadDeployment lookup. @@ -174,11 +173,7 @@ func runList(cmd *cobra.Command, opts *listOptions) error { depName = wdNameFromInstanceName(inst.Name) } if dep, ok := deploymentMap[depName]; ok { - if labelCity != "" { - city = labelCity - } else { - city = dep.Spec.CityCode - } + location = dep.Spec.LocationRef.Name if labelWLName != "" { wlName = labelWLName } else if dep.Spec.WorkloadRef.Name != "" { @@ -186,8 +181,8 @@ func runList(cmd *cobra.Command, opts *listOptions) error { } } else { // Deployment not found — use whatever labels we do have. - if labelCity != "" { - city = labelCity + if inst.Spec.Location != nil { + location = inst.Spec.Location.Name } if labelWLName != "" { wlName = labelWLName @@ -195,8 +190,8 @@ func runList(cmd *cobra.Command, opts *listOptions) error { } } - // Client-side city filter. - if opts.city != "" && city != opts.city { + // Client-side location filter. + if opts.location != "" && location != opts.location { continue } @@ -216,7 +211,7 @@ func runList(cmd *cobra.Command, opts *listOptions) error { rows = append(rows, instanceRow{ name: inst.Name, workload: wlName, - city: city, + location: location, internalIP: intIP, runtimeKind: runtimeKind, instType: inst.Spec.Runtime.Resources.InstanceType, @@ -225,13 +220,13 @@ func runList(cmd *cobra.Command, opts *listOptions) error { }) } - // Sort: workload ASC, city ASC, name ASC. + // Sort: workload ASC, location ASC, name ASC. sort.Slice(rows, func(i, j int) bool { if rows[i].workload != rows[j].workload { return rows[i].workload < rows[j].workload } - if rows[i].city != rows[j].city { - return rows[i].city < rows[j].city + if rows[i].location != rows[j].location { + return rows[i].location < rows[j].location } return rows[i].name < rows[j].name }) @@ -246,18 +241,18 @@ func runList(cmd *cobra.Command, opts *listOptions) error { tw := util.NewTabWriter(out) if !noHeaders { if wide { - _, _ = fmt.Fprintf(tw, "NAME\tWORKLOAD\tCITY\tINTERNAL IP\tTYPE\tAGE\tSTATUS\tINSTANCE TYPE\n") + _, _ = fmt.Fprintf(tw, "NAME\tWORKLOAD\tLOCATION\tINTERNAL IP\tTYPE\tAGE\tSTATUS\tINSTANCE TYPE\n") } else { - _, _ = fmt.Fprintf(tw, "NAME\tWORKLOAD\tCITY\tINTERNAL IP\tTYPE\tAGE\tSTATUS\n") + _, _ = fmt.Fprintf(tw, "NAME\tWORKLOAD\tLOCATION\tINTERNAL IP\tTYPE\tAGE\tSTATUS\n") } } for _, r := range rows { if wide { _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - r.name, r.workload, r.city, r.internalIP, r.runtimeKind, r.age, r.status, r.instType) + r.name, r.workload, r.location, r.internalIP, r.runtimeKind, r.age, r.status, r.instType) } else { _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - r.name, r.workload, r.city, r.internalIP, r.runtimeKind, r.age, r.status) + r.name, r.workload, r.location, r.internalIP, r.runtimeKind, r.age, r.status) } } _ = tw.Flush() @@ -311,21 +306,20 @@ func runDescribe(cmd *cobra.Command, args []string) error { return fmt.Errorf("getting instance: %w", err) } - // Resolve CITY, WORKLOAD, and PLACEMENT. Prefer self-describing labels + // Resolve LOCATION, WORKLOAD, and PLACEMENT. // stamped at creation time (no join needed). Fall back to a // WorkloadDeployment Get when any of the labels are absent, so that older // instances that predate the stamp still resolve correctly. workloadName := "orphaned" - city := "unknown" + location := "unknown" placementName := "" - labelCity := inst.Labels[computev1alpha.CityCodeLabel] labelWLName := inst.Labels[computev1alpha.WorkloadNameLabel] labelPlacement := inst.Labels[computev1alpha.PlacementNameLabel] - if labelCity != "" && labelWLName != "" && labelPlacement != "" { + if inst.Spec.Location != nil && labelWLName != "" && labelPlacement != "" { // All three labels present: no join needed. - city = labelCity + location = inst.Spec.Location.Name workloadName = labelWLName placementName = labelPlacement } else { @@ -339,11 +333,7 @@ func runDescribe(cmd *cobra.Command, args []string) error { if depName != "" { var dep computev1alpha.WorkloadDeployment if err := c.Get(ctx, types.NamespacedName{Namespace: util.ResourceNamespace, Name: depName}, &dep); err == nil { - if labelCity != "" { - city = labelCity - } else { - city = dep.Spec.CityCode - } + location = dep.Spec.LocationRef.Name if labelPlacement != "" { placementName = labelPlacement } else { @@ -356,8 +346,8 @@ func runDescribe(cmd *cobra.Command, args []string) error { } } else { // WD Get failed — use whatever labels we do have. - if labelCity != "" { - city = labelCity + if inst.Spec.Location != nil { + location = inst.Spec.Location.Name } if labelWLName != "" { workloadName = labelWLName @@ -379,7 +369,7 @@ func runDescribe(cmd *cobra.Command, args []string) error { if placementName != "" { fmt.Fprintf(out, "%-14s %s\n", "Placement", placementName) } - fmt.Fprintf(out, "%-14s %s\n", "City", city) + fmt.Fprintf(out, "%-14s %s\n", "Location", location) fmt.Fprintf(out, "%-14s %s\n", "Age", util.RelativeAgeVerbose(inst.CreationTimestamp)) fmt.Fprintf(out, "%-14s %s\n", "Status", status) if detail != "" { diff --git a/internal/cmd/compute/restart/restart.go b/internal/cmd/compute/restart/restart.go index 9fafd618..e15380d9 100644 --- a/internal/cmd/compute/restart/restart.go +++ b/internal/cmd/compute/restart/restart.go @@ -16,24 +16,25 @@ import ( ) func Command() *cobra.Command { - var city string + var location string cmd := &cobra.Command{ Use: "restart ", Short: "Trigger a rolling restart of a workload", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runRestart(cmd, args, city) + return runRestart(cmd, args, location) }, ValidArgsFunction: util.CompleteWorkloadNames, } - cmd.Flags().StringVar(&city, "city", "", "Restart only instances in a specific city") + cmd.Flags().StringVar(&location, "location", "", "Restart only instances in a specific location") + _ = cmd.RegisterFlagCompletionFunc("location", util.CompleteLocations) return cmd } -func runRestart(cmd *cobra.Command, args []string, city string) error { +func runRestart(cmd *cobra.Command, args []string, location string) error { project := util.ProjectFromCmd(cmd) c, err := util.NewClient(project) @@ -55,7 +56,7 @@ func runRestart(cmd *cobra.Command, args []string, city string) error { restartedAt := time.Now().UTC().Format(time.RFC3339) out := cmd.OutOrStdout() - if city == "" { + if location == "" { // Restart all placements by annotating the workload template. if workload.Spec.Template.Annotations == nil { workload.Spec.Template.Annotations = make(map[string]string) @@ -73,7 +74,7 @@ func runRestart(cmd *cobra.Command, args []string, city string) error { return nil } - // Restart only deployments in the given city. + // Restart only deployments in the given location. selector := labels.SelectorFromSet(labels.Set{ computev1alpha.WorkloadUIDLabel: string(workload.UID), }) @@ -87,13 +88,13 @@ func runRestart(cmd *cobra.Command, args []string, city string) error { var matched []computev1alpha.WorkloadDeployment for _, d := range deployList.Items { - if d.Spec.CityCode == city { + if d.Spec.LocationRef.Name == location { matched = append(matched, d) } } if len(matched) == 0 { - return fmt.Errorf("no deployment found for workload %q in city %q", workloadName, city) + return fmt.Errorf("no deployment found for workload %q in location %q", workloadName, location) } for i := range matched { @@ -103,13 +104,13 @@ func runRestart(cmd *cobra.Command, args []string, city string) error { matched[i].Spec.Template.Annotations[computev1alpha.RestartedAtAnnotation] = restartedAt if err := c.Update(ctx, &matched[i]); err != nil { - return fmt.Errorf("updating deployment in %s: %w", city, err) + return fmt.Errorf("updating deployment in %s: %w", location, err) } } fmt.Fprintf(out, "Restarting workload %q in %s — rolling restart initiated.\nRun 'datumctl compute rollout %s' to watch progress.\n", - workloadName, city, workloadName, + workloadName, location, workloadName, ) return nil } diff --git a/internal/cmd/compute/scale/scale.go b/internal/cmd/compute/scale/scale.go index 1ce704ee..a7e2db74 100644 --- a/internal/cmd/compute/scale/scale.go +++ b/internal/cmd/compute/scale/scale.go @@ -26,7 +26,7 @@ func Command() *cobra.Command { ValidArgsFunction: util.CompleteWorkloadNames, } - cmd.Flags().Int32Var(&min, "min", 0, "Minimum number of instances per city") + cmd.Flags().Int32Var(&min, "min", 0, "Minimum number of instances per location") _ = cmd.MarkFlagRequired("min") return cmd diff --git a/internal/cmd/compute/util/client.go b/internal/cmd/compute/util/client.go index feafc7ab..463a71e2 100644 --- a/internal/cmd/compute/util/client.go +++ b/internal/cmd/compute/util/client.go @@ -7,6 +7,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/datumctl/plugin" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" quotav1alpha1 "go.miloapis.com/milo/pkg/apis/quota/v1alpha1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" @@ -52,6 +53,9 @@ func NewClient(project string) (client.Client, error) { if err := networkingv1alpha.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("registering networking scheme: %w", err) } + if err := locationsv1alpha1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("registering locations scheme: %w", err) + } if err := quotav1alpha1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("registering quota scheme: %w", err) } diff --git a/internal/cmd/compute/util/completion.go b/internal/cmd/compute/util/completion.go index c6dcb3fb..7eed2b2f 100644 --- a/internal/cmd/compute/util/completion.go +++ b/internal/cmd/compute/util/completion.go @@ -2,12 +2,14 @@ package util import ( "context" + "sort" "github.com/spf13/cobra" "sigs.k8s.io/controller-runtime/pkg/client" computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/datumctl/plugin" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) // CompleteInstanceNames is a ValidArgsFunction that lists instance names from the API. @@ -34,31 +36,52 @@ func CompleteInstanceNames(cmd *cobra.Command, args []string, _ string) ([]strin return names, cobra.ShellCompDirectiveNoFileComp } -// CompleteCityCodes is a ValidArgsFunction that returns unique city codes from -// all WorkloadDeployments in the project. -func CompleteCityCodes(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { - project := ProjectFromCmd(cmd) - c, err := NewClient(project) - if err != nil { +// CompleteLocations returns the projected locations available to the project. +func CompleteLocations(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { + list, ok := projectedLocations(cmd) + if !ok { return nil, cobra.ShellCompDirectiveNoFileComp } + names := make([]string, 0, len(list.Items)) + for _, location := range list.Items { + names = append(names, location.Name) + } + sort.Strings(names) + return names, cobra.ShellCompDirectiveNoFileComp +} - var list computev1alpha.WorkloadDeploymentList - if err := c.List(context.Background(), &list, client.InNamespace(ResourceNamespace)); err != nil { +// CompleteCityCodes returns unique city codes from projected Locations. +func CompleteCityCodes(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { + list, ok := projectedLocations(cmd) + if !ok { return nil, cobra.ShellCompDirectiveNoFileComp } - seen := make(map[string]bool) var codes []string - for _, d := range list.Items { - if !seen[d.Spec.CityCode] { - seen[d.Spec.CityCode] = true - codes = append(codes, d.Spec.CityCode) + for _, location := range list.Items { + code := location.Spec.Topology[locationsv1alpha1.TopologyCityCodeKey] + if code != "" && !seen[code] { + seen[code] = true + codes = append(codes, code) } } + sort.Strings(codes) return codes, cobra.ShellCompDirectiveNoFileComp } +func projectedLocations(cmd *cobra.Command) (locationsv1alpha1.LocationList, bool) { + project := ProjectFromCmd(cmd) + c, err := NewClient(project) + if err != nil { + return locationsv1alpha1.LocationList{}, false + } + var list locationsv1alpha1.LocationList + if err := c.List(context.Background(), &list); err != nil { + return locationsv1alpha1.LocationList{}, false + } + return list, true +} + // CompleteOutputFormats returns a ValidArgsFunction that completes -o/--output // to the given allowed values. func CompleteOutputFormats(allowed ...string) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { diff --git a/internal/cmd/compute/watch/watch.go b/internal/cmd/compute/watch/watch.go index 13b2651d..f1d55b7d 100644 --- a/internal/cmd/compute/watch/watch.go +++ b/internal/cmd/compute/watch/watch.go @@ -26,7 +26,7 @@ const ( type deploymentState struct { placement string - city string + location string desired int32 ready int32 current int32 @@ -35,7 +35,7 @@ type deploymentState struct { } // Rollout polls WorkloadDeployment objects for the given workload UID, printing -// per-city progress rows as state changes. It returns when all deployments +// per-location progress rows as state changes. It returns when all deployments // reach Done, or when ctx is cancelled (Ctrl-C detach). func Rollout(ctx context.Context, c client.Client, out io.Writer, project string, workloadUID types.UID) error { start := time.Now() @@ -76,7 +76,7 @@ func Rollout(ctx context.Context, c client.Client, out io.Writer, project string } if !headerPrinted { - _, _ = fmt.Fprintln(out, "\n PLACEMENT\tCITY\tUPDATED\tREADY\tOLD\tPHASE") + _, _ = fmt.Fprintln(out, "\n PLACEMENT\tLOCATION\tUPDATED\tREADY\tOLD\tPHASE") headerPrinted = true } @@ -109,7 +109,7 @@ func processDeployments( ) bool { allDone := true for _, d := range deployments { - key := d.Spec.CityCode + key := d.Spec.LocationRef.Name prev, exists := states[key] desired := resolveDesired(d) @@ -143,7 +143,7 @@ func updateDeploymentState( ) deploymentPhase { st := &deploymentState{ placement: d.Spec.PlacementName, - city: d.Spec.CityCode, + location: d.Spec.LocationRef.Name, desired: desired, ready: ready, current: current, @@ -190,7 +190,7 @@ func printDeploymentRow( _, _ = fmt.Fprintf(tw, " %s\t%s\t%d\t%d\t%d\t%s\n", d.Spec.PlacementName, - d.Spec.CityCode, + d.Spec.LocationRef.Name, current, ready, old, diff --git a/internal/cmd/compute/watch/watch_test.go b/internal/cmd/compute/watch/watch_test.go index 33734954..b90a4c48 100644 --- a/internal/cmd/compute/watch/watch_test.go +++ b/internal/cmd/compute/watch/watch_test.go @@ -142,7 +142,7 @@ func TestUpdateDeploymentState(t *testing.T) { makeDeployment := func() computev1alpha.WorkloadDeployment { var d computev1alpha.WorkloadDeployment d.Spec.PlacementName = "default" - d.Spec.CityCode = key + d.Spec.LocationRef.Name = key return d } diff --git a/internal/cmd/compute/workloads/workloads.go b/internal/cmd/compute/workloads/workloads.go index e01cc480..17d7f6e0 100644 --- a/internal/cmd/compute/workloads/workloads.go +++ b/internal/cmd/compute/workloads/workloads.go @@ -21,7 +21,7 @@ func Command() *cobra.Command { cmd := &cobra.Command{ Use: "workloads", Short: "List or inspect workloads", - Long: `List all workloads in the project, optionally filtered by health or city. + Long: `List all workloads in the project, optionally filtered by health or location. Use the describe subcommand for a unified config + health view of a single workload.`, Example: ` # List all workloads datumctl compute workloads @@ -29,8 +29,8 @@ Use the describe subcommand for a unified config + health view of a single workl # Filter by health datumctl compute workloads --health=degraded - # Filter by city - datumctl compute workloads --city=DFW + # Filter by location + datumctl compute workloads --location=us-east-1 # Machine-readable output datumctl compute workloads -o json @@ -44,12 +44,12 @@ Use the describe subcommand for a unified config + health view of a single workl // List flags. cmd.Flags().String("health", "", "Filter by health: available, degraded, progressing, unknown") - cmd.Flags().String("city", "", "Filter to workloads with a placement in this city") + cmd.Flags().String("location", "", "Filter to workloads with a placement in this location") cmd.Flags().StringP("output", "o", "table", "Output format: table, wide, json, yaml") cmd.Flags().Bool("no-headers", false, "Omit the table header row (table and wide only)") _ = cmd.RegisterFlagCompletionFunc("health", util.CompleteOutputFormats("available", "degraded", "progressing", "unknown")) - _ = cmd.RegisterFlagCompletionFunc("city", util.CompleteCityCodes) + _ = cmd.RegisterFlagCompletionFunc("location", util.CompleteLocations) _ = cmd.RegisterFlagCompletionFunc("output", util.CompleteOutputFormats("table", "wide", "json", "yaml")) cmd.AddCommand(describeCommand()) @@ -68,7 +68,7 @@ func runList(cmd *cobra.Command, _ []string) error { outputFlag, _ := cmd.Flags().GetString("output") healthFilter, _ := cmd.Flags().GetString("health") - cityFilter, _ := cmd.Flags().GetString("city") + locationFilter, _ := cmd.Flags().GetString("location") noHeaders, _ := cmd.Flags().GetBool("no-headers") c, err := util.NewClient(project) @@ -102,14 +102,13 @@ func runList(cmd *cobra.Command, _ []string) error { deploysByWorkload[wUID] = append(deploysByWorkload[wUID], d) } - // City filter: collect the set of workload UIDs that have a deployment in - // the requested city code. - cityFilteredUIDs := map[string]bool{} - if cityFilter != "" { + // Location filter: collect workload UIDs with a deployment in the requested location. + locationFilteredUIDs := map[string]bool{} + if locationFilter != "" { for _, d := range deployList.Items { - if d.Spec.CityCode == cityFilter { + if d.Spec.LocationRef.Name == locationFilter { wUID := d.Labels[computev1alpha.WorkloadUIDLabel] - cityFilteredUIDs[wUID] = true + locationFilteredUIDs[wUID] = true } } } @@ -133,8 +132,8 @@ func runList(cmd *cobra.Command, _ []string) error { for _, wl := range wlList.Items { wUID := string(wl.UID) - // City filter. - if cityFilter != "" && !cityFilteredUIDs[wUID] { + // Location filter. + if locationFilter != "" && !locationFilteredUIDs[wUID] { continue } @@ -213,12 +212,12 @@ func runList(cmd *cobra.Command, _ []string) error { if len(rows) == 0 { if healthFilter != "" { fmt.Fprintf(out, "No workloads in project %s match health=%s.\n", project, healthFilter) - } else if cityFilter != "" { - fmt.Fprintf(out, "No workloads in project %s have a placement in city %s.\n", project, cityFilter) + } else if locationFilter != "" { + fmt.Fprintf(out, "No workloads in project %s have a placement in location %s.\n", project, locationFilter) } else { fmt.Fprintf(out, "No workloads found in project %s.\n\n", project) fmt.Fprintf(out, "Get started:\n") - fmt.Fprintf(out, " datumctl compute deploy --name=api --image=ghcr.io/acme/api:v1.0.0 --city=DFW\n") + fmt.Fprintf(out, " datumctl compute deploy api --image=ghcr.io/acme/api:v1.0.0 --location=us-east-1\n") } return nil } @@ -278,7 +277,7 @@ func describeCommand() *cobra.Command { Use: "describe ", Short: "Show config and health for a single workload", Long: `Display a unified view of workload configuration (container spec, scale settings) -and runtime health (per-city ready/desired counts). Replaces 'datumctl compute status'.`, +and runtime health (per-location ready/desired counts). Replaces 'datumctl compute status'.`, Args: cobra.ExactArgs(1), Example: ` datumctl compute workloads describe api`, RunE: func(cmd *cobra.Command, args []string) error { @@ -377,11 +376,14 @@ func runDescribe(cmd *cobra.Command, args []string) error { if p.ScaleSettings.MaxReplicas != nil { maxStr = fmt.Sprintf("%d", *p.ScaleSettings.MaxReplicas) } - cityCodes := strings.Join(p.CityCodes, ", ") - fmt.Fprintf(out, " %-10s cities: %-24s scale: %d..%s\n", - p.Name, cityCodes, p.ScaleSettings.MinReplicas, maxStr) + locationNames := make([]string, 0, len(p.Locations)) + for _, ref := range p.Locations { + locationNames = append(locationNames, ref.Name) + } + fmt.Fprintf(out, " %-10s locations: %-24s scale: %d..%s\n", + p.Name, strings.Join(locationNames, ", "), p.ScaleSettings.MinReplicas, maxStr) - // Per-city lines from deployments. + // Per-location lines from deployments. for _, d := range deplsByPlacement[p.Name] { readyStr := fmt.Sprintf("%d/%d", d.Status.ReadyReplicas, d.Status.DesiredReplicas) annotation := "" @@ -390,9 +392,9 @@ func runDescribe(cmd *cobra.Command, args []string) error { annotation = degradedAnnotation(ctx, c, d) } if annotation != "" { - fmt.Fprintf(out, " %-8s ready: %-10s %s\n", d.Spec.CityCode, readyStr, annotation) + fmt.Fprintf(out, " %-8s ready: %-10s %s\n", d.Spec.LocationRef.Name, readyStr, annotation) } else { - fmt.Fprintf(out, " %-8s ready: %s\n", d.Spec.CityCode, readyStr) + fmt.Fprintf(out, " %-8s ready: %s\n", d.Spec.LocationRef.Name, readyStr) } } } @@ -442,7 +444,7 @@ func runDescribe(cmd *cobra.Command, args []string) error { return nil } -// degradedAnnotation returns a short annotation for a per-city line when the +// degradedAnnotation returns a short annotation for a per-location line when the // deployment is not fully ready. It reads the blocking reason+message from the // deployment's own Available condition, which the server rolls up from the // underlying instances. No per-instance fetch or reason branching needed. diff --git a/internal/controller/indexers.go b/internal/controller/indexers.go index 9816273b..69f855fb 100644 --- a/internal/controller/indexers.go +++ b/internal/controller/indexers.go @@ -81,12 +81,12 @@ func deploymentWorkloadUIDIndexFunc(o client.Object) []string { func deploymentLocationIndexFunc(o client.Object) []string { deployment := o.(*computev1alpha.WorkloadDeployment) - if deployment.Status.Location == nil { + if deployment.Spec.LocationRef.Name == "" { return nil } // Locations are cluster-scoped, so the name alone identifies one. - return []string{deployment.Status.Location.Name} + return []string{deployment.Spec.LocationRef.Name} } func addWorkloadIndexers(ctx context.Context, mgr mcmanager.Manager) error { diff --git a/internal/controller/instance_controller.go b/internal/controller/instance_controller.go index e637ef44..9d6d4e3f 100644 --- a/internal/controller/instance_controller.go +++ b/internal/controller/instance_controller.go @@ -1278,7 +1278,7 @@ func (r *InstanceReconciler) writeBackToUpstream(ctx context.Context, instance * computev1alpha.WorkloadDeploymentUIDLabel, computev1alpha.InstanceIndexLabel, computev1alpha.WorkloadDeploymentNameLabel, - computev1alpha.CityCodeLabel, + computev1alpha.LocationLabel, computev1alpha.WorkloadNameLabel, computev1alpha.PlacementNameLabel, } { @@ -1322,7 +1322,7 @@ func (r *InstanceReconciler) writeBackToUpstream(ctx context.Context, instance * computev1alpha.WorkloadDeploymentUIDLabel: instance.Labels[computev1alpha.WorkloadDeploymentUIDLabel], computev1alpha.InstanceIndexLabel: instance.Labels[computev1alpha.InstanceIndexLabel], computev1alpha.WorkloadDeploymentNameLabel: instance.Labels[computev1alpha.WorkloadDeploymentNameLabel], - computev1alpha.CityCodeLabel: instance.Labels[computev1alpha.CityCodeLabel], + computev1alpha.LocationLabel: instance.Labels[computev1alpha.LocationLabel], computev1alpha.WorkloadNameLabel: instance.Labels[computev1alpha.WorkloadNameLabel], computev1alpha.PlacementNameLabel: instance.Labels[computev1alpha.PlacementNameLabel], }, diff --git a/internal/controller/instance_controller_test.go b/internal/controller/instance_controller_test.go index 4b11fc12..2aa2d5bf 100644 --- a/internal/controller/instance_controller_test.go +++ b/internal/controller/instance_controller_test.go @@ -30,6 +30,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.datum.net/compute/internal/controller/instancecontrol" "go.datum.net/compute/internal/quota" @@ -74,6 +75,7 @@ func newTestScheme(t *testing.T) *runtime.Scheme { require.NoError(t, computev1alpha.AddToScheme(s)) require.NoError(t, quotav1alpha1.AddToScheme(s)) require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, locationsv1alpha1.AddToScheme(s)) return s } diff --git a/internal/controller/instance_projector_test.go b/internal/controller/instance_projector_test.go index 0ae15e1b..c960ebdb 100644 --- a/internal/controller/instance_projector_test.go +++ b/internal/controller/instance_projector_test.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" computev1alpha "go.datum.net/compute/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.miloapis.com/milo/pkg/downstreamclient" ) @@ -92,7 +93,7 @@ func projTestWorkloadDeployment() *computev1alpha.WorkloadDeployment { UID: projTestWDUID, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: "LAX", + LocationRef: locationsv1alpha1.LocationReference{Name: "us-west-1"}, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: "my-workload"}, ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, diff --git a/internal/controller/instance_setup_test.go b/internal/controller/instance_setup_test.go index 7782adbd..44c94742 100644 --- a/internal/controller/instance_setup_test.go +++ b/internal/controller/instance_setup_test.go @@ -12,6 +12,8 @@ import ( "time" "github.com/stretchr/testify/require" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cluster" @@ -37,6 +39,22 @@ func startComputeEnvtest(t *testing.T) *rest.Config { env := &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "base", "crd", "bases")}, ErrorIfCRDPathMissing: true, + CRDs: []*apiextensionsv1.CustomResourceDefinition{{ + ObjectMeta: metav1.ObjectMeta{Name: "servinglocations.locations.miloapis.com"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "locations.miloapis.com", + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "servinglocations", Singular: "servinglocation", Kind: "ServingLocation", ListKind: "ServingLocationList", + }, + Scope: apiextensionsv1.ClusterScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: "v1alpha1", Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", XPreserveUnknownFields: new(true), + }}, + }}, + }, + }}, } // Respect KUBEBUILDER_ASSETS (set by `make test`); fall back to the assets // `setup-envtest` writes under bin/ for local `go test` runs. diff --git a/internal/controller/instance_writeback_test.go b/internal/controller/instance_writeback_test.go index f454e455..28cc3aec 100644 --- a/internal/controller/instance_writeback_test.go +++ b/internal/controller/instance_writeback_test.go @@ -36,7 +36,7 @@ const ( // The four self-describing labels. wbTestWDName = "my-workload-deployment" - wbTestCityCode = "DFW" + wbTestLocation = "us-east-1" wbTestWorkloadName = "my-workload" wbTestPlacement = "us-central" @@ -58,7 +58,7 @@ func wbTestCellInstance() *computev1alpha.Instance { computev1alpha.WorkloadDeploymentUIDLabel: wbTestWDUID, computev1alpha.InstanceIndexLabel: wbTestInstanceIndex, computev1alpha.WorkloadDeploymentNameLabel: wbTestWDName, - computev1alpha.CityCodeLabel: wbTestCityCode, + computev1alpha.LocationLabel: wbTestLocation, computev1alpha.WorkloadNameLabel: wbTestWorkloadName, computev1alpha.PlacementNameLabel: wbTestPlacement, }, @@ -314,7 +314,7 @@ func TestWriteBackToUpstream_MissingLinkingLabels_Error(t *testing.T) { computev1alpha.WorkloadDeploymentUIDLabel, computev1alpha.InstanceIndexLabel, computev1alpha.WorkloadDeploymentNameLabel, - computev1alpha.CityCodeLabel, + computev1alpha.LocationLabel, computev1alpha.WorkloadNameLabel, computev1alpha.PlacementNameLabel, } { @@ -412,7 +412,7 @@ func TestWriteBackToUpstream_MissingSelfDescribingLabel_Error(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), computev1alpha.WorkloadDeploymentNameLabel, "error must name the missing label") - assert.NotContains(t, err.Error(), computev1alpha.CityCodeLabel, + assert.NotContains(t, err.Error(), computev1alpha.LocationLabel, "a present label must not be reported missing") var created computev1alpha.Instance @@ -551,8 +551,8 @@ func TestWriteBackToUpstream_FourNewLabels_CreatePath(t *testing.T) { assert.Equal(t, wbTestWDName, created.Labels[computev1alpha.WorkloadDeploymentNameLabel], "WorkloadDeploymentNameLabel must propagate to Karmada object") - assert.Equal(t, wbTestCityCode, created.Labels[computev1alpha.CityCodeLabel], - "CityCodeLabel must propagate to Karmada object") + assert.Equal(t, wbTestLocation, created.Labels[computev1alpha.LocationLabel], + "LocationLabel must propagate to Karmada object") assert.Equal(t, wbTestWorkloadName, created.Labels[computev1alpha.WorkloadNameLabel], "WorkloadNameLabel must propagate to Karmada object") assert.Equal(t, wbTestPlacement, created.Labels[computev1alpha.PlacementNameLabel], @@ -604,8 +604,8 @@ func TestWriteBackToUpstream_FourNewLabels_UpdatePath(t *testing.T) { assert.Equal(t, wbTestWDName, updated.Labels[computev1alpha.WorkloadDeploymentNameLabel], "WorkloadDeploymentNameLabel must be set on update path") - assert.Equal(t, wbTestCityCode, updated.Labels[computev1alpha.CityCodeLabel], - "CityCodeLabel must be set on update path") + assert.Equal(t, wbTestLocation, updated.Labels[computev1alpha.LocationLabel], + "LocationLabel must be set on update path") assert.Equal(t, wbTestWorkloadName, updated.Labels[computev1alpha.WorkloadNameLabel], "WorkloadNameLabel must be set on update path") assert.Equal(t, wbTestPlacement, updated.Labels[computev1alpha.PlacementNameLabel], diff --git a/internal/controller/instancecontrol/stateful/stateful_control.go b/internal/controller/instancecontrol/stateful/stateful_control.go index 352983c3..77119c24 100644 --- a/internal/controller/instancecontrol/stateful/stateful_control.go +++ b/internal/controller/instancecontrol/stateful/stateful_control.go @@ -100,10 +100,7 @@ func (c *statefulControl) GetActions( }, Spec: deployment.Spec.Template.Spec, } - // Set Location best-effort: when Status.Location is nil (no matching - // Location object for the city code) Instance.Spec.Location stays nil and - // instance creation proceeds normally — this must not block scheduling. - desiredInstances[i].Spec.Location = deployment.Status.Location + desiredInstances[i].Spec.Location = &deployment.Spec.LocationRef // TODO(jreese) consider adding scheduling gates via mutating webhooks gates := []v1alpha.SchedulingGate{ @@ -255,7 +252,7 @@ func desiredControllerLabels(index int, deployment *v1alpha.WorkloadDeployment) v1alpha.WorkloadDeploymentUIDLabel: string(deployment.GetUID()), // Self-describing labels for routing, filtering, and observability. v1alpha.WorkloadDeploymentNameLabel: deployment.GetName(), - v1alpha.CityCodeLabel: deployment.Spec.CityCode, + v1alpha.LocationLabel: deployment.Spec.LocationRef.Name, v1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, v1alpha.PlacementNameLabel: deployment.Spec.PlacementName, // Scopes consumer-project cleanup: the consumer provider deletes diff --git a/internal/controller/instancecontrol/stateful/stateful_control_test.go b/internal/controller/instancecontrol/stateful/stateful_control_test.go index 393ab437..ce69beec 100644 --- a/internal/controller/instancecontrol/stateful/stateful_control_test.go +++ b/internal/controller/instancecontrol/stateful/stateful_control_test.go @@ -14,7 +14,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/utils/ptr" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.datum.net/compute/api/v1alpha" "go.datum.net/compute/internal/controller/instancecontrol" @@ -278,8 +278,8 @@ func TestInstanceLabels_FourNewLabelsStamped(t *testing.T) { assert.Equal(t, deployment.GetName(), instance.Labels[v1alpha.WorkloadDeploymentNameLabel], "WorkloadDeploymentNameLabel must equal deployment name") - assert.Equal(t, deployment.Spec.CityCode, instance.Labels[v1alpha.CityCodeLabel], - "CityCodeLabel must equal deployment.Spec.CityCode") + assert.Equal(t, deployment.Spec.LocationRef.Name, instance.Labels[v1alpha.LocationLabel], + "LocationLabel must equal deployment.Spec.LocationRef.Name") assert.Equal(t, deployment.Spec.WorkloadRef.Name, instance.Labels[v1alpha.WorkloadNameLabel], "WorkloadNameLabel must equal deployment.Spec.WorkloadRef.Name") assert.Equal(t, deployment.Spec.PlacementName, instance.Labels[v1alpha.PlacementNameLabel], @@ -322,24 +322,22 @@ func TestInstanceLabels_RefreshedOnRecreate(t *testing.T) { assert.Equal(t, deployment.GetName(), instance.Labels[v1alpha.WorkloadDeploymentNameLabel], "WorkloadDeploymentNameLabel must be set on the recreated instance") - assert.Equal(t, deployment.Spec.CityCode, instance.Labels[v1alpha.CityCodeLabel], - "CityCodeLabel must be set on the recreated instance") + assert.Equal(t, deployment.Spec.LocationRef.Name, instance.Labels[v1alpha.LocationLabel], + "LocationLabel must be set on the recreated instance") assert.Equal(t, deployment.Spec.WorkloadRef.Name, instance.Labels[v1alpha.WorkloadNameLabel], "WorkloadNameLabel must be set on the recreated instance") assert.Equal(t, deployment.Spec.PlacementName, instance.Labels[v1alpha.PlacementNameLabel], "PlacementNameLabel must be set on the recreated instance") } -// TestInstanceLocation_SetWhenDeploymentStatusLocationPresent verifies that when -// deployment.Status.Location is set, the new Instance receives it as Spec.Location. -func TestInstanceLocation_SetWhenDeploymentStatusLocationPresent(t *testing.T) { +// TestInstanceLocation_SetFromDeploymentSpec verifies that the deployment's +// requested location is copied to each new Instance. +func TestInstanceLocation_SetFromDeploymentSpec(t *testing.T) { ctx := context.Background() control := New() deployment := getWorkloadDeployment("test-location-set", 1) - deployment.Status.Location = &networkingv1alpha.LocationReference{ - Name: "loc-dfw-1", - } + deployment.Spec.LocationRef = locationsv1alpha1.LocationReference{Name: "loc-dfw-1"} var currentInstances []v1alpha.Instance actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) @@ -350,35 +348,10 @@ func TestInstanceLocation_SetWhenDeploymentStatusLocationPresent(t *testing.T) { instance, ok := actions[0].Object.(*v1alpha.Instance) assert.True(t, ok) assert.NotNil(t, instance.Spec.Location, - "Spec.Location must be set when deployment.Status.Location is non-nil") + "Spec.Location must be set from deployment.spec.locationRef") assert.Equal(t, "loc-dfw-1", instance.Spec.Location.Name) } -// TestInstanceLocation_NilWhenDeploymentStatusLocationAbsent verifies that when -// deployment.Status.Location is nil (no Location object matches the city code), -// instance creation still succeeds and Spec.Location remains nil — no regression -// on the "create instances regardless of Location" contract. -func TestInstanceLocation_NilWhenDeploymentStatusLocationAbsent(t *testing.T) { - ctx := context.Background() - control := New() - - deployment := getWorkloadDeployment("test-location-nil", 1) - // deployment.Status.Location is intentionally not set (nil) - - var currentInstances []v1alpha.Instance - actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) - - assert.NoError(t, err, "instance creation must succeed even when Status.Location is nil") - assert.Len(t, actions, 1, "exactly one create action must be produced") - - instance, ok := actions[0].Object.(*v1alpha.Instance) - assert.True(t, ok) - assert.Nil(t, instance.Spec.Location, - "Spec.Location must remain nil when deployment.Status.Location is not set") - assert.Equal(t, instancecontrol.ActionTypeCreate, actions[0].ActionType(), - "action must be a Create, proving instance creation is not gated on Location") -} - // TestLabelBackfill_NotReadyMatchingHash verifies that a not-Ready instance // with an unchanged template hash receives a PatchLabels action when it is // missing controller-managed labels. The action must not be a rollout recreate, @@ -445,7 +418,7 @@ func TestLabelBackfill_NotReadyMatchingHash(t *testing.T) { patched, ok := patchActions[0].Object.(*v1alpha.Instance) assert.True(t, ok) assert.Equal(t, deployment.GetName(), patched.Labels[v1alpha.WorkloadDeploymentNameLabel]) - assert.Equal(t, deployment.Spec.CityCode, patched.Labels[v1alpha.CityCodeLabel]) + assert.Equal(t, deployment.Spec.LocationRef.Name, patched.Labels[v1alpha.LocationLabel]) assert.Equal(t, deployment.Spec.WorkloadRef.Name, patched.Labels[v1alpha.WorkloadNameLabel]) assert.Equal(t, deployment.Spec.PlacementName, patched.Labels[v1alpha.PlacementNameLabel]) @@ -471,7 +444,7 @@ func TestLabelBackfill_Idempotent(t *testing.T) { v1alpha.WorkloadUIDLabel: string(deployment.Spec.WorkloadRef.UID), v1alpha.WorkloadDeploymentUIDLabel: string(deployment.GetUID()), v1alpha.WorkloadDeploymentNameLabel: deployment.GetName(), - v1alpha.CityCodeLabel: deployment.Spec.CityCode, + v1alpha.LocationLabel: deployment.Spec.LocationRef.Name, v1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, v1alpha.PlacementNameLabel: deployment.Spec.PlacementName, labelServiceKey: labelServiceValue, @@ -500,7 +473,7 @@ func TestLabelBackfill_ReadyInstanceCorrected(t *testing.T) { // Ready instance with matching hash but missing city-code label. instance := getInstanceForDeployment(deployment, 0) // Remove the city-code label to simulate drift. - delete(instance.Labels, v1alpha.CityCodeLabel) + delete(instance.Labels, v1alpha.LocationLabel) currentInstances := []v1alpha.Instance{*instance} actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) @@ -524,8 +497,8 @@ func TestLabelBackfill_ReadyInstanceCorrected(t *testing.T) { assert.Len(t, patchActions, 1, "PatchLabels action must be produced for the label-drifted ready instance") patched, ok := patchActions[0].Object.(*v1alpha.Instance) assert.True(t, ok) - assert.Equal(t, deployment.Spec.CityCode, patched.Labels[v1alpha.CityCodeLabel], - "city-code label must be corrected by the backfill") + assert.Equal(t, deployment.Spec.LocationRef.Name, patched.Labels[v1alpha.LocationLabel], + "location label must be corrected by the backfill") } // TestLabelBackfill_DoesNotAffectRollingUpdate verifies that a genuine template @@ -545,7 +518,7 @@ func TestLabelBackfill_DoesNotAffectRollingUpdate(t *testing.T) { v1alpha.WorkloadUIDLabel: string(deployment.Spec.WorkloadRef.UID), v1alpha.WorkloadDeploymentUIDLabel: string(deployment.GetUID()), v1alpha.WorkloadDeploymentNameLabel: deployment.GetName(), - v1alpha.CityCodeLabel: deployment.Spec.CityCode, + v1alpha.LocationLabel: deployment.Spec.LocationRef.Name, v1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, v1alpha.PlacementNameLabel: deployment.Spec.PlacementName, labelServiceKey: labelServiceValue, @@ -556,7 +529,7 @@ func TestLabelBackfill_DoesNotAffectRollingUpdate(t *testing.T) { v1alpha.WorkloadUIDLabel: string(deployment.Spec.WorkloadRef.UID), v1alpha.WorkloadDeploymentUIDLabel: string(deployment.GetUID()), v1alpha.WorkloadDeploymentNameLabel: deployment.GetName(), - v1alpha.CityCodeLabel: deployment.Spec.CityCode, + v1alpha.LocationLabel: deployment.Spec.LocationRef.Name, v1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, v1alpha.PlacementNameLabel: deployment.Spec.PlacementName, labelServiceKey: labelServiceValue, @@ -606,7 +579,7 @@ func getWorkloadDeployment(name string, minReplicas int32) *v1alpha.WorkloadDepl UID: "test-workload-uid", }, PlacementName: "test-placement", - CityCode: "DFW", + LocationRef: locationsv1alpha1.LocationReference{Name: "loc-dfw-1"}, ScaleSettings: v1alpha.HorizontalScaleSettings{ MinReplicas: minReplicas, InstanceManagementPolicy: v1alpha.OrderedReadyInstanceManagementPolicyType, @@ -637,7 +610,7 @@ func getInstanceForDeployment(deployment *v1alpha.WorkloadDeployment, ordinal in instance.Labels[v1alpha.WorkloadUIDLabel] = string(deployment.Spec.WorkloadRef.UID) instance.Labels[v1alpha.WorkloadDeploymentUIDLabel] = string(deployment.GetUID()) instance.Labels[v1alpha.WorkloadDeploymentNameLabel] = deployment.GetName() - instance.Labels[v1alpha.CityCodeLabel] = deployment.Spec.CityCode + instance.Labels[v1alpha.LocationLabel] = deployment.Spec.LocationRef.Name instance.Labels[v1alpha.WorkloadNameLabel] = deployment.Spec.WorkloadRef.Name instance.Labels[v1alpha.PlacementNameLabel] = deployment.Spec.PlacementName instance.Labels[labelServiceKey] = labelServiceValue diff --git a/internal/controller/networkinterfaceclaim.go b/internal/controller/networkinterfaceclaim.go index 7648a970..e6e1950d 100644 --- a/internal/controller/networkinterfaceclaim.go +++ b/internal/controller/networkinterfaceclaim.go @@ -223,7 +223,7 @@ func desiredNetworkInterfaceClaimLabels( candidates := map[string]string{ computev1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, computev1alpha.PlacementNameLabel: deployment.Spec.PlacementName, - computev1alpha.CityCodeLabel: deployment.Spec.CityCode, + computev1alpha.LocationLabel: deployment.Spec.LocationRef.Name, computev1alpha.InstanceIndexLabel: instance.Labels[computev1alpha.InstanceIndexLabel], } diff --git a/internal/controller/networkinterfaceclaim_controller_test.go b/internal/controller/networkinterfaceclaim_controller_test.go index 98d74d1d..4bee7e17 100644 --- a/internal/controller/networkinterfaceclaim_controller_test.go +++ b/internal/controller/networkinterfaceclaim_controller_test.go @@ -17,6 +17,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.datum.net/compute/internal/controller/instancecontrol" ) @@ -59,7 +60,7 @@ func newClaimTestDeployment() *computev1alpha.WorkloadDeployment { UID: "claim-test-wd-uid", }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: wdControllerTestCityCode, + LocationRef: locationsv1alpha1.LocationReference{Name: wdControllerTestLocation}, PlacementName: claimTestPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: claimTestWorkload}, }, @@ -602,7 +603,7 @@ func TestDesiredNetworkInterfaceClaimLabels(t *testing.T) { want: map[string]string{ computev1alpha.WorkloadNameLabel: claimTestWorkload, computev1alpha.PlacementNameLabel: claimTestPlacement, - computev1alpha.CityCodeLabel: wdControllerTestCityCode, + computev1alpha.LocationLabel: wdControllerTestLocation, computev1alpha.InstanceIndexLabel: "0", }, }, @@ -613,7 +614,7 @@ func TestDesiredNetworkInterfaceClaimLabels(t *testing.T) { }, want: map[string]string{ computev1alpha.WorkloadNameLabel: claimTestWorkload, - computev1alpha.CityCodeLabel: wdControllerTestCityCode, + computev1alpha.LocationLabel: wdControllerTestLocation, computev1alpha.InstanceIndexLabel: "0", }, }, @@ -625,7 +626,7 @@ func TestDesiredNetworkInterfaceClaimLabels(t *testing.T) { want: map[string]string{ computev1alpha.WorkloadNameLabel: claimTestWorkload, computev1alpha.PlacementNameLabel: claimTestPlacement, - computev1alpha.CityCodeLabel: wdControllerTestCityCode, + computev1alpha.LocationLabel: wdControllerTestLocation, }, }, } @@ -658,7 +659,7 @@ func TestReconcileNetworkInterfaceClaims_Labels(t *testing.T) { wantLabels := map[string]string{ computev1alpha.WorkloadNameLabel: claimTestWorkload, computev1alpha.PlacementNameLabel: claimTestPlacement, - computev1alpha.CityCodeLabel: wdControllerTestCityCode, + computev1alpha.LocationLabel: wdControllerTestLocation, computev1alpha.InstanceIndexLabel: "0", } @@ -699,7 +700,7 @@ func TestReconcileNetworkInterfaceClaims_Labels(t *testing.T) { existing: map[string]string{ computev1alpha.WorkloadNameLabel: "a-workload-renamed-since", computev1alpha.PlacementNameLabel: claimTestPlacement, - computev1alpha.CityCodeLabel: wdControllerTestCityCode, + computev1alpha.LocationLabel: wdControllerTestLocation, computev1alpha.InstanceIndexLabel: "0", }, expectExisted: true, diff --git a/internal/controller/referenceddata_controller.go b/internal/controller/referenceddata_controller.go index a113b929..09d76867 100644 --- a/internal/controller/referenceddata_controller.go +++ b/internal/controller/referenceddata_controller.go @@ -180,7 +180,7 @@ func (w *localCompanionWriter) GetSecret(ctx context.Context, namespace, name st // downstreamCompanionWriter implements companionWriter by materialising // companions into the `ns-{project-uid}` namespace on the Karmada hub using // MappedNamespaceResourceStrategy. Companions written here are propagated to -// cells via the always-on referenced-data ResourceSelectors in the city-code +// cells via the always-on referenced-data ResourceSelectors in the location // PropagationPolicy. // // The downstreamNamespace field is pre-computed by the controller from the diff --git a/internal/controller/referenceddata_controller_test.go b/internal/controller/referenceddata_controller_test.go index f8594369..08b33615 100644 --- a/internal/controller/referenceddata_controller_test.go +++ b/internal/controller/referenceddata_controller_test.go @@ -30,6 +30,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/compute/internal/referenceddata" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) const ( @@ -1507,10 +1508,10 @@ func TestFederator_StatusSync_PreservesReferencedDataReadyCondition(t *testing.T ObjectMeta: metav1.ObjectMeta{ Name: testWDName, Namespace: testKarmadaNSStr, - Labels: map[string]string{cityCodeLabel: testCityCodeLAX}, + Labels: map[string]string{locationLabel: "us-west-1"}, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: testCityCodeLAX, + LocationRef: locationsv1alpha1.LocationReference{Name: "us-west-1"}, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: rdTestWorkloadName}, ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, diff --git a/internal/controller/testing_helpers_test.go b/internal/controller/testing_helpers_test.go index 303161ef..d584074a 100644 --- a/internal/controller/testing_helpers_test.go +++ b/internal/controller/testing_helpers_test.go @@ -20,6 +20,7 @@ import ( karmadapolicyv1alpha1 "github.com/karmada-io/api/policy/v1alpha1" computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) // ─── Scheme helpers ─────────────────────────────────────────────────────────── @@ -31,6 +32,7 @@ func newProjectScheme() *runtime.Scheme { _ = autoscalingv2.AddToScheme(s) _ = corev1.AddToScheme(s) _ = computev1alpha.AddToScheme(s) + _ = locationsv1alpha1.AddToScheme(s) return s } @@ -41,6 +43,7 @@ func newKarmadaScheme() *runtime.Scheme { _ = corev1.AddToScheme(s) _ = computev1alpha.AddToScheme(s) _ = networkingv1alpha.AddToScheme(s) + _ = locationsv1alpha1.AddToScheme(s) _ = karmadapolicyv1alpha1.Install(s) return s } diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index c9d8ac98..7365fb8b 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -32,6 +32,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) const ( @@ -55,6 +56,7 @@ type WorkloadReconciler struct { // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloads/status,verbs=get;update;patch // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloads/finalizers,verbs=update // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=locations.miloapis.com,resources=locations,verbs=get;list;watch func (r *WorkloadReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) @@ -446,30 +448,23 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( existingDeployments.Insert(deployment.Name) } - var locations networkingv1alpha.LocationBindingList + var locations locationsv1alpha1.LocationList if err := upstreamClient.List(ctx, &locations); err != nil { - return nil, nil, fmt.Errorf("failed to list location bindings: %w", err) - } - - if len(locations.Items) == 0 { - return nil, nil, fmt.Errorf("no locations are registered with the system") + return nil, nil, fmt.Errorf("failed to list locations: %w", err) } // Remember this: namespace, name, err := cache.SplitMetaNamespaceKey(key) for _, placement := range workload.Spec.Placements { - for _, cityCode := range placement.CityCodes { - foundLocation := false - for _, location := range locations.Items { - locationCityCode, ok := location.Spec.Topology[networkingv1alpha.TopologyCityCodeKey] - if ok && cityCode == locationCityCode { - foundLocation = true + for _, locationRef := range placement.Locations { + var location *locationsv1alpha1.Location + for i := range locations.Items { + if locations.Items[i].Name == locationRef.Name { + location = &locations.Items[i] break } } - if !foundLocation { - // TODO(jreese) update status condition on placement if no locations are - // found. + if location == nil || !apimeta.IsStatusConditionTrue(location.Status.Conditions, locationsv1alpha1.LocationConditionReady) { continue } @@ -477,7 +472,7 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( // them via labels instead? Would help with race conditions on workload // recreation. - deploymentName := fmt.Sprintf("%s-%s-%s", workload.Name, placement.Name, strings.ToLower(cityCode)) + deploymentName := fmt.Sprintf("%s-%s-%s", workload.Name, placement.Name, strings.ToLower(locationRef.Name)) desiredDeployments.Insert(deploymentName) desired = append(desired, computev1alpha.WorkloadDeployment{ @@ -486,6 +481,7 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( Name: deploymentName, Labels: map[string]string{ computev1alpha.WorkloadUIDLabel: string(workload.UID), + computev1alpha.LocationLabel: locationRef.Name, labelServiceName: labelServiceNameValue, }, }, @@ -495,7 +491,7 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( UID: workload.UID, }, PlacementName: placement.Name, - CityCode: cityCode, + LocationRef: locationRef, Template: workload.Spec.Template, ScaleSettings: placement.ScaleSettings, Replicas: new(placement.ScaleSettings.MinReplicas), diff --git a/internal/controller/workload_controller_test.go b/internal/controller/workload_controller_test.go index d8dbef80..164a5ee5 100644 --- a/internal/controller/workload_controller_test.go +++ b/internal/controller/workload_controller_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -16,7 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" computev1alpha "go.datum.net/compute/api/v1alpha" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) // makeWorkload builds a Workload with the given generation for use in @@ -95,7 +94,7 @@ func TestGetDeploymentsForWorkload_InitializesReplicas(t *testing.T) { Placements: []computev1alpha.WorkloadPlacement{ { Name: testDefaultPlacement, - CityCodes: []string{"DFW"}, + Locations: []locationsv1alpha1.LocationReference{{Name: "dfw"}}, ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: 2, }, @@ -103,15 +102,15 @@ func TestGetDeploymentsForWorkload_InitializesReplicas(t *testing.T) { }, }, } - location := &networkingv1alpha.LocationBinding{ + location := &locationsv1alpha1.Location{ ObjectMeta: metav1.ObjectMeta{Name: "dfw"}, - Spec: networkingv1alpha.LocationBindingSpec{ - LocationRef: corev1.LocalObjectReference{Name: "dfw"}, - Topology: map[string]string{"topology.datum.net/city-code": "DFW"}, + Status: locationsv1alpha1.LocationStatus{ + Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue}}, }, } s := newNetworkingScheme() + require.NoError(t, locationsv1alpha1.AddToScheme(s)) cl := fake.NewClientBuilder(). WithScheme(s). WithObjects(location). @@ -125,6 +124,8 @@ func TestGetDeploymentsForWorkload_InitializesReplicas(t *testing.T) { require.Len(t, desired, 1) require.NotNil(t, desired[0].Spec.Replicas) assert.Equal(t, int32(2), *desired[0].Spec.Replicas) + assert.Equal(t, "dfw", desired[0].Spec.LocationRef.Name) + assert.Equal(t, "dfw", desired[0].Labels[computev1alpha.LocationLabel]) } // TestReconcileWorkloadStatus_AllDeploymentsSameReason verifies that when all diff --git a/internal/controller/workloaddeployment_controller.go b/internal/controller/workloaddeployment_controller.go index 06bc90b8..c273d441 100644 --- a/internal/controller/workloaddeployment_controller.go +++ b/internal/controller/workloaddeployment_controller.go @@ -32,6 +32,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.datum.net/compute/internal/controller/instancecontrol" instancecontrolstateful "go.datum.net/compute/internal/controller/instancecontrol/stateful" @@ -80,7 +81,7 @@ func workloadDeploymentPodSelector(deployment *computev1alpha.WorkloadDeployment // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments/finalizers,verbs=update -// +kubebuilder:rbac:groups=networking.datumapis.com,resources=servinglocations,verbs=get;list;watch +// +kubebuilder:rbac:groups=locations.miloapis.com,resources=servinglocations,verbs=get;list;watch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces,verbs=get;list;watch // The management-mode WorkloadReconciler watches Networks. Declare the grant as @@ -138,24 +139,13 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco // Status().Update call when nothing changed (see loop-prevention comment below). existingStatus := *deployment.Status.DeepCopy() - // Resolve the cell's location before instances are built: the instance - // control strategy stamps Instance.Spec.Location from Status.Location as it - // creates them, so resolving afterwards left the first generation of - // instances permanently without one. + // Verify that federation delivered this deployment to the requested location. var location servingLocationResult - if r.NetworkingEnabled { - location, err = r.resolveLocation(ctx, cl.GetClient()) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed resolving location: %w", err) - } - location.evaluate(&deployment) - - // A location the cell contradicts is never written to status: an Instance - // carrying the wrong location is worse than one carrying none. - if location.reference != nil { - deployment.Status.Location = location.reference - } + location, err = r.resolveLocation(ctx, cl.GetClient()) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed resolving location: %w", err) } + location.evaluate(&deployment) // Collect all instances for this deployment listOpts := client.MatchingLabels{ @@ -325,14 +315,11 @@ func (r *WorkloadDeploymentReconciler) reconcileInstanceGates( ) (currentReplicas, updatedReplicas, readyReplicas, quotaBlockedReplicas, referencedDataBlockedReplicas int, err error) { templateHash := instancecontrol.ComputeHash(deployment.Spec.Template) for _, instance := range instances { - // Instances are stamped with the deployment's location as they are - // created, which leaves any instance that predates the cell learning its - // own location without one, and nothing else ever revisits it. Backfill - // it here. Best-effort by design: a failure is logged and the instance - // keeps running, because location has never gated scheduling. - if deployment.Status.Location != nil && instance.Spec.Location == nil { + // Backfill the canonical desired location on instances created before the + // location contract was introduced. + if instance.Spec.Location == nil { base := instance.DeepCopy() - instance.Spec.Location = deployment.Status.Location + instance.Spec.Location = &deployment.Spec.LocationRef if patchErr := c.Patch(ctx, &instance, client.MergeFrom(base)); patchErr != nil { log.FromContext(ctx).Error(patchErr, "failed backfilling instance location", "instance", instance.Name) } @@ -597,7 +584,7 @@ func selectWDBlockingCondition( // 5 - SourceNotFound / SourceTooLarge / SourceUnauthorized (hard spec error) // 6 - NetworkNotFound (hard error; user action required) // 7 - NetworkFailedToCreate (hard infra error) -// 8 - CityCodeMismatch / AmbiguousServingLocation (the deployment is on a cell +// 8 - LocationMismatch / AmbiguousServingLocation (the deployment is on a cell // that cannot serve it; nothing the user does clears it, and no other // blocker is worth reporting until it is fixed) func wdBlockingReasonPriority(reason string) int { @@ -622,7 +609,7 @@ func wdBlockingReasonPriority(reason string) int { return 6 case reasonNetworkFailedToCreate: return 7 - case computev1alpha.WorkloadDeploymentReasonCityCodeMismatch, + case computev1alpha.WorkloadDeploymentReasonLocationMismatch, computev1alpha.WorkloadDeploymentReasonAmbiguousServingLocation: return 8 default: @@ -635,11 +622,11 @@ func wdBlockingReasonPriority(reason string) int { type servingLocationResult struct { // reference is the location to stamp on the deployment and its instances. It // is nil whenever the cell's answer is missing or unusable. - reference *networkingv1alpha.LocationReference + reference *locationsv1alpha1.LocationReference // servingLocation is the single ServingLocation the cell was delivered, or // nil when it was delivered none or more than one. - servingLocation *networkingv1alpha.ServingLocation + servingLocation *locationsv1alpha1.ServingLocation // reason and message are the Available condition the deployment should // report while the location is unusable. Both are empty once it is usable. @@ -662,7 +649,7 @@ func (r *WorkloadDeploymentReconciler) resolveLocation( ctx context.Context, c client.Client, ) (servingLocationResult, error) { - var servingLocations networkingv1alpha.ServingLocationList + var servingLocations locationsv1alpha1.ServingLocationList if err := c.List(ctx, &servingLocations); err != nil { return servingLocationResult{}, fmt.Errorf("failed to list serving locations: %w", err) } @@ -690,7 +677,7 @@ func (r *WorkloadDeploymentReconciler) resolveLocation( return servingLocationResult{ reason: computev1alpha.WorkloadDeploymentReasonNoMatchingLocation, message: fmt.Sprintf("This cell has not been told which location it serves; it needs the %s cluster label, or its location has not reached it yet", - networkingv1alpha.ServingLocationTopologyLabel), + locationsv1alpha1.ServingLocationTopologyLabel), }, nil } @@ -700,7 +687,7 @@ func (r *WorkloadDeploymentReconciler) resolveLocation( // Location is cluster scoped, so the reference carries a name and no // namespace. return servingLocationResult{ - reference: &networkingv1alpha.LocationReference{Name: servingLocation.Name}, + reference: &locationsv1alpha1.LocationReference{Name: servingLocation.Name}, servingLocation: servingLocation, }, nil } @@ -714,15 +701,15 @@ func (s *servingLocationResult) evaluate(deployment *computev1alpha.WorkloadDepl return } - cityCode := s.servingLocation.CityCode() - if cityCode == deployment.Spec.CityCode { + locationName := s.servingLocation.Name + if locationName == deployment.Spec.LocationRef.Name { return } s.reference = nil - s.reason = computev1alpha.WorkloadDeploymentReasonCityCodeMismatch - s.message = fmt.Sprintf("Deployment asked for city %q but this cell serves %q; it was delivered to the wrong cell", - deployment.Spec.CityCode, cityCode) + s.reason = computev1alpha.WorkloadDeploymentReasonLocationMismatch + s.message = fmt.Sprintf("Deployment asked for location %q but this cell serves %q; it was delivered to the wrong cell", + deployment.Spec.LocationRef.Name, locationName) s.blocked = true } @@ -915,6 +902,12 @@ func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager, o ). Owns(&computev1alpha.Instance{}) + b = b.Watches(&locationsv1alpha1.ServingLocation{}, func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []mcreconcile.Request { + return enqueueWorkloadDeploymentsForServingLocation(ctx, cl.GetClient(), clusterName) + }) + }) + // Only watch networking resources when the networking integration is enabled. // On cells without network-services-operator these watches would log spurious // errors for missing CRDs. @@ -928,15 +921,6 @@ func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager, o return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request { return enqueueWorkloadDeploymentForClaim(ctx, cl.GetClient(), clusterName, o) }) - }). - // A deployment on a cell that does not yet know its own location waits - // without any other wake-up event, and the reconciler does not poll. - // Watching ServingLocations re-reconciles those deployments as soon as - // the cell learns where it is, so Status.Location is filled in. - Watches(&networkingv1alpha.ServingLocation{}, func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { - return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []mcreconcile.Request { - return enqueueWorkloadDeploymentsForServingLocation(ctx, cl.GetClient(), clusterName) - }) }) } diff --git a/internal/controller/workloaddeployment_controller_test.go b/internal/controller/workloaddeployment_controller_test.go index c19dee2b..dc441966 100644 --- a/internal/controller/workloaddeployment_controller_test.go +++ b/internal/controller/workloaddeployment_controller_test.go @@ -19,6 +19,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/compute/internal/controller/instancecontrol" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) const ( @@ -28,9 +29,9 @@ const ( wdControllerTestNS = "default" wdControllerTestUID = "wd-uid-test" - // wdControllerTestCityCode is the shared CityCode fixture for + // wdControllerTestLocation is the shared canonical Location fixture for // WorkloadDeployment controller tests. - wdControllerTestCityCode = "DFW" + wdControllerTestLocation = "us-east-1" // wdControllerTestWorkload is the shared WorkloadRef fixture. wdControllerTestWorkload = "test-workload" @@ -52,7 +53,7 @@ func wdControllerTestDeployment(minReplicas int32) *computev1alpha.WorkloadDeplo UID: wdControllerTestUID, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: wdControllerTestCityCode, + LocationRef: locationsv1alpha1.LocationReference{Name: wdControllerTestLocation}, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: wdControllerTestWorkload}, Replicas: new(minReplicas), @@ -203,7 +204,7 @@ func TestReconcileInstanceGates_NilSpecController_DoesNotPanic(t *testing.T) { deployment := &computev1alpha.WorkloadDeployment{ ObjectMeta: metav1.ObjectMeta{Name: wdControllerTestName, Namespace: wdControllerTestNS, UID: wdControllerTestUID}, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: wdControllerTestCityCode, + LocationRef: locationsv1alpha1.LocationReference{Name: wdControllerTestLocation}, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: wdControllerTestWorkload}, ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, diff --git a/internal/controller/workloaddeployment_federator.go b/internal/controller/workloaddeployment_federator.go index 95892492..57a18b8a 100644 --- a/internal/controller/workloaddeployment_federator.go +++ b/internal/controller/workloaddeployment_federator.go @@ -31,6 +31,7 @@ import ( karmadapolicyv1alpha1 "github.com/karmada-io/api/policy/v1alpha1" computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.miloapis.com/milo/pkg/downstreamclient" milosource "go.miloapis.com/milo/pkg/multicluster-runtime/source" ) @@ -42,11 +43,9 @@ const ( // object is permanently deleted. federatorFinalizer = "compute.datumapis.com/federator" - // cityCodeLabel is applied to WorkloadDeployments in the downstream namespace - // and is used by PropagationPolicy selectors to route them to the correct - // POP-cell clusters. Downstream Cluster objects are expected to carry this - // label with their city-code value. - cityCodeLabel = networkingv1alpha.TopologyCityCodeKey + // locationLabel is applied to downstream WorkloadDeployments and is used by + // PropagationPolicy selectors to route them to the exact Location-serving cell. + locationLabel = locationsv1alpha1.ServingLocationTopologyLabel kindWorkloadDeployment = "WorkloadDeployment" ) @@ -60,11 +59,11 @@ const ( // convention (matching the MappedNamespaceResourceStrategy used by // go.datum.net/network-services-operator). // 2. Upserts a corresponding WorkloadDeployment in that downstream namespace, -// stamped with label topology.datum.net/city-code=. -// 3. Lazily creates a PropagationPolicy per city code per downstream namespace -// that selects WorkloadDeployments by the city-code label and targets +// stamped with label topology.datum.net/location=. +// 3. Lazily creates a PropagationPolicy per location per downstream namespace +// that selects WorkloadDeployments by the location label and targets // clusters carrying the same label. The PP is deleted once no deployments -// with that city code remain in the namespace. +// with that location remain in the namespace. // 4. Reads the aggregated status from the downstream control plane and writes // it back to the project-namespace object. // 5. On deletion: removes the downstream WorkloadDeployment and cleans up @@ -163,7 +162,7 @@ func (r *WorkloadDeploymentFederator) Reconcile(ctx context.Context, req mcrecon return ctrl.Result{}, err } - if err := r.ensurePropagationPolicy(ctx, downstreamNS, deployment.Spec.CityCode); err != nil { + if err := r.ensurePropagationPolicy(ctx, downstreamNS, deployment.Spec.LocationRef.Name); err != nil { return ctrl.Result{}, err } @@ -230,7 +229,7 @@ func (r *WorkloadDeploymentFederator) Finalize(ctx context.Context, obj client.O } logger.Info("deleted downstream WorkloadDeployment", "downstreamNamespace", downstreamNS) - if err := r.cleanupPropagationPolicyIfUnused(ctx, downstreamNS, deployment.Spec.CityCode); err != nil { + if err := r.cleanupPropagationPolicyIfUnused(ctx, downstreamNS, deployment.Spec.LocationRef.Name); err != nil { return finalizer.Result{}, err } @@ -306,7 +305,7 @@ func (r *WorkloadDeploymentFederator) upsertDownstreamDeployment( if kd.Labels == nil { kd.Labels = make(map[string]string) } - kd.Labels[cityCodeLabel] = deployment.Spec.CityCode + kd.Labels[locationLabel] = deployment.Spec.LocationRef.Name kd.Labels[downstreamclient.UpstreamOwnerNamespaceLabel] = deployment.Namespace kd.Spec = deployment.Spec // Propagate controller-managed annotations from the project WD to the @@ -349,26 +348,26 @@ func (r *WorkloadDeploymentFederator) upsertDownstreamDeployment( } // ensurePropagationPolicy creates or updates a PropagationPolicy in the downstream -// namespace that selects all WorkloadDeployments with the given city-code label +// namespace that selects all WorkloadDeployments with the given location label // and targets clusters carrying the same label. func (r *WorkloadDeploymentFederator) ensurePropagationPolicy( ctx context.Context, downstreamNS string, - cityCode string, + locationName string, ) error { pp := &karmadapolicyv1alpha1.PropagationPolicy{ ObjectMeta: metav1.ObjectMeta{ - Name: propagationPolicyNameFor(cityCode), + Name: propagationPolicyNameFor(locationName), Namespace: downstreamNS, }, } result, err := controllerutil.CreateOrPatch(ctx, r.FederationClient, pp, func() error { pp.Spec = karmadapolicyv1alpha1.PropagationSpec{ - // Select WorkloadDeployments by city-code label, plus ALL + // Select WorkloadDeployments by location label, plus ALL // companion ConfigMaps and Secrets in this namespace that carry the // referenced-data label. The label selector on ConfigMap/Secret is - // city-code-agnostic — companions are shared across city codes when + // location-agnostic — companions are shared across locations when // multiple WDs reference the same source. Karmada propagates the // entire set to matching clusters in one policy, so companions // co-arrive with their WorkloadDeployment. @@ -382,14 +381,14 @@ func (r *WorkloadDeploymentFederator) ensurePropagationPolicy( Kind: kindWorkloadDeployment, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - cityCodeLabel: cityCode, + locationLabel: locationName, }, }, }, { // Propagate companion ConfigMaps alongside WorkloadDeployments. // The referenced-data label is the only selector needed; there - // is no per-city partitioning of companions. + // is no per-location partitioning of companions. APIVersion: corev1.SchemeGroupVersion.String(), Kind: kindConfigMap, LabelSelector: &metav1.LabelSelector{ @@ -410,13 +409,13 @@ func (r *WorkloadDeploymentFederator) ensurePropagationPolicy( }, }, Placement: karmadapolicyv1alpha1.Placement{ - // Route to clusters that carry the same city-code label. POP-cell + // Route to clusters that carry the same location label. POP-cell // clusters registered with the downstream control plane must be // labeled accordingly. ClusterAffinity: &karmadapolicyv1alpha1.ClusterAffinity{ LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - cityCodeLabel: cityCode, + locationLabel: locationName, }, }, }, @@ -425,10 +424,10 @@ func (r *WorkloadDeploymentFederator) ensurePropagationPolicy( return nil }) if err != nil { - return fmt.Errorf("failed to upsert PropagationPolicy for city %q in %s: %w", cityCode, downstreamNS, err) + return fmt.Errorf("failed to upsert PropagationPolicy for location %q in %s: %w", locationName, downstreamNS, err) } - log.FromContext(ctx).Info("upserted PropagationPolicy", "result", result, "cityCode", cityCode, "downstreamNamespace", downstreamNS) + log.FromContext(ctx).Info("upserted PropagationPolicy", "result", result, "location", locationName, "downstreamNamespace", downstreamNS) return nil } @@ -505,21 +504,21 @@ func (r *WorkloadDeploymentFederator) syncStatusFromDownstream( func (r *WorkloadDeploymentFederator) cleanupPropagationPolicyIfUnused( ctx context.Context, downstreamNS string, - cityCode string, + locationName string, ) error { - // The webhook requires cityCode, so an empty value here is corruption. An + // The webhook requires locationRef.name, so an empty value here is corruption. An // empty-valued label selector would match the wrong deployment set and // mis-decide whether the PropagationPolicy is still in use. - if cityCode == "" { - return fmt.Errorf("cannot evaluate PropagationPolicy usage in namespace %q: city code is empty", downstreamNS) + if locationName == "" { + return fmt.Errorf("cannot evaluate PropagationPolicy usage in namespace %q: location name is empty", downstreamNS) } var remaining computev1alpha.WorkloadDeploymentList if err := r.FederationClient.List(ctx, &remaining, client.InNamespace(downstreamNS), - client.MatchingLabels{cityCodeLabel: cityCode}, + client.MatchingLabels{locationLabel: locationName}, ); err != nil { - return fmt.Errorf("failed to list remaining downstream deployments for city %q: %w", cityCode, err) + return fmt.Errorf("failed to list remaining downstream deployments for location %q: %w", locationName, err) } if len(remaining.Items) > 0 { @@ -529,15 +528,15 @@ func (r *WorkloadDeploymentFederator) cleanupPropagationPolicyIfUnused( pp := &karmadapolicyv1alpha1.PropagationPolicy{ ObjectMeta: metav1.ObjectMeta{ - Name: propagationPolicyNameFor(cityCode), + Name: propagationPolicyNameFor(locationName), Namespace: downstreamNS, }, } if err := r.FederationClient.Delete(ctx, pp); client.IgnoreNotFound(err) != nil { - return fmt.Errorf("failed to delete PropagationPolicy for city %q in %s: %w", cityCode, downstreamNS, err) + return fmt.Errorf("failed to delete PropagationPolicy for location %q in %s: %w", locationName, downstreamNS, err) } - log.FromContext(ctx).Info("deleted PropagationPolicy (no more deployments for city)", "cityCode", cityCode, "downstreamNamespace", downstreamNS) + log.FromContext(ctx).Info("deleted PropagationPolicy (no more deployments for location)", "location", locationName, "downstreamNamespace", downstreamNS) return nil } @@ -718,7 +717,7 @@ func projectClusterNameFromLabel(encoded string) string { // propagationPolicyNameFor returns the PropagationPolicy name for a given city // code. The name is stable and deterministic so that multiple reconciles of // different deployments sharing the same city code converge on the same policy. -func propagationPolicyNameFor(cityCode string) string { - sanitized := strings.ToLower(strings.ReplaceAll(cityCode, " ", "-")) - return fmt.Sprintf("city-%s", sanitized) +func propagationPolicyNameFor(locationName string) string { + sanitized := strings.ToLower(strings.ReplaceAll(locationName, " ", "-")) + return fmt.Sprintf("location-%s", sanitized) } diff --git a/internal/controller/workloaddeployment_federator_test.go b/internal/controller/workloaddeployment_federator_test.go index bee4f07f..50086c35 100644 --- a/internal/controller/workloaddeployment_federator_test.go +++ b/internal/controller/workloaddeployment_federator_test.go @@ -21,18 +21,19 @@ import ( mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" computev1alpha "go.datum.net/compute/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.miloapis.com/milo/pkg/downstreamclient" ) // ─── Shared test constants ──────────────────────────────────────────────────── const ( - testCluster = "test-project-cluster" - testProjNS = "my-project" - testProjNSUID = types.UID("aabbccdd-0000-1111-2222-333344445555") - testKarmadaNSStr = "ns-aabbccdd-0000-1111-2222-333344445555" - testWDName = "my-workload-deployment" - testCityCodeLAX = "LAX" + testCluster = "test-project-cluster" + testProjNS = "my-project" + testProjNSUID = types.UID("aabbccdd-0000-1111-2222-333344445555") + testKarmadaNSStr = "ns-aabbccdd-0000-1111-2222-333344445555" + testWDName = "my-workload-deployment" + testFederatorLocation = "us-west-2" ) // ─── Test helpers ───────────────────────────────────────────────────────────── @@ -57,7 +58,7 @@ func testWorkloadDeployment(opts ...func(*computev1alpha.WorkloadDeployment)) *c UID: "wd-uid-1111", }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: testCityCodeLAX, + LocationRef: locationsv1alpha1.LocationReference{Name: testFederatorLocation}, WorkloadRef: computev1alpha.WorkloadReference{ Name: rdTestWorkloadName, }, @@ -263,20 +264,20 @@ func TestPropagationPolicyNameFor(t *testing.T) { t.Parallel() tests := []struct { - cityCode string + location string want string }{ - {"LAX", "city-lax"}, - {"lax", "city-lax"}, - {"New York", "city-new-york"}, - {"LOS ANGELES", "city-los-angeles"}, - {"SEA", "city-sea"}, + {"LAX", "location-lax"}, + {"lax", "location-lax"}, + {"New York", "location-new-york"}, + {"LOS ANGELES", "location-los-angeles"}, + {"SEA", "location-sea"}, } for _, tt := range tests { - t.Run(tt.cityCode, func(t *testing.T) { + t.Run(tt.location, func(t *testing.T) { t.Parallel() - got := propagationPolicyNameFor(tt.cityCode) + got := propagationPolicyNameFor(tt.location) assert.Equal(t, tt.want, got) }) } @@ -375,13 +376,13 @@ func TestWorkloadDeploymentFederator_FederatesToKarmada(t *testing.T) { Namespace: testKarmadaNSStr, }, &karmadaWD) require.NoError(t, err, "Karmada WorkloadDeployment should exist") - assert.Equal(t, testCityCodeLAX, karmadaWD.Labels[cityCodeLabel], + assert.Equal(t, testFederatorLocation, karmadaWD.Labels[locationLabel], "city-code label should be set on Karmada WD") - assert.Equal(t, testCityCodeLAX, karmadaWD.Spec.CityCode, + assert.Equal(t, testFederatorLocation, karmadaWD.Spec.LocationRef.Name, "spec.cityCode should be copied from project WD") // PropagationPolicy for the city code must exist. - ppName := propagationPolicyNameFor(testCityCodeLAX) + ppName := propagationPolicyNameFor(testFederatorLocation) var pp karmadapolicyv1alpha1.PropagationPolicy err = karmadaClient.Get(ctx, types.NamespacedName{ Name: ppName, @@ -397,7 +398,7 @@ func TestWorkloadDeploymentFederator_FederatesToKarmada(t *testing.T) { assert.Equal(t, computev1alpha.GroupVersion.String(), wdSel.APIVersion) assert.Equal(t, kindWorkloadDeployment, wdSel.Kind) require.NotNil(t, wdSel.LabelSelector) - assert.Equal(t, testCityCodeLAX, wdSel.LabelSelector.MatchLabels[cityCodeLabel]) + assert.Equal(t, testFederatorLocation, wdSel.LabelSelector.MatchLabels[locationLabel]) cmSel := pp.Spec.ResourceSelectors[1] assert.Equal(t, "v1", cmSel.APIVersion) @@ -414,8 +415,8 @@ func TestWorkloadDeploymentFederator_FederatesToKarmada(t *testing.T) { // The PP cluster affinity must target clusters carrying the same city-code. require.NotNil(t, pp.Spec.Placement.ClusterAffinity) require.NotNil(t, pp.Spec.Placement.ClusterAffinity.LabelSelector) - assert.Equal(t, testCityCodeLAX, - pp.Spec.Placement.ClusterAffinity.LabelSelector.MatchLabels[cityCodeLabel]) + assert.Equal(t, testFederatorLocation, + pp.Spec.Placement.ClusterAffinity.LabelSelector.MatchLabels[locationLabel]) } // TestWorkloadDeploymentFederator_Finalization covers the deletion scenarios: @@ -423,7 +424,7 @@ func TestWorkloadDeploymentFederator_FederatesToKarmada(t *testing.T) { func TestWorkloadDeploymentFederator_Finalization(t *testing.T) { t.Parallel() - ppName := propagationPolicyNameFor(testCityCodeLAX) + ppName := propagationPolicyNameFor(testFederatorLocation) tests := []struct { name string @@ -444,10 +445,10 @@ func TestWorkloadDeploymentFederator_Finalization(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "other-deployment", Namespace: testKarmadaNSStr, - Labels: map[string]string{cityCodeLabel: testCityCodeLAX}, + Labels: map[string]string{locationLabel: testFederatorLocation}, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: testCityCodeLAX, + LocationRef: locationsv1alpha1.LocationReference{Name: testFederatorLocation}, PlacementName: "other", WorkloadRef: computev1alpha.WorkloadReference{Name: "other"}, ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, @@ -471,10 +472,10 @@ func TestWorkloadDeploymentFederator_Finalization(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: testWDName, Namespace: testKarmadaNSStr, - Labels: map[string]string{cityCodeLabel: testCityCodeLAX}, + Labels: map[string]string{locationLabel: testFederatorLocation}, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: testCityCodeLAX, + LocationRef: locationsv1alpha1.LocationReference{Name: testFederatorLocation}, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: rdTestWorkloadName}, ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, @@ -537,10 +538,10 @@ func TestWorkloadDeploymentFederator_Finalization(t *testing.T) { } } -// TestCleanupPropagationPolicyIfUnused_EmptyCityCode verifies the guard -// against listing with an empty city-code label value, which would match the +// TestCleanupPropagationPolicyIfUnused_EmptyLocation verifies the guard +// against listing with an empty location label value, which would match the // wrong deployment set and mis-decide PropagationPolicy cleanup. -func TestCleanupPropagationPolicyIfUnused_EmptyCityCode(t *testing.T) { +func TestCleanupPropagationPolicyIfUnused_EmptyLocation(t *testing.T) { t.Parallel() projectClient := newProjectFakeClient(testProjectNamespace()) @@ -549,7 +550,7 @@ func TestCleanupPropagationPolicyIfUnused_EmptyCityCode(t *testing.T) { err := r.cleanupPropagationPolicyIfUnused(context.Background(), testKarmadaNSStr, "") require.Error(t, err) - assert.Contains(t, err.Error(), "city code is empty") + assert.Contains(t, err.Error(), "location name is empty") } // TestWorkloadDeploymentFederator_PropagationPolicyHasReferencedDataSelectors @@ -567,7 +568,7 @@ func TestWorkloadDeploymentFederator_PropagationPolicyHasReferencedDataSelectors _, err := r.Reconcile(context.Background(), reconcileRequest()) require.NoError(t, err) - ppName := propagationPolicyNameFor(testCityCodeLAX) + ppName := propagationPolicyNameFor(testFederatorLocation) var pp karmadapolicyv1alpha1.PropagationPolicy require.NoError(t, karmadaClient.Get(context.Background(), types.NamespacedName{ Name: ppName, @@ -726,7 +727,7 @@ func TestWorkloadDeploymentFederator_FinalizeIsSelfContained(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: testWDName, Namespace: testKarmadaNSStr, - Labels: map[string]string{cityCodeLabel: testCityCodeLAX}, + Labels: map[string]string{locationLabel: testFederatorLocation}, }, Spec: wd.Spec, } diff --git a/internal/controller/workloaddeployment_hpa_controller.go b/internal/controller/workloaddeployment_hpa_controller.go index be20c84f..17e83771 100644 --- a/internal/controller/workloaddeployment_hpa_controller.go +++ b/internal/controller/workloaddeployment_hpa_controller.go @@ -132,7 +132,7 @@ func workloadDeploymentHPALabels(deployment *computev1alpha.WorkloadDeployment) computev1alpha.WorkloadDeploymentNameLabel: deployment.Name, computev1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, computev1alpha.PlacementNameLabel: deployment.Spec.PlacementName, - computev1alpha.CityCodeLabel: deployment.Spec.CityCode, + computev1alpha.LocationLabel: deployment.Spec.LocationRef.Name, } } diff --git a/internal/controller/workloaddeployment_hpa_controller_test.go b/internal/controller/workloaddeployment_hpa_controller_test.go index df6cca4e..cdc8c95b 100644 --- a/internal/controller/workloaddeployment_hpa_controller_test.go +++ b/internal/controller/workloaddeployment_hpa_controller_test.go @@ -20,6 +20,7 @@ import ( mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" computev1alpha "go.datum.net/compute/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) const ( @@ -37,7 +38,7 @@ func hpaTestDeployment() *computev1alpha.WorkloadDeployment { UID: wdControllerTestUID, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: wdControllerTestCityCode, + LocationRef: locationsv1alpha1.LocationReference{Name: wdControllerTestLocation}, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: wdControllerTestWorkload}, ScaleSettings: computev1alpha.HorizontalScaleSettings{ @@ -105,7 +106,7 @@ func TestWorkloadDeploymentHPAReconciler_CreatesHPA(t *testing.T) { computev1alpha.WorkloadDeploymentNameLabel: deployment.Name, computev1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, computev1alpha.PlacementNameLabel: deployment.Spec.PlacementName, - computev1alpha.CityCodeLabel: deployment.Spec.CityCode, + computev1alpha.LocationLabel: deployment.Spec.LocationRef.Name, }, hpa.Labels) require.Len(t, hpa.OwnerReferences, 1) assert.Equal(t, deployment.Name, hpa.OwnerReferences[0].Name) diff --git a/internal/controller/workloaddeployment_location_test.go b/internal/controller/workloaddeployment_location_test.go index 20e5739f..5ebbe258 100644 --- a/internal/controller/workloaddeployment_location_test.go +++ b/internal/controller/workloaddeployment_location_test.go @@ -21,6 +21,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.datum.net/compute/internal/controller/instancecontrol" ) @@ -29,8 +30,8 @@ const ( // locTestCityCode / locTestOtherCityCode: deployments under test target // locTestCityCode; locTestOtherCityCode identifies the city a mis-delivered // cell serves. - locTestCityCode = "DFW" - locTestOtherCityCode = "ORD" + locTestLocation = "us-central-1" + locTestOtherLocation = "us-central-2" // locTestWDNamespace is the namespace of the deployments under test. locTestWDNamespace = "default" @@ -41,18 +42,16 @@ func newNetworkingScheme() *runtime.Scheme { s := runtime.NewScheme() _ = computev1alpha.AddToScheme(s) _ = networkingv1alpha.AddToScheme(s) + _ = locationsv1alpha1.AddToScheme(s) return s } // newTestServingLocation builds a ServingLocation fixture shaped like the one a // cell is delivered: cluster scoped, and carrying its city under the // topology.datum.net/city-code key. -func newTestServingLocation(name, cityCode string) *networkingv1alpha.ServingLocation { - return &networkingv1alpha.ServingLocation{ +func newTestServingLocation(name, _ string) *locationsv1alpha1.ServingLocation { + return &locationsv1alpha1.ServingLocation{ ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: networkingv1alpha.ServingLocationSpec{ - Topology: map[string]string{networkingv1alpha.TopologyCityCodeKey: cityCode}, - }, } } @@ -60,7 +59,7 @@ func newTestServingLocation(name, cityCode string) *networkingv1alpha.ServingLoc // serves the city the deployment asked for. func resolvedTestLocation() servingLocationResult { return servingLocationResult{ - reference: &networkingv1alpha.LocationReference{Name: "loc-dfw-1"}, + reference: &locationsv1alpha1.LocationReference{Name: "loc-dfw-1"}, } } @@ -68,7 +67,7 @@ func newLocationTestDeployment(name string) *computev1alpha.WorkloadDeployment { return &computev1alpha.WorkloadDeployment{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: locTestWDNamespace}, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: locTestCityCode, + LocationRef: locationsv1alpha1.LocationReference{Name: locTestLocation}, }, } } @@ -83,10 +82,11 @@ func TestResolveLocation_ExactlyOneServingLocation(t *testing.T) { cl := fake.NewClientBuilder(). WithScheme(newNetworkingScheme()). - WithObjects(newTestServingLocation(locationName, locTestCityCode)). + WithObjects(newTestServingLocation(locationName, locTestLocation)). Build() deployment := newLocationTestDeployment("test-wd") + deployment.Spec.LocationRef = locationsv1alpha1.LocationReference{Name: locationName} r := &WorkloadDeploymentReconciler{} result, err := r.resolveLocation(context.Background(), cl) @@ -119,10 +119,8 @@ func TestResolveLocation_NoServingLocation_IsNonGating(t *testing.T) { assert.False(t, result.blocked, "a cell that has not been identified yet must never hold instances back") assert.Equal(t, computev1alpha.WorkloadDeploymentReasonNoMatchingLocation, result.reason) - assert.Contains(t, result.message, networkingv1alpha.ServingLocationTopologyLabel, + assert.Contains(t, result.message, locationsv1alpha1.ServingLocationTopologyLabel, "the message must name the cluster label that fixes it") - assert.Nil(t, deployment.Status.Location, - "Status.Location must be left alone when nothing resolved") } // TestResolveLocation_MultipleServingLocations_RefusesToGuess verifies the @@ -134,8 +132,8 @@ func TestResolveLocation_MultipleServingLocations_RefusesToGuess(t *testing.T) { cl := fake.NewClientBuilder(). WithScheme(newNetworkingScheme()). WithObjects( - newTestServingLocation("loc-dfw-1", locTestCityCode), - newTestServingLocation("loc-ord-1", locTestOtherCityCode), + newTestServingLocation("loc-dfw-1", locTestLocation), + newTestServingLocation("loc-ord-1", locTestOtherLocation), ). Build() @@ -162,7 +160,7 @@ func TestResolveLocation_CityCodeMismatch(t *testing.T) { cl := fake.NewClientBuilder(). WithScheme(newNetworkingScheme()). - WithObjects(newTestServingLocation("loc-ord-1", locTestOtherCityCode)). + WithObjects(newTestServingLocation(locTestOtherLocation, locTestOtherLocation)). Build() deployment := newLocationTestDeployment("test-wd") @@ -175,9 +173,9 @@ func TestResolveLocation_CityCodeMismatch(t *testing.T) { assert.Nil(t, result.reference, "the wrong cell's location must never be stamped on the deployment") assert.True(t, result.blocked, "a misplaced deployment must not proceed silently") - assert.Equal(t, computev1alpha.WorkloadDeploymentReasonCityCodeMismatch, result.reason) - assert.Contains(t, result.message, locTestCityCode) - assert.Contains(t, result.message, locTestOtherCityCode) + assert.Equal(t, computev1alpha.WorkloadDeploymentReasonLocationMismatch, result.reason) + assert.Contains(t, result.message, locTestLocation) + assert.Contains(t, result.message, locTestOtherLocation) } // newLocationTestWDReconciler builds a WorkloadDeploymentReconciler with @@ -211,7 +209,7 @@ func newLocationTestReconcilableWD(name string) *computev1alpha.WorkloadDeployme Finalizers: []string{workloadControllerFinalizer}, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: locTestCityCode, + LocationRef: locationsv1alpha1.LocationReference{Name: locTestLocation}, WorkloadRef: computev1alpha.WorkloadReference{Name: "location-test-workload"}, Replicas: new(int32(1)), ScaleSettings: computev1alpha.HorizontalScaleSettings{ @@ -287,13 +285,12 @@ func TestWorkloadDeploymentReconcile_UnidentifiedCell_SetsCondition(t *testing.T require.NotNil(t, cond, "Available must be set while the cell has no location") assert.Equal(t, metav1.ConditionFalse, cond.Status) assert.Equal(t, computev1alpha.WorkloadDeploymentReasonNoMatchingLocation, cond.Reason) - assert.Contains(t, cond.Message, networkingv1alpha.ServingLocationTopologyLabel, + assert.Contains(t, cond.Message, locationsv1alpha1.ServingLocationTopologyLabel, "the condition message must name the label that identifies the cell") - assert.Nil(t, updated.Status.Location) // Deliver the cell's location; the next reconcile resolves it and must // replace the waiting reason. - servingLocation := newTestServingLocation("loc-dfw-2", locTestCityCode) + servingLocation := newTestServingLocation(locTestLocation, locTestLocation) require.NoError(t, cl.Create(context.Background(), servingLocation)) _, err = r.Reconcile(context.Background(), req) @@ -304,8 +301,7 @@ func TestWorkloadDeploymentReconcile_UnidentifiedCell_SetsCondition(t *testing.T require.NotNil(t, cond) assert.Equal(t, computev1alpha.WorkloadDeploymentReasonInstancesProvisioning, cond.Reason, "the waiting reason must give way once the cell knows where it is") - require.NotNil(t, updated.Status.Location) - assert.Equal(t, servingLocation.Name, updated.Status.Location.Name) + assert.Equal(t, servingLocation.Name, updated.Spec.LocationRef.Name) } // TestWorkloadDeploymentReconcile_CityCodeMismatch_HoldsInstances verifies that a @@ -320,7 +316,7 @@ func TestWorkloadDeploymentReconcile_CityCodeMismatch_HoldsInstances(t *testing. cl := fake.NewClientBuilder(). WithScheme(newNetworkingScheme()). - WithObjects(deployment, instance, newTestServingLocation("loc-ord-1", locTestOtherCityCode)). + WithObjects(deployment, instance, newTestServingLocation(locTestOtherLocation, locTestOtherLocation)). WithStatusSubresource(deployment). Build() r := newLocationTestWDReconciler(cl) @@ -334,10 +330,8 @@ func TestWorkloadDeploymentReconcile_CityCodeMismatch_HoldsInstances(t *testing. cond := apimeta.FindStatusCondition(updated.Status.Conditions, computev1alpha.WorkloadDeploymentAvailable) require.NotNil(t, cond) - assert.Equal(t, computev1alpha.WorkloadDeploymentReasonCityCodeMismatch, cond.Reason, + assert.Equal(t, computev1alpha.WorkloadDeploymentReasonLocationMismatch, cond.Reason, "a misplaced deployment must report the placement fault over any other blocker") - assert.Nil(t, updated.Status.Location, - "the wrong cell's location must never be written to status") var updatedInstance computev1alpha.Instance require.NoError(t, cl.Get(context.Background(), types.NamespacedName{ @@ -360,7 +354,7 @@ func TestWorkloadDeploymentReconcile_BackfillsInstanceLocation(t *testing.T) { instance := newLocationTestInstance(deployment) require.Nil(t, instance.Spec.Location, "the fixture must start without a location") - servingLocation := newTestServingLocation("loc-dfw-1", locTestCityCode) + servingLocation := newTestServingLocation(locTestLocation, locTestLocation) cl := fake.NewClientBuilder(). WithScheme(newNetworkingScheme()). @@ -390,7 +384,7 @@ func TestEnqueueWorkloadDeploymentsForServingLocation(t *testing.T) { wdDFW := newLocationTestDeployment("wd-dfw") wdORD := newLocationTestDeployment("wd-ord") - wdORD.Spec.CityCode = locTestOtherCityCode + wdORD.Spec.LocationRef.Name = locTestOtherLocation cl := fake.NewClientBuilder(). WithScheme(newNetworkingScheme()). diff --git a/internal/controller/workloaddeployment_network_binding.go b/internal/controller/workloaddeployment_network_binding.go index ea132afd..da4d554c 100644 --- a/internal/controller/workloaddeployment_network_binding.go +++ b/internal/controller/workloaddeployment_network_binding.go @@ -43,11 +43,7 @@ func (r *WorkloadDeploymentFederator) ensureNetworkBinding( ctx context.Context, hubDeployment *computev1alpha.WorkloadDeployment, ) (*networkingv1alpha.NetworkBinding, error) { - // The location is written by the cell that serves the deployment and reaches - // the hub through Karmada status aggregation, so it is absent for as long as - // nothing has placed the deployment. There is no presence to ask for until - // then, and a binding without a location cannot be created at all. - if hubDeployment.Status.Location == nil || hubDeployment.Status.Location.Name == "" { + if hubDeployment.Spec.LocationRef.Name == "" { return nil, nil } @@ -55,7 +51,7 @@ func (r *WorkloadDeploymentFederator) ensureNetworkBinding( if !ok { return nil, nil } - location := *hubDeployment.Status.Location + location := networkingv1alpha.LocationReference{Name: hubDeployment.Spec.LocationRef.Name} key := client.ObjectKey{Namespace: hubDeployment.Namespace, Name: hubDeployment.Name} var existing networkingv1alpha.NetworkBinding diff --git a/internal/controller/workloaddeployment_network_binding_test.go b/internal/controller/workloaddeployment_network_binding_test.go index 8126b3c5..2050e5a1 100644 --- a/internal/controller/workloaddeployment_network_binding_test.go +++ b/internal/controller/workloaddeployment_network_binding_test.go @@ -18,6 +18,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.miloapis.com/milo/pkg/downstreamclient" ) @@ -37,7 +38,7 @@ func testHubDeployment(opts ...func(*computev1alpha.WorkloadDeployment)) *comput UID: testHubWDUID, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - CityCode: testCityCodeLAX, + LocationRef: locationsv1alpha1.LocationReference{Name: testLocationName}, Template: computev1alpha.InstanceTemplateSpec{ Spec: computev1alpha.InstanceSpec{ NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{ @@ -58,7 +59,7 @@ func testHubDeployment(opts ...func(*computev1alpha.WorkloadDeployment)) *comput // which on the hub arrives through Karmada status aggregation. func withServingLocation(name string) func(*computev1alpha.WorkloadDeployment) { return func(wd *computev1alpha.WorkloadDeployment) { - wd.Status.Location = &networkingv1alpha.LocationReference{Name: name} + wd.Spec.LocationRef = locationsv1alpha1.LocationReference{Name: name} } } @@ -98,7 +99,7 @@ func TestEnsureNetworkBinding_DeclaresPresenceWhereDeploymentRuns(t *testing.T) require.NoError(t, err) assert.Equal(t, testNetworkName, stored.Spec.Network.Name) - assert.Equal(t, testLocationName, stored.Spec.Location.Name) + assert.Equal(t, hubWD.Spec.LocationRef.Name, stored.Spec.Location.Name) require.NotNil(t, stored.Spec.Consumer) assert.Equal(t, computev1alpha.GroupVersion.Group, stored.Spec.Consumer.APIGroup) @@ -129,10 +130,6 @@ func TestEnsureNetworkBinding_NothingToDeclareYet(t *testing.T) { name string hubWD *computev1alpha.WorkloadDeployment }{ - { - name: "no serving location", - hubWD: testHubDeployment(), - }, { name: "no network interfaces", hubWD: testHubDeployment(withServingLocation(testLocationName), func(wd *computev1alpha.WorkloadDeployment) { @@ -341,7 +338,7 @@ func TestWorkloadDeploymentFederator_CreatesNetworkBindingOnReconcile(t *testing stored, err := getBinding(t, karmadaClient) require.NoError(t, err) assert.Equal(t, testNetworkName, stored.Spec.Network.Name) - assert.Equal(t, testLocationName, stored.Spec.Location.Name) + assert.Equal(t, wd.Spec.LocationRef.Name, stored.Spec.Location.Name) } // TestMapNetworkBindingToRequest verifies the binding-to-deployment mapping: the diff --git a/internal/controller/workloaddeployment_setup_test.go b/internal/controller/workloaddeployment_setup_test.go index 29522bd2..edf451db 100644 --- a/internal/controller/workloaddeployment_setup_test.go +++ b/internal/controller/workloaddeployment_setup_test.go @@ -21,6 +21,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) // TestWorkloadDeploymentSetupWithManager_CellModeNoNetworkingCRD asserts that with @@ -42,6 +43,7 @@ func TestWorkloadDeploymentSetupWithManager_CellModeNoNetworkingCRD(t *testing.T require.NoError(t, computev1alpha.AddToScheme(scheme)) require.NoError(t, corev1.AddToScheme(scheme)) require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + require.NoError(t, locationsv1alpha1.AddToScheme(scheme)) deploymentCluster, err := cluster.New(cfg, func(o *cluster.Options) { o.Scheme = scheme }) require.NoError(t, err) diff --git a/internal/validation/instance_validation_test.go b/internal/validation/instance_validation_test.go index cf56044b..6bbe3d95 100644 --- a/internal/validation/instance_validation_test.go +++ b/internal/validation/instance_validation_test.go @@ -422,7 +422,7 @@ func TestReferencedDataSAR(t *testing.T) { Context: context.Background(), Workload: tc.workload, AdmissionRequest: admission.Request{}, - ValidCityCodes: []string{"DFW"}, + ValidLocations: []string{"DFW"}, } spec := tc.workload.Spec.Template.Spec @@ -483,7 +483,7 @@ func TestBothRefsSetEnvFrom(t *testing.T) { Context: context.Background(), Workload: workload, AdmissionRequest: admission.Request{}, - ValidCityCodes: []string{testCityCodeDFW}, + ValidLocations: []string{testCityCodeDFW}, } _ = validateReferencedDataAccess(workload.Spec.Template.Spec, specPath, opts) if sarCount != sarCountBefore { @@ -530,7 +530,7 @@ func TestReferencedDataSARInternalError(t *testing.T) { Context: context.Background(), Workload: workload, AdmissionRequest: admission.Request{}, - ValidCityCodes: []string{testCityCodeDFW}, + ValidLocations: []string{testCityCodeDFW}, } errs := validateReferencedDataAccess(workload.Spec.Template.Spec, specPath, opts) @@ -601,7 +601,7 @@ func TestValidateUpdateSARPath(t *testing.T) { Context: context.Background(), Workload: newWorkload, AdmissionRequest: admission.Request{}, - ValidCityCodes: []string{testCityCodeDFW}, + ValidLocations: []string{testCityCodeDFW}, } // ValidateWorkloadCreate is called by ValidateUpdate in the webhook; here @@ -693,7 +693,7 @@ func TestWorkloadWithReferencedDataE2E(t *testing.T) { Client: fakeClient, Context: context.Background(), Workload: workload, - ValidCityCodes: []string{"DFW"}, + ValidLocations: []string{"DFW"}, } errs := ValidateWorkloadCreate(workload, opts) diff --git a/internal/validation/workload_validation.go b/internal/validation/workload_validation.go index 5f320e9a..265b9f9c 100644 --- a/internal/validation/workload_validation.go +++ b/internal/validation/workload_validation.go @@ -31,7 +31,7 @@ type WorkloadValidationOptions struct { AdmissionRequest admission.Request Context context.Context Workload *computev1alpha.Workload - ValidCityCodes []string + ValidLocations []string } func validateWorkloadSpec(spec computev1alpha.WorkloadSpec, opts WorkloadValidationOptions) field.ErrorList { @@ -71,14 +71,19 @@ func validateWorkloadPlacement(placement computev1alpha.WorkloadPlacement, field } } - cityCodesPath := fieldPath.Child("cityCodes") - if len(placement.CityCodes) == 0 { - allErrs = append(allErrs, field.Required(cityCodesPath, "")) + locationsPath := fieldPath.Child("locations") + if len(placement.Locations) == 0 { + allErrs = append(allErrs, field.Required(locationsPath, "")) } else { - for i, cityCode := range placement.CityCodes { - if !slices.Contains(opts.ValidCityCodes, cityCode) { - allErrs = append(allErrs, field.NotSupported(cityCodesPath.Index(i), cityCode, opts.ValidCityCodes)) + seen := sets.New[string]() + for i, location := range placement.Locations { + namePath := locationsPath.Index(i).Child("name") + if !slices.Contains(opts.ValidLocations, location.Name) { + allErrs = append(allErrs, field.NotSupported(namePath, location.Name, opts.ValidLocations)) + } else if seen.Has(location.Name) { + allErrs = append(allErrs, field.Duplicate(namePath, location.Name)) } + seen.Insert(location.Name) } } diff --git a/internal/validation/workload_validation_test.go b/internal/validation/workload_validation_test.go index f235cd39..275b5f5b 100644 --- a/internal/validation/workload_validation_test.go +++ b/internal/validation/workload_validation_test.go @@ -21,6 +21,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) const ( @@ -47,26 +48,26 @@ func TestValidateWorkloads(t *testing.T) { field.Required(field.NewPath("spec.placements"), ""), }, }, - "missing cityCode": { + "missing location": { workload: MakeSandboxWorkload( "test", func(w *computev1alpha.Workload) { - w.Spec.Placements[0].CityCodes = []string{} + w.Spec.Placements[0].Locations = []locationsv1alpha1.LocationReference{} }, ), expectedErrors: field.ErrorList{ - field.Required(field.NewPath("spec.placements[0].cityCodes"), ""), + field.Required(field.NewPath("spec.placements[0].locations"), ""), }, }, - "invalid cityCode": { + "invalid location": { workload: MakeSandboxWorkload( "test", func(w *computev1alpha.Workload) { - w.Spec.Placements[0].CityCodes = []string{"TEST"} + w.Spec.Placements[0].Locations = []locationsv1alpha1.LocationReference{{Name: "TEST"}} }, ), expectedErrors: field.ErrorList{ - field.NotSupported(field.NewPath("spec.placements[0].cityCodes[0]"), "TEST", []string{}), + field.NotSupported(field.NewPath("spec.placements[0].locations[0].name"), "TEST", []string{}), }, }, "missing placement name": { @@ -613,8 +614,8 @@ func TestValidateWorkloads(t *testing.T) { }, ) - if len(scenario.opts.ValidCityCodes) == 0 { - scenario.opts.ValidCityCodes = []string{testCityCodeDFW} + if len(scenario.opts.ValidLocations) == 0 { + scenario.opts.ValidLocations = []string{testCityCodeDFW} } t.Run(name, func(t *testing.T) { @@ -669,7 +670,7 @@ func MakeSandboxWorkload(name string, tweaks ...Tweak) *computev1alpha.Workload Placements: []computev1alpha.WorkloadPlacement{ { Name: "placement1", - CityCodes: []string{testCityCodeDFW}, + Locations: []locationsv1alpha1.LocationReference{{Name: testCityCodeDFW}}, ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: 1, }, @@ -744,7 +745,7 @@ func MakeVMWorkload(name string, tweaks ...Tweak) *computev1alpha.Workload { Placements: []computev1alpha.WorkloadPlacement{ { Name: "placement1", - CityCodes: []string{testCityCodeDFW}, + Locations: []locationsv1alpha1.LocationReference{{Name: testCityCodeDFW}}, ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: 1, }, diff --git a/internal/webhook/v1alpha/workload_webhook.go b/internal/webhook/v1alpha/workload_webhook.go index e8926b60..304d2533 100644 --- a/internal/webhook/v1alpha/workload_webhook.go +++ b/internal/webhook/v1alpha/workload_webhook.go @@ -6,6 +6,7 @@ import ( "fmt" "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -16,7 +17,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/compute/internal/validation" computewebhook "go.datum.net/compute/internal/webhook" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) // SetupWorkloadWebhookWithManager will setup the manager to manage workload @@ -89,16 +90,15 @@ func (r *workloadWebhook) ValidateCreate(ctx context.Context, workload *computev // that means for the scheduling phase, since there would not currently be // sufficient context to know who created the workload and what locations // are valid candidates based on that. Maybe an annotation, or spec field? - var locations networkingv1alpha.LocationBindingList + var locations locationsv1alpha1.LocationList if err := clusterClient.List(ctx, &locations); err != nil { - return nil, fmt.Errorf("failed to list location bindings: %w", err) + return nil, fmt.Errorf("failed to list locations: %w", err) } - validCityCodes := sets.Set[string]{} + validLocations := sets.Set[string]{} for _, location := range locations.Items { - cityCode, ok := location.Spec.Topology[networkingv1alpha.TopologyCityCodeKey] - if ok { - validCityCodes.Insert(cityCode) + if apimeta.IsStatusConditionTrue(location.Status.Conditions, locationsv1alpha1.LocationConditionReady) { + validLocations.Insert(location.Name) } } @@ -107,7 +107,7 @@ func (r *workloadWebhook) ValidateCreate(ctx context.Context, workload *computev Client: clusterClient, AdmissionRequest: req, Workload: workload, - ValidCityCodes: sets.List(validCityCodes), + ValidLocations: sets.List(validLocations), } if errs := validation.ValidateWorkloadCreate(workload, opts); len(errs) > 0 { @@ -134,16 +134,15 @@ func (r *workloadWebhook) ValidateUpdate(ctx context.Context, _ *computev1alpha. return nil, err } - var locations networkingv1alpha.LocationBindingList + var locations locationsv1alpha1.LocationList if err := clusterClient.List(ctx, &locations); err != nil { - return nil, fmt.Errorf("failed to list location bindings: %w", err) + return nil, fmt.Errorf("failed to list locations: %w", err) } - validCityCodes := sets.Set[string]{} + validLocations := sets.Set[string]{} for _, location := range locations.Items { - cityCode, ok := location.Spec.Topology[networkingv1alpha.TopologyCityCodeKey] - if ok { - validCityCodes.Insert(cityCode) + if apimeta.IsStatusConditionTrue(location.Status.Conditions, locationsv1alpha1.LocationConditionReady) { + validLocations.Insert(location.Name) } } @@ -152,7 +151,7 @@ func (r *workloadWebhook) ValidateUpdate(ctx context.Context, _ *computev1alpha. Client: clusterClient, AdmissionRequest: req, Workload: newWorkload, - ValidCityCodes: sets.List(validCityCodes), + ValidLocations: sets.List(validLocations), } if errs := validation.ValidateWorkloadCreate(newWorkload, opts); len(errs) > 0 { diff --git a/test/e2e/chainsaw-config.yaml b/test/e2e/chainsaw-config.yaml index 9524c8e4..0656f624 100644 --- a/test/e2e/chainsaw-config.yaml +++ b/test/e2e/chainsaw-config.yaml @@ -44,9 +44,9 @@ spec: # and Instance write-backs live here. downstream: kubeconfig: tmp/e2e/kubeconfigs/downstream.yaml - # POP DFW cell — downstream member cluster labelled topology.datum.net/city-code=dfw. + # POP DFW cell — downstream member cluster labelled topology.datum.net/location=dfw. pop-dfw: kubeconfig: tmp/e2e/kubeconfigs/pop-dfw.yaml - # POP ORD cell — downstream member cluster labelled topology.datum.net/city-code=ord. + # POP ORD cell — downstream member cluster labelled topology.datum.net/location=ord. pop-ord: kubeconfig: tmp/e2e/kubeconfigs/pop-ord.yaml diff --git a/test/e2e/deletion-cascade/workload-deployment.yaml b/test/e2e/deletion-cascade/workload-deployment.yaml index 39d68a1d..8df72f5f 100644 --- a/test/e2e/deletion-cascade/workload-deployment.yaml +++ b/test/e2e/deletion-cascade/workload-deployment.yaml @@ -3,7 +3,8 @@ kind: WorkloadDeployment metadata: name: test-cascade-wd spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/federation-finalizer-no-wedge/chainsaw-test.yaml b/test/e2e/federation-finalizer-no-wedge/chainsaw-test.yaml index 962783ca..811d8a21 100644 --- a/test/e2e/federation-finalizer-no-wedge/chainsaw-test.yaml +++ b/test/e2e/federation-finalizer-no-wedge/chainsaw-test.yaml @@ -50,7 +50,8 @@ spec: name: test-no-wedge-wd namespace: ${VICTIM_NS} spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/federation-hub-reclaim/chainsaw-test.yaml b/test/e2e/federation-hub-reclaim/chainsaw-test.yaml index bbb10b9a..a65e1109 100644 --- a/test/e2e/federation-hub-reclaim/chainsaw-test.yaml +++ b/test/e2e/federation-hub-reclaim/chainsaw-test.yaml @@ -107,7 +107,8 @@ spec: name: test-hub-reclaim-wd namespace: ${INSTANCE_NS} spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/federation-hub-reclaim/instance-pop-dfw.yaml b/test/e2e/federation-hub-reclaim/instance-pop-dfw.yaml index 4a19f0e5..d4d5ab65 100644 --- a/test/e2e/federation-hub-reclaim/instance-pop-dfw.yaml +++ b/test/e2e/federation-hub-reclaim/instance-pop-dfw.yaml @@ -17,7 +17,7 @@ metadata: compute.datumapis.com/workload-deployment-uid: "00000000-0000-0000-0000-000000000002" compute.datumapis.com/instance-index: "0" compute.datumapis.com/workload-deployment-name: test-hub-reclaim-wd - compute.datumapis.com/city-code: dfw + compute.datumapis.com/location: dfw compute.datumapis.com/workload-name: test-workload compute.datumapis.com/placement-name: default spec: diff --git a/test/e2e/federation-ownership/workload-deployment.yaml b/test/e2e/federation-ownership/workload-deployment.yaml index e23a50ce..6c2bc8b9 100644 --- a/test/e2e/federation-ownership/workload-deployment.yaml +++ b/test/e2e/federation-ownership/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: test-fed-own-wd # namespace is injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/federation-reclaim-without-cell/workload-deployment.yaml b/test/e2e/federation-reclaim-without-cell/workload-deployment.yaml index 7ae2ee9f..748e0a60 100644 --- a/test/e2e/federation-reclaim-without-cell/workload-deployment.yaml +++ b/test/e2e/federation-reclaim-without-cell/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: test-no-cell-wd # namespace is injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/full-federation-ord/chainsaw-test.yaml b/test/e2e/full-federation-ord/chainsaw-test.yaml index b1bf1b64..6a19f8d4 100644 --- a/test/e2e/full-federation-ord/chainsaw-test.yaml +++ b/test/e2e/full-federation-ord/chainsaw-test.yaml @@ -8,14 +8,14 @@ spec: The dfw path is covered by full-federation. This suite proves the same chain routes independently to the OTHER cell: an ord-placed WorkloadDeployment must - produce PropagationPolicy city-ord (not city-dfw), propagate to pop-ord (not + produce PropagationPolicy location-ord (not location-dfw), propagate to pop-ord (not pop-dfw), and create its Instance on pop-ord. Both cells run the same operator - image, so this is the coverage that catches city-code routing regressions and + image, so this is the coverage that catches location routing regressions and a mis-registered second cell. - 1. Create WorkloadDeployment (cityCode: ord) on control-plane. + 1. Create WorkloadDeployment (locationRef.name: ord) on control-plane. 2. WorkloadDeploymentFederator replicates it to Karmada (ns- namespace) - and lazily creates PropagationPolicy city-ord routing to city-code=ord cells. + and lazily creates PropagationPolicy location-ord routing to location=ord cells. 3. Karmada propagates the WD to pop-ord. 4. WorkloadDeploymentReconciler on pop-ord creates Instance test-fullfed-ord-wd-0. 5. InstanceReconciler on pop-ord writes the Instance back to Karmada with @@ -39,7 +39,7 @@ spec: file: workload-deployment.yaml - name: assert-wd-and-policy-in-downstream - description: Assert the WD federated to Karmada and PropagationPolicy city-ord was created for ord. + description: Assert the WD federated to Karmada and PropagationPolicy location-ord was created for ord. cluster: downstream try: - script: @@ -59,17 +59,17 @@ spec: namespace: ($downstreamNS) name: test-fullfed-ord-wd labels: - topology.datum.net/city-code: ord + topology.datum.net/location: ord - assert: - # The federator names the policy city- and routes it to cells - # carrying the same city-code label, so ord must land on pop-ord alone. + # The federator names the policy location- and routes it to cells + # carrying the same location label, so ord must land on pop-ord alone. timeout: 120s resource: apiVersion: policy.karmada.io/v1alpha1 kind: PropagationPolicy metadata: namespace: ($downstreamNS) - name: city-ord + name: location-ord spec: # Three selectors: the WorkloadDeployment plus the always-on ConfigMap # and Secret referenced-data selectors. Chainsaw matches list length @@ -79,7 +79,7 @@ spec: kind: WorkloadDeployment labelSelector: matchLabels: - topology.datum.net/city-code: ord + topology.datum.net/location: ord - apiVersion: v1 kind: ConfigMap labelSelector: @@ -94,7 +94,7 @@ spec: clusterAffinity: labelSelector: matchLabels: - topology.datum.net/city-code: ord + topology.datum.net/location: ord - name: assert-wd-on-pop-ord description: Assert Karmada propagated the WD to pop-ord and the cell reconciler set status. diff --git a/test/e2e/full-federation-ord/workload-deployment.yaml b/test/e2e/full-federation-ord/workload-deployment.yaml index 74663331..42097bc0 100644 --- a/test/e2e/full-federation-ord/workload-deployment.yaml +++ b/test/e2e/full-federation-ord/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: test-fullfed-ord-wd # namespace is injected by Chainsaw from ($namespace) spec: - cityCode: ord + locationRef: + name: ord placementName: default workloadRef: name: test-workload diff --git a/test/e2e/full-federation/workload-deployment.yaml b/test/e2e/full-federation/workload-deployment.yaml index 70b4cb94..f130351b 100644 --- a/test/e2e/full-federation/workload-deployment.yaml +++ b/test/e2e/full-federation/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: test-full-fed-wd # namespace is injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/instance-projection/workload-deployment.yaml b/test/e2e/instance-projection/workload-deployment.yaml index 4d2d5979..580467e4 100644 --- a/test/e2e/instance-projection/workload-deployment.yaml +++ b/test/e2e/instance-projection/workload-deployment.yaml @@ -3,7 +3,8 @@ kind: WorkloadDeployment metadata: name: test-projector-wd spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/instance-writeback/chainsaw-test.yaml b/test/e2e/instance-writeback/chainsaw-test.yaml index 537a5026..ada22886 100644 --- a/test/e2e/instance-writeback/chainsaw-test.yaml +++ b/test/e2e/instance-writeback/chainsaw-test.yaml @@ -108,7 +108,8 @@ spec: name: test-writeback-wd namespace: ${INSTANCE_NS} spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/instance-writeback/instance-pop-dfw.yaml b/test/e2e/instance-writeback/instance-pop-dfw.yaml index aa5f11b2..c147a879 100644 --- a/test/e2e/instance-writeback/instance-pop-dfw.yaml +++ b/test/e2e/instance-writeback/instance-pop-dfw.yaml @@ -16,7 +16,7 @@ metadata: compute.datumapis.com/workload-deployment-uid: "00000000-0000-0000-0000-000000000002" compute.datumapis.com/instance-index: "0" compute.datumapis.com/workload-deployment-name: test-writeback-wd - compute.datumapis.com/city-code: dfw + compute.datumapis.com/location: dfw compute.datumapis.com/workload-name: test-workload compute.datumapis.com/placement-name: default spec: diff --git a/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml b/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml index 5678c398..66f9aba9 100644 --- a/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml +++ b/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml @@ -6,13 +6,13 @@ spec: description: | Verifies the PropagationPolicy lifecycle managed by the WorkloadDeploymentFederator: - - A PropagationPolicy (city-dfw) is lazily created when the first WorkloadDeployment + - A PropagationPolicy (location-dfw) is lazily created when the first WorkloadDeployment for city code "dfw" is federated to Karmada. - The PropagationPolicy is RETAINED while at least one WorkloadDeployment for that city code remains in the Karmada namespace. - The PropagationPolicy is DELETED when the last deployment for the city is removed. - The test creates two WDs (wd-alpha, wd-beta) both targeting cityCode=dfw, verifies + The test creates two WDs (wd-alpha, wd-beta) both targeting location dfw, verifies the PP appears, deletes wd-alpha and asserts the PP is still present, then deletes wd-beta and waits for the PP to disappear. @@ -66,7 +66,7 @@ spec: kind: PropagationPolicy metadata: namespace: ($downstreamNS) - name: city-dfw + name: location-dfw - name: delete-alpha description: Delete wd-alpha; wd-beta still targets dfw so the PP must be retained. @@ -99,7 +99,7 @@ spec: kind: PropagationPolicy metadata: namespace: ($downstreamNS) - name: city-dfw + name: location-dfw - name: delete-beta description: Delete wd-beta (the last WD for city dfw). @@ -127,7 +127,7 @@ spec: apiVersion: policy.karmada.io/v1alpha1 kind: PropagationPolicy namespace: ($downstreamNS) - name: city-dfw + name: location-dfw timeout: 30s for: deletion: {} diff --git a/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml b/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml index f9eb27fd..527ec9b3 100644 --- a/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml +++ b/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml @@ -3,7 +3,8 @@ kind: WorkloadDeployment metadata: name: wd-alpha spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml b/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml index fd1d65c1..30267a8d 100644 --- a/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml +++ b/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml @@ -3,7 +3,8 @@ kind: WorkloadDeployment metadata: name: wd-beta spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml b/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml index 99534bae..d902239c 100644 --- a/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml +++ b/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: gc-test-wd # namespace injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: gc-test-workload diff --git a/test/e2e/referenced-data-mounts/chainsaw-test.yaml b/test/e2e/referenced-data-mounts/chainsaw-test.yaml index bcef1234..dd3a94d3 100644 --- a/test/e2e/referenced-data-mounts/chainsaw-test.yaml +++ b/test/e2e/referenced-data-mounts/chainsaw-test.yaml @@ -18,7 +18,7 @@ spec: Companions are NOT written to the control-plane cluster. [Hop 3] Karmada hub holds the companion ConfigMap + Secret in ns-{project-uid}, - propagated by the always-on label selector in the city-dfw + propagated by the always-on label selector in the location-dfw PropagationPolicy (which includes ConfigMap and Secret selectors). [Hop 4] Karmada propagates the WD + companions to pop-dfw. @@ -197,7 +197,7 @@ spec: - name: assert-propagation-policy-has-companion-selectors description: | - Assert the PropagationPolicy city-dfw on the hub includes ConfigMap and + Assert the PropagationPolicy location-dfw on the hub includes ConfigMap and Secret resource selectors so companions co-propagate with the WD. cluster: downstream try: @@ -216,7 +216,7 @@ spec: kind: PropagationPolicy metadata: namespace: ($downstreamNS) - name: city-dfw + name: location-dfw spec: resourceSelectors: - apiVersion: compute.datumapis.com/v1alpha @@ -224,7 +224,7 @@ spec: namespace: ($downstreamNS) labelSelector: matchLabels: - topology.datum.net/city-code: dfw + topology.datum.net/location: dfw - apiVersion: v1 kind: ConfigMap namespace: ($downstreamNS) diff --git a/test/e2e/referenced-data-mounts/workload-deployment.yaml b/test/e2e/referenced-data-mounts/workload-deployment.yaml index d849be0b..78df8da6 100644 --- a/test/e2e/referenced-data-mounts/workload-deployment.yaml +++ b/test/e2e/referenced-data-mounts/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: test-refdata-wd # namespace injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/referenced-data-pull-secret/chainsaw-test.yaml b/test/e2e/referenced-data-pull-secret/chainsaw-test.yaml index 77805dca..96eec719 100644 --- a/test/e2e/referenced-data-pull-secret/chainsaw-test.yaml +++ b/test/e2e/referenced-data-pull-secret/chainsaw-test.yaml @@ -18,7 +18,7 @@ spec: ReferencedDataController materialises a companion Secret in ns-{project-uid} on the Karmada hub and lists it in the expected-referenced-data annotation. - [Hop 3] Karmada propagates the companion to pop-dfw via the city-dfw + [Hop 3] Karmada propagates the companion to pop-dfw via the location-dfw PropagationPolicy Secret selector (label-based, source-agnostic). [Hop 4] The Instance's ReferencedData scheduling gate clears only once the credential is present on the cell. diff --git a/test/e2e/referenced-data-pull-secret/workload-deployment-no-pull-secret.yaml b/test/e2e/referenced-data-pull-secret/workload-deployment-no-pull-secret.yaml index 19c6eed1..d65773f2 100644 --- a/test/e2e/referenced-data-pull-secret/workload-deployment-no-pull-secret.yaml +++ b/test/e2e/referenced-data-pull-secret/workload-deployment-no-pull-secret.yaml @@ -4,7 +4,8 @@ metadata: name: test-pullsecret-wd # namespace injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/referenced-data-pull-secret/workload-deployment.yaml b/test/e2e/referenced-data-pull-secret/workload-deployment.yaml index f32f012f..e4d1a24c 100644 --- a/test/e2e/referenced-data-pull-secret/workload-deployment.yaml +++ b/test/e2e/referenced-data-pull-secret/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: test-pullsecret-wd # namespace injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/suspension/workload-deployment-suspended.yaml b/test/e2e/suspension/workload-deployment-suspended.yaml index 40aa416e..b9505ac7 100644 --- a/test/e2e/suspension/workload-deployment-suspended.yaml +++ b/test/e2e/suspension/workload-deployment-suspended.yaml @@ -6,7 +6,8 @@ metadata: annotations: compute.datumapis.com/suspended: "true" spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/suspension/workload-deployment.yaml b/test/e2e/suspension/workload-deployment.yaml index 213ed8f4..883599f6 100644 --- a/test/e2e/suspension/workload-deployment.yaml +++ b/test/e2e/suspension/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: test-suspend-wd # namespace is injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/e2e/workload-deployment-federation/chainsaw-test.yaml b/test/e2e/workload-deployment-federation/chainsaw-test.yaml index 8d589310..6659243b 100644 --- a/test/e2e/workload-deployment-federation/chainsaw-test.yaml +++ b/test/e2e/workload-deployment-federation/chainsaw-test.yaml @@ -6,7 +6,7 @@ spec: description: | Verifies that the WorkloadDeploymentFederator replicates a WorkloadDeployment from the project namespace (control-plane cluster) to the Karmada API server - with the correct city-code label and PropagationPolicy. + with the correct location label and PropagationPolicy. The federator follows the ns- convention for Karmada namespaces, matching the MappedNamespaceResourceStrategy used by NSO. The test derives @@ -14,9 +14,9 @@ spec: Verified: - WorkloadDeployment exists in Karmada at ns- - - Karmada copy carries label topology.datum.net/city-code: dfw - - PropagationPolicy city-dfw exists in the Karmada namespace, - selecting WDs by city-code and routing them to matching POP-cell clusters. + - Karmada copy carries label topology.datum.net/location: dfw + - PropagationPolicy location-dfw exists in the Karmada namespace, + selecting WDs by location and routing them to matching POP-cell clusters. template: true @@ -28,7 +28,7 @@ spec: file: workload-deployment.yaml - name: assert-wd-in-downstream - description: Assert WorkloadDeployment federated to Karmada with city-code label. + description: Assert WorkloadDeployment federated to Karmada with location label. cluster: downstream try: - script: @@ -48,10 +48,10 @@ spec: namespace: ($downstreamNS) name: test-federation-wd labels: - topology.datum.net/city-code: dfw + topology.datum.net/location: dfw - name: assert-propagation-policy-in-downstream - description: Assert PropagationPolicy created for city-dfw. + description: Assert PropagationPolicy created for location-dfw. cluster: downstream try: - script: @@ -69,7 +69,7 @@ spec: kind: PropagationPolicy metadata: namespace: ($downstreamNS) - name: city-dfw + name: location-dfw spec: # The federator emits three resource selectors, not one: the # WorkloadDeployment plus the always-on ConfigMap and Secret @@ -80,7 +80,7 @@ spec: kind: WorkloadDeployment labelSelector: matchLabels: - topology.datum.net/city-code: dfw + topology.datum.net/location: dfw - apiVersion: v1 kind: ConfigMap labelSelector: @@ -95,4 +95,4 @@ spec: clusterAffinity: labelSelector: matchLabels: - topology.datum.net/city-code: dfw + topology.datum.net/location: dfw diff --git a/test/e2e/workload-deployment-federation/workload-deployment.yaml b/test/e2e/workload-deployment-federation/workload-deployment.yaml index 0cd2347a..739d2f6d 100644 --- a/test/e2e/workload-deployment-federation/workload-deployment.yaml +++ b/test/e2e/workload-deployment-federation/workload-deployment.yaml @@ -4,7 +4,8 @@ metadata: name: test-federation-wd # namespace is injected by Chainsaw from ($namespace) spec: - cityCode: dfw + locationRef: + name: dfw placementName: default workloadRef: name: test-workload diff --git a/test/interpreter/workloaddeployment-retain-desired.yaml b/test/interpreter/workloaddeployment-retain-desired.yaml index 73c75383..4229e1ff 100644 --- a/test/interpreter/workloaddeployment-retain-desired.yaml +++ b/test/interpreter/workloaddeployment-retain-desired.yaml @@ -7,7 +7,8 @@ spec: workloadRef: name: test placementName: default - cityCode: DFW + locationRef: + name: us-east-1 template: {} scaleSettings: minReplicas: 1 diff --git a/test/interpreter/workloaddeployment-retain-observed.yaml b/test/interpreter/workloaddeployment-retain-observed.yaml index f1f2d36b..a7c6af92 100644 --- a/test/interpreter/workloaddeployment-retain-observed.yaml +++ b/test/interpreter/workloaddeployment-retain-observed.yaml @@ -7,7 +7,8 @@ spec: workloadRef: name: test placementName: default - cityCode: DFW + locationRef: + name: us-east-1 template: {} scaleSettings: minReplicas: 1 diff --git a/ui/consumer/src/adapter.ts b/ui/consumer/src/adapter.ts index 64dbb50c..73412e40 100644 --- a/ui/consumer/src/adapter.ts +++ b/ui/consumer/src/adapter.ts @@ -61,13 +61,13 @@ interface RawRuntime { export const INSTANCE_LABELS = { workloadName: 'compute.datumapis.com/workload-name', workloadUid: 'compute.datumapis.com/workload-uid', - cityCode: 'compute.datumapis.com/city-code', + location: 'compute.datumapis.com/location', placementName: 'compute.datumapis.com/placement-name', } as const; interface RawWorkloadPlacement { name: string; - cityCodes?: string[]; + locations?: Array<{ name: string }>; scaleSettings?: { minReplicas?: number; maxReplicas?: number }; } @@ -209,7 +209,7 @@ function toPlacementRegions( return { name: p.name, - cityCodes: p.cityCodes ?? [], + locations: (p.locations ?? []).map((location) => location.name), readyReplicas: ready, desiredReplicas: desired, health, @@ -243,7 +243,7 @@ export function toWorkload(raw: RawWorkload): Workload { runtimeType: runtime ? (runtime.sandbox ? 'Container sandbox' : 'Virtual machine') : undefined, tags: deriveTags(runtime), ports, - regions: Array.from(new Set(placements.flatMap((p) => p.cityCodes ?? []))), + locations: Array.from(new Set(placements.flatMap((p) => (p.locations ?? []).map((location) => location.name)))), resources: deriveResources(runtime), replicasPerRegion: deriveReplicasPerRegion(placements), conditions: conditions.map((c) => ({ @@ -344,7 +344,7 @@ export function toInstance(raw: RawInstance): Instance { : new Date(), workloadName: labels[INSTANCE_LABELS.workloadName], workloadUid: labels[INSTANCE_LABELS.workloadUid], - city: labels[INSTANCE_LABELS.cityCode], + location: labels[INSTANCE_LABELS.location], placement: labels[INSTANCE_LABELS.placementName], instanceType: raw.spec?.runtime?.resources?.instanceType, cpu, diff --git a/ui/consumer/src/components/instance-page-chrome.tsx b/ui/consumer/src/components/instance-page-chrome.tsx index 1e4d19b9..606cccdf 100644 --- a/ui/consumer/src/components/instance-page-chrome.tsx +++ b/ui/consumer/src/components/instance-page-chrome.tsx @@ -101,12 +101,12 @@ export function InstancePageChrome({ aria-hidden /> {instance.status} - {instance.city ? ( + {instance.location ? ( <> · - {instance.city} + {instance.location} ) : null} diff --git a/ui/consumer/src/pages/workload-detail.tsx b/ui/consumer/src/pages/workload-detail.tsx index ffd27969..04da275f 100644 --- a/ui/consumer/src/pages/workload-detail.tsx +++ b/ui/consumer/src/pages/workload-detail.tsx @@ -86,7 +86,7 @@ function InstanceCard({ instance, onClick }: { instance: Instance; onClick: () =

    {instance.name}

    -

    {instance.city ?? 'Unknown region'}

    +

    {instance.location ?? 'Unknown location'}

    {instance.status} @@ -211,14 +211,14 @@ function ConfigurationCard({ workload }: { workload: Workload }) { label: 'Replicas', content: workload.replicasPerRegion !== undefined - ? `${workload.replicasPerRegion}/region · ${workload.desiredReplicas} total` + ? `${workload.replicasPerRegion}/location · ${workload.desiredReplicas} total` : `${workload.desiredReplicas} total`, }, { - label: 'Regions', + label: 'Locations', content: - workload.regions.length > 0 ? ( - workload.regions.join(', ') + workload.locations.length > 0 ? ( + workload.locations.join(', ') ) : ( — ), @@ -252,7 +252,7 @@ export default function WorkloadDetail() { const totalCount = workload ? instances.length || workload.desiredReplicas : 0; const allHealthy = totalCount > 0 && healthyCount === totalCount; - const regions = workload?.regions ?? []; + const locations = workload?.locations ?? []; const stats: Stat[] | null = workload ? [ @@ -271,7 +271,7 @@ export default function WorkloadDetail() { ? 'text-green-600 dark:text-green-500' : undefined, }, - { label: 'Regions', value: String(regions.length) }, + { label: 'Locations', value: String(locations.length) }, { label: 'Requests', value: COMING_SOON, @@ -358,9 +358,9 @@ export default function WorkloadDetail() { Instance Locations

    - {regions.length > 0 - ? `Regions where this workload is deployed: ${regions.join(', ')}.` - : 'Regions where this workload is deployed.'} + {locations.length > 0 + ? `Locations where this workload is deployed: ${locations.join(', ')}.` + : 'Locations where this workload is deployed.'}

    diff --git a/ui/consumer/src/pages/workload-list.tsx b/ui/consumer/src/pages/workload-list.tsx index 7cd1e1ca..a86a84c3 100644 --- a/ui/consumer/src/pages/workload-list.tsx +++ b/ui/consumer/src/pages/workload-list.tsx @@ -62,7 +62,7 @@ function statusLabel(workload: Workload): string { } function regionLabel(region: WorkloadPlacementRegion): string { - if (region.cityCodes.length > 0) return region.cityCodes.join(", "); + if (region.locations.length > 0) return region.locations.join(", "); return region.name; } @@ -232,7 +232,7 @@ function WorkloadCard({ {workload.placementRegions.length > 0 && (

    - Regions + Locations

    {workload.placementRegions.map((region) => (
    {isLoading && } diff --git a/ui/consumer/src/schema.ts b/ui/consumer/src/schema.ts index 9f550a71..395ef05c 100644 --- a/ui/consumer/src/schema.ts +++ b/ui/consumer/src/schema.ts @@ -22,7 +22,7 @@ const workloadConditionSchema = z.object({ export const workloadPlacementRegionSchema = z.object({ name: z.string(), - cityCodes: z.array(z.string()).default([]), + locations: z.array(z.string()).default([]), readyReplicas: z.number(), desiredReplicas: z.number(), health: z.enum(['Available', 'Degraded', 'Unavailable', 'Unknown']), @@ -54,7 +54,7 @@ export const workloadResourceSchema = z.object({ runtimeType: z.string().optional(), tags: z.array(z.string()).default([]), ports: z.array(z.string()).default([]), - regions: z.array(z.string()).default([]), + locations: z.array(z.string()).default([]), resources: z.string().optional(), replicasPerRegion: z.number().optional(), }); @@ -107,7 +107,7 @@ export const instanceResourceSchema = z.object({ createdAt: z.coerce.date(), workloadName: z.string().optional(), workloadUid: z.string().optional(), - city: z.string().optional(), + location: z.string().optional(), placement: z.string().optional(), instanceType: z.string().optional(), /** Allocated CPU — from requests, or resolved from instanceType catalog (e.g. "1"). */ diff --git a/ui/provider/src/adapter.ts b/ui/provider/src/adapter.ts index a591dfbc..d5bd9dc5 100644 --- a/ui/provider/src/adapter.ts +++ b/ui/provider/src/adapter.ts @@ -74,7 +74,7 @@ export const INSTANCE_LABELS = { interface RawWorkloadPlacement { name: string; - cityCodes?: string[]; + locations?: Array<{ name: string }>; scaleSettings?: { minReplicas?: number; maxReplicas?: number }; } @@ -163,7 +163,7 @@ function toPlacements( return { name: p.name, - cityCodes: p.cityCodes ?? [], + locations: (p.locations ?? []).map((location) => location.name), readyReplicas: ready, desiredReplicas: desired, currentReplicas: current, @@ -194,7 +194,7 @@ export function toWorkload(raw: RawWorkload): Workload { placements: toPlacements(placements, raw.status?.placements ?? []), conditions: toConditions(conditions), runtimeType: runtime ? (runtime.sandbox ? 'Container sandbox' : 'Virtual machine') : undefined, - regions: Array.from(new Set(placements.flatMap((p) => p.cityCodes ?? []))), + locations: Array.from(new Set(placements.flatMap((p) => (p.locations ?? []).map((location) => location.name)))), resources: deriveResources(runtime), replicasPerRegion: deriveReplicasPerRegion(placements), }; @@ -274,7 +274,7 @@ export function toInstance(raw: RawInstance): Instance { createdAt: raw.metadata?.creationTimestamp ? new Date(raw.metadata.creationTimestamp) : new Date(), - city: labels['compute.datumapis.com/city-code'], + location: labels['compute.datumapis.com/location'], placement: labels['compute.datumapis.com/placement-name'], instanceType: raw.spec?.runtime?.resources?.instanceType, cpu, diff --git a/ui/provider/src/pages/workload-detail.tsx b/ui/provider/src/pages/workload-detail.tsx index 7ae4ce64..2892ce2f 100644 --- a/ui/provider/src/pages/workload-detail.tsx +++ b/ui/provider/src/pages/workload-detail.tsx @@ -112,10 +112,10 @@ function ConfigurationCard({ workload }: { workload: Workload }) { label: 'Replicas', content: workload.replicasPerRegion !== undefined - ? `${workload.replicasPerRegion}/region · ${workload.desiredReplicas} total` + ? `${workload.replicasPerRegion}/location · ${workload.desiredReplicas} total` : `${workload.desiredReplicas} total`, }, - { label: 'Regions', content: workload.regions.join(', ') || '—' }, + { label: 'Locations', content: workload.locations.join(', ') || '—' }, ]} /> @@ -140,7 +140,7 @@ function PlacementsCard({ workload }: { workload: Workload }) {
    {p.name} - {p.cityCodes.join(', ') || 'no city codes'} + {p.locations.join(', ') || 'no locations'}
    @@ -175,7 +175,7 @@ function OverviewTab({ workload }: { workload: Workload }) { { label: 'Ready', value: `${workload.readyReplicas}/${workload.desiredReplicas}` }, { label: 'Current', value: `${workload.currentReplicas}/${workload.desiredReplicas}` }, { label: 'Updated', value: `${workload.updatedReplicas}/${workload.desiredReplicas}` }, - { label: 'Regions', value: String(workload.regions.length) }, + { label: 'Locations', value: String(workload.locations.length) }, ]; return ( @@ -211,7 +211,7 @@ function InstanceRow({ instance }: { instance: Instance }) { )}
    - {instance.city ?? '—'} + {instance.location ?? '—'} {instance.internalIP ?? '—'} @@ -248,7 +248,7 @@ function InstancesTab({ instances }: { instances: Instance[] }) { Instance - Region + Location Internal IP External IP Created diff --git a/ui/provider/src/pages/workload-list.tsx b/ui/provider/src/pages/workload-list.tsx index 34d0592a..32175b35 100644 --- a/ui/provider/src/pages/workload-list.tsx +++ b/ui/provider/src/pages/workload-list.tsx @@ -39,7 +39,7 @@ function WorkloadRow({ workload }: { workload: Workload }) { {workload.readyReplicas}/{workload.desiredReplicas} - {workload.regions.join(', ') || '—'} + {workload.locations.join(', ') || '—'} {formatDistanceToNowStrict(workload.createdAt, { addSuffix: true })} @@ -78,7 +78,7 @@ export default function WorkloadList() { Name Health Ready - Regions + Locations Created diff --git a/ui/provider/src/schema.ts b/ui/provider/src/schema.ts index 60a4ebc9..b062b645 100644 --- a/ui/provider/src/schema.ts +++ b/ui/provider/src/schema.ts @@ -25,7 +25,7 @@ export type Condition = z.infer; export const workloadPlacementSchema = z.object({ name: z.string(), - cityCodes: z.array(z.string()).default([]), + locations: z.array(z.string()).default([]), readyReplicas: z.number(), desiredReplicas: z.number(), currentReplicas: z.number(), @@ -49,7 +49,7 @@ export const workloadResourceSchema = z.object({ placements: z.array(workloadPlacementSchema).default([]), conditions: z.array(conditionSchema).default([]), runtimeType: z.string().optional(), - regions: z.array(z.string()).default([]), + locations: z.array(z.string()).default([]), resources: z.string().optional(), replicasPerRegion: z.number().optional(), }); @@ -89,7 +89,7 @@ export const instanceResourceSchema = z.object({ name: z.string(), namespace: z.string().optional(), createdAt: z.coerce.date(), - city: z.string().optional(), + location: z.string().optional(), placement: z.string().optional(), instanceType: z.string().optional(), cpu: z.string().optional(), From ed484c950da8a210f1207a66d2ea33df20a8a389 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 15:09:22 -0500 Subject: [PATCH 02/12] feat: select placement locations by topology Naming locations stays the default. A placement may instead carry a locationSelector, a label selector matched against the topology of the project's Ready locations (city code, region, and any other key the platform projects). Every matching location receives a deployment, and the set follows the fleet: the workload controller watches the project's placement locations, so a location that starts or stops matching is picked up or torn down without polling. - API: placements[].locationSelector, exactly one of locations or locationSelector (CEL and webhook); status.placements[].locations reports what each placement resolved to; NoMatchingLocations names a placement that resolves to nothing. - Admission rejects an empty or malformed selector and one that matches no Ready location, mirroring how an unknown location name is rejected. - CLI: deploy --location-selector, mutually exclusive with --location and --city; describe shows the selector and resolved locations. - Portal: placements show the selector and the locations it resolved to. The controller watches the project's placement locations so a selector follows the fleet. The watch is the first thing to watch LocationBindings, so the ClusterRole now grants it, and it is kept off any control plane that does not serve the kind: an informer whose list is rejected never syncs and blocks every controller on the manager. Test locations repeated across files are named constants so the linter accepts them. Co-Authored-By: Claude Fable 5.1 --- api/v1alpha/instance_types.go | 5 + api/v1alpha/workload_types.go | 21 +- api/v1alpha/zz_generated.deepcopy.go | 10 + cmd/main_test.go | 58 +++++ .../compute.datumapis.com_workloads.yaml | 76 ++++++- config/components/controller_rbac/role.yaml | 17 +- config/samples/location-selector-sandbox.yaml | 35 +++ docs/agent/llms-full.txt | 2 +- docs/api/workloads.md | 200 +++++++++++++++--- docs/enhancements/datumctl-compute-dx.md | 15 ++ internal/agent/catalog.go | 12 ++ internal/cmd/compute/deploy/deploy.go | 58 +++-- internal/cmd/compute/workloads/workloads.go | 22 +- .../controller/instance_projector_test.go | 2 +- .../stateful/runtime_class_labels_test.go | 3 +- .../stateful/stateful_control_test.go | 6 +- internal/controller/location_source_test.go | 8 +- .../referenceddata_controller_test.go | 4 +- internal/controller/workload_controller.go | 183 ++++++++++++++-- .../controller/workload_controller_test.go | 154 +++++++++++++- .../workload_placementlocation_guard_test.go | 133 ++++++++++++ ...ddeployment_federator_runtimeclass_test.go | 4 +- ...workloaddeployment_network_binding_test.go | 12 +- internal/locations/locations.go | 83 ++++++++ internal/locations/locations_test.go | 107 +++++++++- internal/validation/workload_validation.go | 59 +++++- .../validation/workload_validation_test.go | 89 ++++++++ internal/webhook/v1alpha/workload_webhook.go | 57 +++-- ui/consumer/src/adapter.ts | 56 ++++- ui/consumer/src/pages/workload-list.tsx | 1 + ui/consumer/src/schema.ts | 2 + ui/provider/src/adapter.ts | 56 ++++- ui/provider/src/pages/workload-detail.tsx | 2 +- ui/provider/src/schema.ts | 2 + 34 files changed, 1420 insertions(+), 134 deletions(-) create mode 100644 config/samples/location-selector-sandbox.yaml create mode 100644 internal/controller/workload_placementlocation_guard_test.go diff --git a/api/v1alpha/instance_types.go b/api/v1alpha/instance_types.go index 4bced10f..e2bdb35d 100644 --- a/api/v1alpha/instance_types.go +++ b/api/v1alpha/instance_types.go @@ -924,6 +924,11 @@ const ( // WorkloadReasonNoAvailableDeployments is set on a placement's Available // condition when no deployment in that placement is available. WorkloadReasonNoAvailableDeployments = "NoAvailableDeployments" + + // WorkloadReasonNoMatchingLocations is set on a placement's Available + // condition when none of the locations it names is Ready, or its selector + // matches no Ready location, so the placement has nowhere to run. + WorkloadReasonNoMatchingLocations = "NoMatchingLocations" ) type InstanceTemplateSpec struct { diff --git a/api/v1alpha/workload_types.go b/api/v1alpha/workload_types.go index 0ed67b15..09012f70 100644 --- a/api/v1alpha/workload_types.go +++ b/api/v1alpha/workload_types.go @@ -131,18 +131,31 @@ type WorkloadList struct { Items []Workload `json:"items"` } +// +kubebuilder:validation:XValidation:message="exactly one of locations or locationSelector must be set",rule="has(self.locations) != has(self.locationSelector)" type WorkloadPlacement struct { // The name of the placement // // +kubebuilder:validation:Required Name string `json:"name"` - // A list of locations where the instances should be deployed. + // The locations where the instances should be deployed, by name. Use this + // to pin a placement to specific locations. Exactly one of locations or + // locationSelector must be set. // - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:validation:MinItems=1 Locations []locationsv1alpha1.LocationReference `json:"locations,omitempty"` + // A selector over the topology of the locations available to the project, + // such as topology.datum.net/city-code or topology.datum.net/region. Every + // Ready location whose topology matches receives a deployment, and the set + // is re-evaluated as locations are added, removed, or change readiness. An + // empty selector is rejected rather than treated as matching every + // location. Exactly one of locations or locationSelector must be set. + // + // +kubebuilder:validation:Optional + LocationSelector *metav1.LabelSelector `json:"locationSelector,omitempty"` + // Scale settings such as minimum and maximum replica counts. // // +kubebuilder:validation:Required @@ -153,6 +166,10 @@ type WorkloadPlacementStatus struct { // The name of the placement Name string `json:"name"` + // The locations the placement currently resolves to: the Ready locations + // it names, or every Ready location its selector matches. + Locations []locationsv1alpha1.LocationReference `json:"locations,omitempty"` + // Represents the observations of a placement's current state. // Known condition types are: "Available", "Progressing" Conditions []metav1.Condition `json:"conditions,omitempty"` diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index 80493440..3c54b3bf 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -1356,6 +1356,11 @@ func (in *WorkloadPlacement) DeepCopyInto(out *WorkloadPlacement) { *out = make([]v1alpha1.LocationReference, len(*in)) copy(*out, *in) } + if in.LocationSelector != nil { + in, out := &in.LocationSelector, &out.LocationSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } in.ScaleSettings.DeepCopyInto(&out.ScaleSettings) } @@ -1372,6 +1377,11 @@ func (in *WorkloadPlacement) DeepCopy() *WorkloadPlacement { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadPlacementStatus) DeepCopyInto(out *WorkloadPlacementStatus) { *out = *in + if in.Locations != nil { + in, out := &in.Locations, &out.Locations + *out = make([]v1alpha1.LocationReference, len(*in)) + copy(*out, *in) + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) diff --git a/cmd/main_test.go b/cmd/main_test.go index 89fc4f30..db4d7189 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -5,12 +5,15 @@ package main import ( "os" "path/filepath" + "slices" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + rbacv1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/yaml" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" @@ -90,3 +93,58 @@ func TestServingLocationObjectIsRegistered(t *testing.T) { require.NoErrorf(t, err, "the watch object for source %q must be registered", source) } } + +// TestControllerRoleGrantsPlacementLocationWatches is the RBAC half of the +// locations dependency, and the regression guard for the watch the workload +// reconciler installs on a project's placement locations. +// +// A watch the shipped ClusterRole does not permit is worse than a missing +// feature. The informer retries the rejected list forever, that control plane's +// cache never syncs, and controller-runtime blocks every controller on the +// manager from starting — the workload reconciler, the referenced-data +// reconciler and the deployment federator all stall, so nothing is federated, +// no finalizer is written, and the pod stays Ready while reconciling nothing +// until cluster engagement times out and it restarts into the same state. +// +// Both sources are covered: the manager is deployed with one ClusterRole and +// locationSource is config, so whichever source a deployment selects has to be +// permitted by the role that ships. +func TestControllerRoleGrantsPlacementLocationWatches(t *testing.T) { + body, err := os.ReadFile(filepath.Join("..", "config", "components", "controller_rbac", "role.yaml")) + require.NoError(t, err) + + var role rbacv1.ClusterRole + require.NoError(t, yaml.Unmarshal(body, &role)) + + // granted reports whether the role permits the verb on the resource, taking + // the wildcards RBAC honours into account. + granted := func(group, resource, verb string) bool { + matches := func(values []string, want string) bool { + return slices.Contains(values, want) || slices.Contains(values, rbacv1.ResourceAll) + } + for _, rule := range role.Rules { + if len(rule.ResourceNames) > 0 { + continue + } + if matches(rule.APIGroups, group) && matches(rule.Resources, resource) && matches(rule.Verbs, verb) { + return true + } + } + return false + } + + // The kinds every source watches or lists, named as the API server names + // them in an RBAC rule. + for _, resource := range []struct{ group, name string }{ + {"networking.datumapis.com", "locationbindings"}, + {"networking.datumapis.com", "servinglocations"}, + {"locations.miloapis.com", "locations"}, + {"locations.miloapis.com", "servinglocations"}, + } { + for _, verb := range []string{"get", "list", "watch"} { + assert.Truef(t, granted(resource.group, resource.name, verb), + "the controller ClusterRole must grant %q on %s.%s: a watch it cannot list wedges the manager", + verb, resource.name, resource.group) + } + } +} diff --git a/config/base/crd/bases/compute.datumapis.com_workloads.yaml b/config/base/crd/bases/compute.datumapis.com_workloads.yaml index aa790489..465f6189 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloads.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloads.yaml @@ -71,9 +71,63 @@ spec: will live in, such as in a city, or region. items: properties: + locationSelector: + description: |- + A selector over the topology of the locations available to the project, + such as topology.datum.net/city-code or topology.datum.net/region. Every + Ready location whose topology matches receives a deployment, and the set + is re-evaluated as locations are added, removed, or change readiness. An + empty selector is rejected rather than treated as matching every + location. Exactly one of locations or locationSelector must be set. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic locations: - description: A list of locations where the instances should - be deployed. + description: |- + The locations where the instances should be deployed, by name. Use this + to pin a placement to specific locations. Exactly one of locations or + locationSelector must be set. items: properties: name: @@ -155,10 +209,13 @@ spec: - minReplicas type: object required: - - locations - name - scaleSettings type: object + x-kubernetes-validations: + - message: exactly one of locations or locationSelector must be + set + rule: has(self.locations) != has(self.locationSelector) minItems: 1 type: array template: @@ -1711,6 +1768,19 @@ spec: description: The desired number of instances format: int32 type: integer + locations: + description: |- + The locations the placement currently resolves to: the Ready locations + it names, or every Ready location its selector matches. + items: + properties: + name: + description: Name of a datum location + type: string + required: + - name + type: object + type: array name: description: The name of the placement type: string diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index 574a646c..5e9e0500 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -96,24 +96,25 @@ rules: - apiGroups: - networking.datumapis.com resources: - - networkinterfaceclaims + - locationbindings + - networkinterfaces + - networks + - servinglocations verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - networking.datumapis.com resources: - - networkinterfaces - - networks - - servinglocations + - networkinterfaceclaims verbs: + - create + - delete - get - list + - patch + - update - watch - apiGroups: - networking.datumapis.com diff --git a/config/samples/location-selector-sandbox.yaml b/config/samples/location-selector-sandbox.yaml new file mode 100644 index 00000000..fc41c7c2 --- /dev/null +++ b/config/samples/location-selector-sandbox.yaml @@ -0,0 +1,35 @@ +--- +# A placement that selects its locations by topology instead of naming them. +# Every Ready location in the project whose city code is DFW receives a +# deployment, and the set follows locations as they are added or removed. +apiVersion: compute.datumapis.com/v1alpha +kind: Workload +metadata: + name: location-selector-sample +spec: + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: netdata + image: docker.io/netdata/netdata:latest + networkInterfaces: + - network: + name: default + networkPolicy: + ingress: + - ports: + - port: 19999 + from: + - ipBlock: + cidr: 0.0.0.0/0 + placements: + - name: dfw + locationSelector: + matchLabels: + topology.datum.net/city-code: DFW + scaleSettings: + minReplicas: 1 diff --git a/docs/agent/llms-full.txt b/docs/agent/llms-full.txt index 60168738..24729af3 100644 --- a/docs/agent/llms-full.txt +++ b/docs/agent/llms-full.txt @@ -90,7 +90,7 @@ user-actionable Something in the workload the customer wrote, or in their project's setup. They can fix it. Examples: ImageUnavailable, InstanceCrashing, ConfigurationError, QuotaExceeded, SourceNotFound, SourceTooLarge, - NetworkNotFound. + NetworkNotFound, NoMatchingLocations. platform fault Datum's to fix. No change to the workload will help, and suggesting one wastes diff --git a/docs/api/workloads.md b/docs/api/workloads.md index 9bc9f8d7..13668af5 100644 --- a/docs/api/workloads.md +++ b/docs/api/workloads.md @@ -119,13 +119,6 @@ will live in, such as in a city, or region.
    - locations - []object - - A list of locations where the instances should be deployed.
    - - true - name string @@ -139,33 +132,27 @@ will live in, such as in a city, or region.
    Scale settings such as minimum and maximum replica counts.
    true - - - - -### Workload.spec.placements[index].locations[index] -[↩ Parent](#workloadspecplacementsindex) - - - - - - - - - - - - - - - - - + + + - + + + + + +
    NameTypeDescriptionRequired
    namestring
    locationSelectorobject - Name of a datum location
    + A selector over the topology of the locations available to the project, +such as topology.datum.net/city-code or topology.datum.net/region. Every +Ready location whose topology matches receives a deployment, and the set +is re-evaluated as locations are added, removed, or change readiness. An +empty selector is rejected rather than treated as matching every +location. Exactly one of locations or locationSelector must be set.
    		truefalse
    locations[]object + The locations where the instances should be deployed, by name. Use this +to pin a placement to specific locations. Exactly one of locations or +locationSelector must be set.
    +     		false
    @@ -332,6 +319,120 @@ the requested value of the resource for the instances.
    +### Workload.spec.placements[index].locationSelector +[↩ Parent](#workloadspecplacementsindex) + + + +A selector over the topology of the locations available to the project, +such as topology.datum.net/city-code or topology.datum.net/region. Every +Ready location whose topology matches receives a deployment, and the set +is re-evaluated as locations are added, removed, or change readiness. An +empty selector is rejected rather than treated as matching every +location. Exactly one of locations or locationSelector must be set. + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
    +     		false
    matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
    +     		false
    + + +### Workload.spec.placements[index].locationSelector.matchExpressions[index] +[↩ Parent](#workloadspecplacementsindexlocationselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    keystring + key is the label key that the selector applies to.
    +     		true
    operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
    +     		true
    values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
    +     		false
    + + +### Workload.spec.placements[index].locations[index] +[↩ Parent](#workloadspecplacementsindex) + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of a datum location
    +     		true
    + + ### Workload.spec.template [↩ Parent](#workloadspec) @@ -3199,6 +3300,14 @@ of readiness. Lags Replicas during a rolling update or restart.
    Known condition types are: "Available", "Progressing"
    false + + locations + []object + + The locations the placement currently resolves to: the Ready locations +it names, or every Ready location its selector matches.
    + + false @@ -3278,3 +3387,30 @@ with respect to the current state of the instance.
    false + + +### Workload.status.placements[index].locations[index] +[↩ Parent](#workloadstatusplacementsindex) + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of a datum location
    +     		true
    diff --git a/docs/enhancements/datumctl-compute-dx.md b/docs/enhancements/datumctl-compute-dx.md index a999febd..325d426a 100644 --- a/docs/enhancements/datumctl-compute-dx.md +++ b/docs/enhancements/datumctl-compute-dx.md @@ -79,6 +79,21 @@ Rollout complete in 47s. Saved workload config to ./workload.yaml — commit this file to manage deployments declaratively. ``` +Naming locations is the default. When a developer would rather describe where to run than list it, `--location-selector` selects every ready location whose topology matches, and the placement follows locations as they are added or removed: + +``` +$ datumctl compute deploy api \ + --image=ghcr.io/acme/api:1.4.2 \ + --location-selector='topology.datum.net/city-code=DFW' \ + --min=2 + +Resolving workload "api" in project acme-prod... + Workload does not exist — creating. + Placement "default": selector=[topology.datum.net/city-code=DFW], min=2 +``` + +The saved config records the selector, and `datumctl compute workloads describe api` shows which locations it currently resolves to. + If a developer prefers an interactive walk-through: ``` diff --git a/internal/agent/catalog.go b/internal/agent/catalog.go index 54aefd52..57b37855 100644 --- a/internal/agent/catalog.go +++ b/internal/agent/catalog.go @@ -514,6 +514,18 @@ var catalog = []ReasonInfo{ Remediation: "Look at what this placement created; the cause is there.", Skill: SkillWorkloadNotAvailable, }, + { + Reason: computev1alpha.WorkloadReasonNoMatchingLocations, + ConditionTypes: []string{computev1alpha.WorkloadAvailable}, + Actionability: ActionabilityUser, + Explanation: "This placement resolves to no location, so nothing was created for it. Either none " + + "of the locations it names is ready, or its location selector matches none of the " + + "project's ready locations.", + Remediation: "Compare the placement against the project's locations and their topology " + + "(city code, region). Name a location that exists, or widen the selector until it " + + "matches one.", + Skill: SkillWorkloadNotAvailable, + }, // Runtime classes. A workload selects a class to say how it should be // executed; the class is Datum's catalog object, and its Accepted status is diff --git a/internal/cmd/compute/deploy/deploy.go b/internal/cmd/compute/deploy/deploy.go index 5abd61c2..b0423017 100644 --- a/internal/cmd/compute/deploy/deploy.go +++ b/internal/cmd/compute/deploy/deploy.go @@ -33,6 +33,7 @@ type options struct { build string instanceType string locations []string + locationSelector string cities []string allMatchingLocations bool min int32 @@ -69,6 +70,9 @@ a fix is applied — same as 'datumctl compute build --fix'.`, # Build from another directory datumctl compute deploy api --build=./api --image=ghcr.io/acme/api:1.4.2 --location=us-east-1,eu-west-1 + # Select locations by topology instead of naming them + datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --location-selector='topology.datum.net/city-code=DFW' + # Interactive mode datumctl compute deploy @@ -85,6 +89,7 @@ a fix is applied — same as 'datumctl compute build --fix'.`, cmd.Flags().Lookup("build").NoOptDefVal = "." cmd.Flags().StringVar(&opts.instanceType, "instance-type", "datumcloud/d1-standard-2", "Instance type (e.g. datumcloud/d1-standard-2)") cmd.Flags().StringSliceVar(&opts.locations, "location", nil, "One or more locations to deploy to (e.g. us-east-1,eu-west-1)") + cmd.Flags().StringVar(&opts.locationSelector, "location-selector", "", "Select every location whose topology matches a label selector (e.g. 'topology.datum.net/city-code=DFW' or 'topology.datum.net/region in (us-east-1,eu-west-1)')") cmd.Flags().StringSliceVar(&opts.cities, "city", nil, "Resolve a city code to an available location (e.g. IAD)") cmd.Flags().BoolVar(&opts.allMatchingLocations, "all-matching-locations", false, "Deploy to every location matching --city") cmd.Flags().Int32Var(&opts.min, "min", 1, "Minimum number of instances per location") @@ -144,11 +149,25 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err if opts.image == "" { return fmt.Errorf("--image is required") } - if len(opts.locations) == 0 && len(opts.cities) == 0 { - return fmt.Errorf("--location is required (e.g. --location=us-east-1,eu-west-1); use --city to resolve a city code") + placementFlags := 0 + for _, set := range []bool{len(opts.locations) > 0, len(opts.cities) > 0, opts.locationSelector != ""} { + if set { + placementFlags++ + } + } + if placementFlags == 0 { + return fmt.Errorf("--location is required (e.g. --location=us-east-1,eu-west-1); use --city to resolve a city code, or --location-selector to select locations by topology") } - if len(opts.locations) > 0 && len(opts.cities) > 0 { - return fmt.Errorf("--location and --city are mutually exclusive") + if placementFlags > 1 { + return fmt.Errorf("--location, --city, and --location-selector are mutually exclusive") + } + var locationSelector *metav1.LabelSelector + if opts.locationSelector != "" { + parsed, err := metav1.ParseToLabelSelector(opts.locationSelector) + if err != nil { + return fmt.Errorf("invalid --location-selector %q: %w", opts.locationSelector, err) + } + locationSelector = parsed } instanceType := opts.instanceType if instanceType == "" { @@ -211,8 +230,9 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err } // All locations go into one "default" placement. placement := computev1alpha.WorkloadPlacement{ - Name: "default", - Locations: locationRefs, + Name: "default", + Locations: locationRefs, + LocationSelector: locationSelector, ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: opts.min, InstanceManagementPolicy: computev1alpha.OrderedReadyInstanceManagementPolicyType, @@ -241,8 +261,7 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err Placements: []computev1alpha.WorkloadPlacement{placement}, } - fmt.Fprintf(out, " Placement \"default\": locations=[%s], min=%d\n", - strings.Join(locations, ", "), opts.min) + fmt.Fprintf(out, " Placement \"default\": %s, min=%d\n", describePlacementLocations(placement), opts.min) // Prompt unless --yes or non-interactive. if !opts.yes && term.IsTerminal(int(os.Stdin.Fd())) { @@ -476,13 +495,11 @@ func manifestDiff(existing, desired computev1alpha.Workload) []string { lines = append(lines, fmt.Sprintf(" placement %q min replicas: %d → %d", name, op.ScaleSettings.MinReplicas, np.ScaleSettings.MinReplicas)) } - } else { - locationNames := make([]string, 0, len(np.Locations)) - for _, ref := range np.Locations { - locationNames = append(locationNames, ref.Name) + if before, after := describePlacementLocations(op), describePlacementLocations(np); before != after { + lines = append(lines, fmt.Sprintf(" placement %q: %s → %s", name, before, after)) } - lines = append(lines, fmt.Sprintf(" + new placement %q: locations=[%s]", - name, strings.Join(locationNames, ", "))) + } else { + lines = append(lines, fmt.Sprintf(" + new placement %q: %s", name, describePlacementLocations(np))) } } for name := range oldPlacements { @@ -519,3 +536,16 @@ func resolveCities(ctx context.Context, c client.Client, cities []string, all bo } return resolved, nil } + +// describePlacementLocations says where a placement runs the way the CLI +// prints it: the locations it names, or the selector it resolves through. +func describePlacementLocations(p computev1alpha.WorkloadPlacement) string { + if p.LocationSelector != nil { + return fmt.Sprintf("selector=[%s]", metav1.FormatLabelSelector(p.LocationSelector)) + } + names := make([]string, 0, len(p.Locations)) + for _, ref := range p.Locations { + names = append(names, ref.Name) + } + return fmt.Sprintf("locations=[%s]", strings.Join(names, ", ")) +} diff --git a/internal/cmd/compute/workloads/workloads.go b/internal/cmd/compute/workloads/workloads.go index 17d7f6e0..cf29dd9d 100644 --- a/internal/cmd/compute/workloads/workloads.go +++ b/internal/cmd/compute/workloads/workloads.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -376,12 +377,8 @@ func runDescribe(cmd *cobra.Command, args []string) error { if p.ScaleSettings.MaxReplicas != nil { maxStr = fmt.Sprintf("%d", *p.ScaleSettings.MaxReplicas) } - locationNames := make([]string, 0, len(p.Locations)) - for _, ref := range p.Locations { - locationNames = append(locationNames, ref.Name) - } - fmt.Fprintf(out, " %-10s locations: %-24s scale: %d..%s\n", - p.Name, strings.Join(locationNames, ", "), p.ScaleSettings.MinReplicas, maxStr) + fmt.Fprintf(out, " %-10s %-34s scale: %d..%s\n", + p.Name, placementLocationsSummary(p), p.ScaleSettings.MinReplicas, maxStr) // Per-location lines from deployments. for _, d := range deplsByPlacement[p.Name] { @@ -474,3 +471,16 @@ func formatEnvVar(e corev1.EnvVar) string { } return fmt.Sprintf("%-20s %s", e.Name, e.Value) } + +// placementLocationsSummary says where a placement runs: the locations it +// names, or the topology selector it resolves through. +func placementLocationsSummary(p computev1alpha.WorkloadPlacement) string { + if p.LocationSelector != nil { + return "selector: " + metav1.FormatLabelSelector(p.LocationSelector) + } + names := make([]string, 0, len(p.Locations)) + for _, ref := range p.Locations { + names = append(names, ref.Name) + } + return "locations: " + strings.Join(names, ", ") +} diff --git a/internal/controller/instance_projector_test.go b/internal/controller/instance_projector_test.go index c960ebdb..3c489469 100644 --- a/internal/controller/instance_projector_test.go +++ b/internal/controller/instance_projector_test.go @@ -93,7 +93,7 @@ func projTestWorkloadDeployment() *computev1alpha.WorkloadDeployment { UID: projTestWDUID, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - LocationRef: locationsv1alpha1.LocationReference{Name: "us-west-1"}, + LocationRef: locationsv1alpha1.LocationReference{Name: testWestLocationName}, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: "my-workload"}, ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, diff --git a/internal/controller/instancecontrol/stateful/runtime_class_labels_test.go b/internal/controller/instancecontrol/stateful/runtime_class_labels_test.go index cd0ac41e..9a6b9df2 100644 --- a/internal/controller/instancecontrol/stateful/runtime_class_labels_test.go +++ b/internal/controller/instancecontrol/stateful/runtime_class_labels_test.go @@ -17,6 +17,7 @@ import ( const ( testClassAzurite = "azurite" testClassBasalt = "basalt" + testLocationName = "loc-dfw-1" ) // TestInstanceLabels_RuntimeClassStamped verifies that an instance carries the @@ -72,7 +73,7 @@ func TestInstanceLabels_PreClassesSetIsUnchanged(t *testing.T) { v1alpha.WorkloadUIDLabel: "test-workload-uid", v1alpha.WorkloadDeploymentUIDLabel: "test-wd-uid", v1alpha.WorkloadDeploymentNameLabel: "test-pre-classes-labels", - v1alpha.LocationLabel: "loc-dfw-1", + v1alpha.LocationLabel: testLocationName, v1alpha.WorkloadNameLabel: "test-workload", v1alpha.PlacementNameLabel: "test-placement", labelServiceKey: labelServiceValue, diff --git a/internal/controller/instancecontrol/stateful/stateful_control_test.go b/internal/controller/instancecontrol/stateful/stateful_control_test.go index ced106cc..3105db5d 100644 --- a/internal/controller/instancecontrol/stateful/stateful_control_test.go +++ b/internal/controller/instancecontrol/stateful/stateful_control_test.go @@ -337,7 +337,7 @@ func TestInstanceLocation_SetFromDeploymentSpec(t *testing.T) { control := New() deployment := getWorkloadDeployment("test-location-set", 1) - deployment.Spec.LocationRef = locationsv1alpha1.LocationReference{Name: "loc-dfw-1"} + deployment.Spec.LocationRef = locationsv1alpha1.LocationReference{Name: testLocationName} var currentInstances []v1alpha.Instance actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) @@ -349,7 +349,7 @@ func TestInstanceLocation_SetFromDeploymentSpec(t *testing.T) { assert.True(t, ok) assert.NotNil(t, instance.Spec.Location, "Spec.Location must be set from deployment.spec.locationRef") - assert.Equal(t, "loc-dfw-1", instance.Spec.Location.Name) + assert.Equal(t, testLocationName, instance.Spec.Location.Name) } // TestLabelBackfill_NotReadyMatchingHash verifies that a not-Ready instance @@ -620,7 +620,7 @@ func getWorkloadDeployment(name string, minReplicas int32) *v1alpha.WorkloadDepl UID: "test-workload-uid", }, PlacementName: "test-placement", - LocationRef: locationsv1alpha1.LocationReference{Name: "loc-dfw-1"}, + LocationRef: locationsv1alpha1.LocationReference{Name: testLocationName}, ScaleSettings: v1alpha.HorizontalScaleSettings{ MinReplicas: minReplicas, InstanceManagementPolicy: v1alpha.OrderedReadyInstanceManagementPolicyType, diff --git a/internal/controller/location_source_test.go b/internal/controller/location_source_test.go index 162453e2..9025841e 100644 --- a/internal/controller/location_source_test.go +++ b/internal/controller/location_source_test.go @@ -86,7 +86,7 @@ func TestGetDeploymentsForWorkload_LocationsSource(t *testing.T) { Placements: []computev1alpha.WorkloadPlacement{ { Name: testDefaultPlacement, - Locations: []locationsv1alpha1.LocationReference{{Name: "dfw"}}, + Locations: []locationsv1alpha1.LocationReference{{Name: testLocationName}}, ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: 1, }, @@ -97,7 +97,7 @@ func TestGetDeploymentsForWorkload_LocationsSource(t *testing.T) { cl := fake.NewClientBuilder(). WithScheme(newLocationsServiceScheme()). - WithObjects(newLocationsServiceLocation("dfw", locSourceTestCityCode)). + WithObjects(newLocationsServiceLocation(testLocationName, locSourceTestCityCode)). WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). Build() @@ -107,7 +107,7 @@ func TestGetDeploymentsForWorkload_LocationsSource(t *testing.T) { require.NoError(t, err) require.Empty(t, orphaned) require.Len(t, desired, 1) - assert.Equal(t, "dfw", desired[0].Spec.LocationRef.Name) + assert.Equal(t, testLocationName, desired[0].Spec.LocationRef.Name) } // TestGetDeploymentsForWorkload_LocationsSourceIgnoresBindings verifies the @@ -126,7 +126,7 @@ func TestGetDeploymentsForWorkload_LocationsSourceIgnoresBindings(t *testing.T) cl := fake.NewClientBuilder(). WithScheme(newLocationsServiceScheme()). - WithObjects(newTestLocationBinding("dfw", locSourceTestCityCode)). + WithObjects(newTestLocationBinding(testLocationName, locSourceTestCityCode)). WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). Build() diff --git a/internal/controller/referenceddata_controller_test.go b/internal/controller/referenceddata_controller_test.go index 08b33615..fa60c530 100644 --- a/internal/controller/referenceddata_controller_test.go +++ b/internal/controller/referenceddata_controller_test.go @@ -1508,10 +1508,10 @@ func TestFederator_StatusSync_PreservesReferencedDataReadyCondition(t *testing.T ObjectMeta: metav1.ObjectMeta{ Name: testWDName, Namespace: testKarmadaNSStr, - Labels: map[string]string{locationLabel: "us-west-1"}, + Labels: map[string]string{locationLabel: testWestLocationName}, }, Spec: computev1alpha.WorkloadDeploymentSpec{ - LocationRef: locationsv1alpha1.LocationReference{Name: "us-west-1"}, + LocationRef: locationsv1alpha1.LocationReference{Name: testWestLocationName}, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: rdTestWorkloadName}, ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index c3bedc67..4e5d393c 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -33,6 +33,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/compute/internal/locations" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) const ( @@ -61,6 +62,7 @@ type WorkloadReconciler struct { // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloads/status,verbs=get;update;patch // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloads/finalizers,verbs=update // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=locationbindings,verbs=get;list;watch // +kubebuilder:rbac:groups=locations.miloapis.com,resources=locations,verbs=get;list;watch func (r *WorkloadReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { @@ -252,15 +254,20 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( // Reconcile placement status newWorkloadStatus.Placements = []computev1alpha.WorkloadPlacementStatus{} - // Sort placement names for deterministic iteration so that equal-priority - // deployments in different placements are compared in a stable order. - placementNames := make([]string, 0, len(placementDeployments)) + // Every placement in the spec is reported, including one that currently + // resolves to no location, so the reason it runs nowhere is visible. + // Deployments whose placement has left the spec are reported until they + // are gone. Names are sorted so that equal-priority deployments in + // different placements are compared in a stable order. + placementNames := sets.New[string]() + for _, placement := range workload.Spec.Placements { + placementNames.Insert(placement.Name) + } for name := range placementDeployments { - placementNames = append(placementNames, name) + placementNames.Insert(name) } - sort.Strings(placementNames) - for _, placementName := range placementNames { + for _, placementName := range sets.List(placementNames) { placementDeployments := placementDeployments[placementName] placementStatus := computev1alpha.WorkloadPlacementStatus{ Name: placementName, @@ -331,6 +338,21 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( placementStatus.UpdatedReplicas = updatedReplicas placementStatus.DesiredReplicas = desiredReplicas placementStatus.ReadyReplicas = readyReplicas + placementStatus.Locations = deploymentLocations(sortedDeployments) + + if len(sortedDeployments) == 0 { + // Nothing was created for this placement: none of the locations it + // names is Ready, or its selector matched no Ready location. The + // user resolves this by changing the placement, so it outranks the + // generic no-deployments answer. + placementAvailableCondition.Reason = computev1alpha.WorkloadReasonNoMatchingLocations + placementAvailableCondition.Message = "No Ready location matches this placement, so it has nowhere to run" + if p := workloadBlockingReasonPriority(placementAvailableCondition.Reason); p > worstPriority { + worstPriority = p + worstReason = placementAvailableCondition.Reason + worstMessage = fmt.Sprintf("Placement %q matches no Ready location", placementName) + } + } if foundAvailableDeployment { placementAvailableCondition.Status = metav1.ConditionTrue @@ -466,18 +488,19 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( // Remember this: namespace, name, err := cache.SplitMetaNamespaceKey(key) for _, placement := range workload.Spec.Placements { - for _, locationRef := range placement.Locations { - if !readyLocations.Has(locationRef.Name) { - // TODO(jreese) update status condition on placement if the - // location is unknown or not Ready. - continue - } + // A placement that resolves to nothing is reported on its status by + // reconcileWorkloadStatus rather than failing the workload here. + locationNames, err := resolvePlacementLocations(placement, placementLocations, readyLocations) + if err != nil { + return nil, nil, fmt.Errorf("placement %q: %w", placement.Name, err) + } + for _, locationName := range locationNames { // TODO(jreese) should we use GenerateName for deployments and identify // them via labels instead? Would help with race conditions on workload // recreation. - deploymentName := fmt.Sprintf("%s-%s-%s", workload.Name, placement.Name, strings.ToLower(locationRef.Name)) + deploymentName := fmt.Sprintf("%s-%s-%s", workload.Name, placement.Name, strings.ToLower(locationName)) desiredDeployments.Insert(deploymentName) desired = append(desired, computev1alpha.WorkloadDeployment{ @@ -486,7 +509,7 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( Name: deploymentName, Labels: map[string]string{ computev1alpha.WorkloadUIDLabel: string(workload.UID), - computev1alpha.LocationLabel: locationRef.Name, + computev1alpha.LocationLabel: locationName, labelServiceName: labelServiceNameValue, }, }, @@ -496,7 +519,7 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( UID: workload.UID, }, PlacementName: placement.Name, - LocationRef: locationRef, + LocationRef: locationsv1alpha1.LocationReference{Name: locationName}, Template: workload.Spec.Template, ScaleSettings: placement.ScaleSettings, Replicas: new(placement.ScaleSettings.MinReplicas), @@ -517,6 +540,36 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( return desired, orphaned, nil } +// resolvePlacementLocations returns the names of the Ready locations a +// placement runs at, in a stable order: the locations it names that are Ready, +// or every Ready location its selector matches. A placement that names +// locations keeps their declared order; a selector yields locations by name. +func resolvePlacementLocations( + placement computev1alpha.WorkloadPlacement, + available []locations.PlacementLocation, + ready sets.Set[string], +) ([]string, error) { + if placement.LocationSelector != nil { + matched, err := locations.Select(available, placement.LocationSelector) + if err != nil { + return nil, err + } + names := make([]string, 0, len(matched)) + for _, location := range matched { + names = append(names, location.Name) + } + return names, nil + } + + names := make([]string, 0, len(placement.Locations)) + for _, ref := range placement.Locations { + if ready.Has(ref.Name) { + names = append(names, ref.Name) + } + } + return names, nil +} + // mergeDeploymentMetadata copies the controller-owned labels and annotations // from desired onto deployment without discarding peer-owned keys. Only the // spec is fully owned by this controller; metadata is shared with the @@ -539,9 +592,35 @@ func (r *WorkloadReconciler) SetupWithManager(mgr mcmanager.Manager) error { return fmt.Errorf("failed to register finalizer: %w", err) } + // A placement that selects locations by topology gains and loses + // deployments as locations come and go, and one that names a location not + // yet Ready starts running when it becomes Ready. Neither has any other + // wake-up event, so every workload in the project is re-reconciled when + // its placement locations change. The kind watched follows the location + // source, which every reconcile already reads. + placementLocationObject, err := locations.PlacementLocationObject(r.LocationSource) + if err != nil { + return err + } + b := mcbuilder.ControllerManagedBy(mgr). For(&computev1alpha.Workload{}, mcbuilder.WithEngageWithLocalCluster(false)). - Owns(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)) + Owns(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)). + Watches(placementLocationObject, func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []mcreconcile.Request { + return enqueueAllWorkloads(ctx, cl.GetClient(), clusterName) + }) + }, + // Workloads live in project control planes, never in the + // management cluster this manager runs against, so the management + // cluster is not watched and is not asked to serve the kind. The + // builder already defaults this to false whenever a provider is + // configured; For() and Owns() state it anyway, and so does this. + mcbuilder.WithEngageWithLocalCluster(false), + // A control plane that does not serve the kind is skipped rather + // than watched. See placementLocationClusterFilter. + mcbuilder.WithClusterFilter(placementLocationClusterFilter(r.LocationSource)), + ) if !r.NetworkingEnabled { return b.Complete(r) @@ -624,7 +703,8 @@ func workloadBlockingReasonPriority(reason string) int { computev1alpha.ReferencedDataReasonSourceTooLarge, computev1alpha.ReferencedDataReasonSourceUnauthorized: return 5 - case computev1alpha.WorkloadReasonNetworkNotFound: + case computev1alpha.WorkloadReasonNetworkNotFound, + computev1alpha.WorkloadReasonNoMatchingLocations: return 6 // This reason outranks every other blocker. No cell can accept the // deployment, so nothing else can make progress, and the user resolves the @@ -635,3 +715,72 @@ func workloadBlockingReasonPriority(reason string) int { return 0 } } + +// deploymentLocations returns the locations the given deployments run at, +// ordered by name, which is what a placement's status reports as resolved. +func deploymentLocations(deployments []computev1alpha.WorkloadDeployment) []locationsv1alpha1.LocationReference { + names := sets.New[string]() + for _, deployment := range deployments { + if deployment.Spec.LocationRef.Name != "" { + names.Insert(deployment.Spec.LocationRef.Name) + } + } + + refs := make([]locationsv1alpha1.LocationReference, 0, names.Len()) + for _, name := range sets.List(names) { + refs = append(refs, locationsv1alpha1.LocationReference{Name: name}) + } + return refs +} + +// placementLocationClusterFilter keeps the placement-location watch off a +// control plane that does not serve the kind. +// +// A watch is not a read. ListPlacementLocations degrades to no locations +// against a control plane that does not serve the kind, but an informer cannot: +// it retries the rejected list forever, that cluster's cache never syncs, and +// controller-runtime then blocks every controller on the manager from starting. +// The manager stays Ready and reconciles nothing until cluster engagement times +// out, at which point it exits and the pod restarts into the same state. +// +// The question is asked per control plane rather than once at startup, because +// the manager engages many of them and only the one being engaged can answer +// it. A control plane that gains the kind later is picked up the next time it +// is engaged. +func placementLocationClusterFilter(source locations.Source) mcbuilder.ClusterFilterFunc { + return func(clusterName multicluster.ClusterName, cl cluster.Cluster) bool { + logger := log.Log.WithName("workload").WithValues("cluster", clusterName) + + served, err := locations.ServesPlacementLocationKind(cl.GetRESTMapper(), source) + if err != nil { + logger.Error(err, "failed to determine whether the cluster serves placement locations; not watching them") + return false + } + if !served { + logger.Info("cluster does not serve the placement location kind; not watching placement locations", + "locationSource", source) + } + return served + } +} + +// enqueueAllWorkloads maps a change in the project's placement locations to +// every workload in the project, since any placement may resolve differently. +func enqueueAllWorkloads(ctx context.Context, c client.Client, clusterName multicluster.ClusterName) []mcreconcile.Request { + var workloads computev1alpha.WorkloadList + if err := c.List(ctx, &workloads); err != nil { + log.FromContext(ctx).Error(err, "failed to list workloads for placement location change") + return nil + } + + requests := make([]mcreconcile.Request, 0, len(workloads.Items)) + for _, workload := range workloads.Items { + requests = append(requests, mcreconcile.Request{ + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: workload.Namespace, Name: workload.Name}, + }, + ClusterName: clusterName, + }) + } + return requests +} diff --git a/internal/controller/workload_controller_test.go b/internal/controller/workload_controller_test.go index cf569768..b1addc0c 100644 --- a/internal/controller/workload_controller_test.go +++ b/internal/controller/workload_controller_test.go @@ -108,7 +108,7 @@ func TestGetDeploymentsForWorkload_InitializesReplicas(t *testing.T) { Placements: []computev1alpha.WorkloadPlacement{ { Name: testDefaultPlacement, - Locations: []locationsv1alpha1.LocationReference{{Name: "dfw"}}, + Locations: []locationsv1alpha1.LocationReference{{Name: testLocationName}}, ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: 2, }, @@ -116,7 +116,7 @@ func TestGetDeploymentsForWorkload_InitializesReplicas(t *testing.T) { }, }, } - location := newTestLocationBinding("dfw", "DFW") + location := newTestLocationBinding(testLocationName, "DFW") s := newNetworkingScheme() require.NoError(t, locationsv1alpha1.AddToScheme(s)) @@ -133,8 +133,8 @@ func TestGetDeploymentsForWorkload_InitializesReplicas(t *testing.T) { require.Len(t, desired, 1) require.NotNil(t, desired[0].Spec.Replicas) assert.Equal(t, int32(2), *desired[0].Spec.Replicas) - assert.Equal(t, "dfw", desired[0].Spec.LocationRef.Name) - assert.Equal(t, "dfw", desired[0].Labels[computev1alpha.LocationLabel]) + assert.Equal(t, testLocationName, desired[0].Spec.LocationRef.Name) + assert.Equal(t, testLocationName, desired[0].Labels[computev1alpha.LocationLabel]) } // TestReconcileWorkloadStatus_AllDeploymentsSameReason verifies that when all @@ -344,3 +344,149 @@ func TestWorkloadBlockingReasonPriority(t *testing.T) { }) } } + +// TestGetDeploymentsForWorkload_LocationSelector verifies that a placement +// selecting locations by topology gets one deployment per Ready location the +// selector matches, in name order, and nothing for locations it excludes. +func TestGetDeploymentsForWorkload_LocationSelector(t *testing.T) { + t.Parallel() + + workload := &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: rdTestWorkloadName, + Namespace: testDefaultNamespace, + UID: types.UID("workload-uid"), + }, + Spec: computev1alpha.WorkloadSpec{ + Placements: []computev1alpha.WorkloadPlacement{ + { + Name: testDefaultPlacement, + LocationSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{networkingv1alpha.TopologyCityCodeKey: "DFW"}, + }, + ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, + }, + }, + }, + } + + s := newNetworkingScheme() + cl := fake.NewClientBuilder(). + WithScheme(s). + WithObjects( + newTestLocationBinding("dfw-b", "DFW"), + newTestLocationBinding("dfw-a", "DFW"), + newTestLocationBinding(testOtherLocationName, "ORD"), + ). + WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). + Build() + r := &WorkloadReconciler{} + + desired, orphaned, err := r.getDeploymentsForWorkload(context.Background(), cl, workload) + require.NoError(t, err) + require.Empty(t, orphaned) + require.Len(t, desired, 2) + assert.Equal(t, "dfw-a", desired[0].Spec.LocationRef.Name) + assert.Equal(t, "dfw-b", desired[1].Spec.LocationRef.Name) + assert.Equal(t, rdTestWorkloadName+"-"+testDefaultPlacement+"-dfw-a", desired[0].Name) + assert.Equal(t, "dfw-a", desired[0].Labels[computev1alpha.LocationLabel]) +} + +// TestGetDeploymentsForWorkload_LocationSelectorFollowsLocations verifies the +// dynamic half of a selector: a deployment at a location the selector no +// longer matches is orphaned, so the placement follows the fleet. +func TestGetDeploymentsForWorkload_LocationSelectorFollowsLocations(t *testing.T) { + t.Parallel() + + workload := &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: rdTestWorkloadName, + Namespace: testDefaultNamespace, + UID: types.UID("workload-uid"), + }, + Spec: computev1alpha.WorkloadSpec{ + Placements: []computev1alpha.WorkloadPlacement{ + { + Name: testDefaultPlacement, + LocationSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{networkingv1alpha.TopologyCityCodeKey: "DFW"}, + }, + ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, + }, + }, + }, + } + // The ord deployment was created when the location matched; the binding + // has since changed city. + stale := &computev1alpha.WorkloadDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: rdTestWorkloadName + "-" + testDefaultPlacement + "-ord", + Namespace: testDefaultNamespace, + Labels: map[string]string{computev1alpha.WorkloadUIDLabel: "workload-uid"}, + }, + Spec: computev1alpha.WorkloadDeploymentSpec{ + WorkloadRef: computev1alpha.WorkloadReference{Name: rdTestWorkloadName, UID: types.UID("workload-uid")}, + PlacementName: testDefaultPlacement, + LocationRef: locationsv1alpha1.LocationReference{Name: testOtherLocationName}, + }, + } + + cl := fake.NewClientBuilder(). + WithScheme(newNetworkingScheme()). + WithObjects(newTestLocationBinding(testLocationName, "DFW"), newTestLocationBinding(testOtherLocationName, "ORD"), stale). + WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). + Build() + r := &WorkloadReconciler{} + + desired, orphaned, err := r.getDeploymentsForWorkload(context.Background(), cl, workload) + require.NoError(t, err) + require.Len(t, desired, 1) + assert.Equal(t, testLocationName, desired[0].Spec.LocationRef.Name) + require.Len(t, orphaned, 1) + assert.Equal(t, stale.Name, orphaned[0].Name) +} + +// TestReconcileWorkloadStatus_PlacementWithNoLocations verifies that a +// placement in the spec that resolved to nothing is still reported, with a +// reason that says so, and that the reason reaches the workload condition. +func TestReconcileWorkloadStatus_PlacementWithNoLocations(t *testing.T) { + workload := makeWorkload(1) + workload.Spec.Placements = []computev1alpha.WorkloadPlacement{{ + Name: testPlacementA, + LocationSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{networkingv1alpha.TopologyCityCodeKey: "LHR"}, + }, + }} + + cond := runReconcileWorkloadStatus(t, workload, map[string][]computev1alpha.WorkloadDeployment{}) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, computev1alpha.WorkloadReasonNoMatchingLocations, cond.Reason) + + require.Len(t, workload.Status.Placements, 1) + placement := workload.Status.Placements[0] + assert.Equal(t, testPlacementA, placement.Name) + assert.Empty(t, placement.Locations) + placementCond := apimeta.FindStatusCondition(placement.Conditions, workloadConditionTypeAvailable) + require.NotNil(t, placementCond) + assert.Equal(t, computev1alpha.WorkloadReasonNoMatchingLocations, placementCond.Reason) +} + +// TestReconcileWorkloadStatus_ReportsResolvedLocations verifies the placement +// status names the locations its deployments run at, in name order. +func TestReconcileWorkloadStatus_ReportsResolvedLocations(t *testing.T) { + workload := makeWorkload(1) + wdB := makeWDWithAvailCond("wd-b", metav1.ConditionTrue, "AvailableDeploymentFound", "") + wdB.Spec.LocationRef = locationsv1alpha1.LocationReference{Name: testOtherLocationName} + wdA := makeWDWithAvailCond("wd-a", metav1.ConditionTrue, "AvailableDeploymentFound", "") + wdA.Spec.LocationRef = locationsv1alpha1.LocationReference{Name: testLocationName} + + cond := runReconcileWorkloadStatus(t, workload, map[string][]computev1alpha.WorkloadDeployment{ + testPlacementA: {wdB, wdA}, + }) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionTrue, cond.Status) + + require.Len(t, workload.Status.Placements, 1) + assert.Equal(t, []locationsv1alpha1.LocationReference{{Name: testLocationName}, {Name: testOtherLocationName}}, workload.Status.Placements[0].Locations) +} diff --git a/internal/controller/workload_placementlocation_guard_test.go b/internal/controller/workload_placementlocation_guard_test.go new file mode 100644 index 00000000..320b774f --- /dev/null +++ b/internal/controller/workload_placementlocation_guard_test.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcsingle "sigs.k8s.io/multicluster-runtime/providers/single" + + "go.datum.net/compute/internal/locations" +) + +// TestWorkloadSetupWithManager_PlacementLocationKindAbsent is the regression +// test for the placement-location watch. +// +// The watch is registered against every project control plane the manager +// engages. A control plane that does not answer for the kind — because the CRD +// is absent, or because the manager is not allowed to list it — never syncs +// that informer, and controller-runtime then blocks EVERY controller on the +// manager from starting: the workload reconciler, the referenced-data +// reconciler and the deployment federator all sit on "Starting EventSource" +// forever, so nothing is federated and no finalizer is ever written. The +// manager eventually gives up waiting for cluster engagement and exits. +// +// So the watch has to be skipped for a control plane that does not serve the +// kind rather than registered and left to wedge. Here the compute CRDs are +// installed but the location CRDs are not, which is the same shape as a control +// plane that refuses the list. +func TestWorkloadSetupWithManager_PlacementLocationKindAbsent(t *testing.T) { + ctrl.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stderr))) + + cfg := startComputeEnvtest(t) + // The location types resolve on the scheme but have no CRD on the API + // server, so a registered watch fails at cache sync — the real failure — + // rather than at Complete() with a scheme lookup error. + scheme := newLocationsServiceScheme() + + newCluster := func(t *testing.T) cluster.Cluster { + t.Helper() + cl, err := cluster.New(rest.CopyConfig(cfg), func(o *cluster.Options) { o.Scheme = scheme }) + require.NoError(t, err) + return cl + } + + // serves asks the guard the way the builder asks it: per cluster, against + // that cluster's live REST mapper. + serves := func(t *testing.T, source locations.Source) bool { + t.Helper() + return placementLocationClusterFilter(source)(multicluster.ClusterName("single"), newCluster(t)) + } + + t.Run("neither kind served", func(t *testing.T) { + assert.False(t, serves(t, ""), + "the default source must not watch a kind this control plane does not serve") + assert.False(t, serves(t, locations.SourceNetworkServices)) + assert.False(t, serves(t, locations.SourceLocations)) + }) + + // The manager itself has to survive that: setup succeeds, the watch is + // simply not engaged, and every other controller starts. A build that + // registered the watch unconditionally exits here during cluster + // engagement. + t.Run("manager starts with the kind absent", func(t *testing.T) { + deploymentCluster := newCluster(t) + + mgr, err := mcmanager.New(rest.CopyConfig(cfg), + mcsingle.New(multicluster.ClusterName("single"), deploymentCluster), + ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "0", + }) + require.NoError(t, err) + + r := &WorkloadReconciler{} + require.NoError(t, r.SetupWithManager(mgr)) + + // The single provider does not start the cluster it engages, so — like + // cmd/main.go — the cluster and the manager run as sibling goroutines. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + errCh := make(chan error, 2) + go func() { errCh <- deploymentCluster.Start(ctx) }() + go func() { errCh <- mgr.Start(ctx) }() + + select { + case err := <-errCh: + t.Fatalf("manager exited during startup, want it to stay up: %v", err) + case <-time.After(startupObservationWindow): + } + }) + + // Each source gates on the one kind it reads. Installing the locations + // service must not satisfy the default source, and vice versa. + installCRD := func(t *testing.T, path string) { + t.Helper() + _, err := envtest.InstallCRDs(cfg, envtest.CRDInstallOptions{Paths: []string{path}}) + require.NoError(t, err) + } + + installCRD(t, moduleCRD(t, "go.miloapis.com/locations", + "config/base/crd/bases", "locations.miloapis.com_locations.yaml")) + + t.Run("only the locations service kind served", func(t *testing.T) { + assert.True(t, serves(t, locations.SourceLocations), + "the selected source's kind is served, so the watch must engage") + assert.False(t, serves(t, ""), + "installing the locations service must not satisfy the default source") + }) + + installCRD(t, moduleCRD(t, "go.datum.net/network-services-operator", + "config/crd/bases", "networking.datumapis.com_locationbindings.yaml")) + + t.Run("both kinds served", func(t *testing.T) { + for _, source := range []locations.Source{"", locations.SourceNetworkServices, locations.SourceLocations} { + assert.Truef(t, serves(t, source), + "the watch must engage for source %q once its kind is served", source) + } + }) +} diff --git a/internal/controller/workloaddeployment_federator_runtimeclass_test.go b/internal/controller/workloaddeployment_federator_runtimeclass_test.go index afaa98b7..4f5febd1 100644 --- a/internal/controller/workloaddeployment_federator_runtimeclass_test.go +++ b/internal/controller/workloaddeployment_federator_runtimeclass_test.go @@ -221,7 +221,7 @@ func TestCleanupPropagationPolicyIfUnused_PerLocationAndClass(t *testing.T) { classesEnabled: true, location: testFederatorLocation, runtimeClass: testClassAzurite, - remaining: []client.Object{hubSiblingDeployment("us-west-1", testClassAzurite)}, + remaining: []client.Object{hubSiblingDeployment(testWestLocationName, testClassAzurite)}, wantPPGone: true, }, { @@ -297,7 +297,7 @@ func TestWorkloadDeploymentFederator_UnservedRuntimeClassCondition(t *testing.T) name: "the class is served elsewhere, not here", classesEnabled: true, specClass: testClassBasalt, - cells: []client.Object{testCell("sea-cell", "us-west-1", testClassBasalt)}, + cells: []client.Object{testCell("sea-cell", testWestLocationName, testClassBasalt)}, wantReason: computev1alpha.WorkloadDeploymentReasonRuntimeClassNotServed, }, { diff --git a/internal/controller/workloaddeployment_network_binding_test.go b/internal/controller/workloaddeployment_network_binding_test.go index 2050e5a1..2d808265 100644 --- a/internal/controller/workloaddeployment_network_binding_test.go +++ b/internal/controller/workloaddeployment_network_binding_test.go @@ -23,9 +23,11 @@ import ( ) const ( - testNetworkName = "default" - testLocationName = "dfw" - testHubWDUID = types.UID("hub-wd-uid-9999") + testNetworkName = "default" + testLocationName = "dfw" + testOtherLocationName = "ord" + testWestLocationName = "us-west-1" + testHubWDUID = types.UID("hub-wd-uid-9999") ) // testHubDeployment returns the hub copy of the test WorkloadDeployment, already @@ -243,9 +245,9 @@ func TestEnsureNetworkBinding_RecreatesOnDivergence(t *testing.T) { }, { name: "serving location changed", - changed: withServingLocation("ord"), + changed: withServingLocation(testOtherLocationName), wantNetwork: testNetworkName, - wantLocation: "ord", + wantLocation: testOtherLocationName, }, } diff --git a/internal/locations/locations.go b/internal/locations/locations.go index 0881d723..4b867357 100644 --- a/internal/locations/locations.go +++ b/internal/locations/locations.go @@ -11,11 +11,15 @@ package locations import ( "context" + "errors" "fmt" + "sort" "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" @@ -173,6 +177,85 @@ func ListServingLocations(ctx context.Context, c client.Client, source Source) ( return found, nil } +// Select returns the Ready locations whose topology matches the selector, +// sorted by name so callers derive a stable set of deployments from it. +// +// An empty selector is an error rather than a match for every location. The +// webhook rejects one, so reaching this with one means the stored object was +// not admitted through it. +func Select(found []PlacementLocation, selector *metav1.LabelSelector) ([]PlacementLocation, error) { + if selector == nil || (len(selector.MatchLabels) == 0 && len(selector.MatchExpressions) == 0) { + return nil, errors.New("location selector is empty") + } + + sel, err := metav1.LabelSelectorAsSelector(selector) + if err != nil { + return nil, fmt.Errorf("invalid location selector: %w", err) + } + + var matched []PlacementLocation + for _, location := range found { + if location.Ready && sel.Matches(labels.Set(location.Topology)) { + matched = append(matched, location) + } + } + sort.Slice(matched, func(i, j int) bool { return matched[i].Name < matched[j].Name }) + return matched, nil +} + +// PlacementLocationObject returns the object a controller watches to learn +// that the locations a project may place workloads at have changed. +func PlacementLocationObject(source Source) (client.Object, error) { + resolved, err := source.Resolve() + if err != nil { + return nil, err + } + + if resolved == SourceNetworkServices { + return &networkingv1alpha.LocationBinding{}, nil + } + return &locationsv1alpha1.Location{}, nil +} + +// PlacementLocationGVK returns the kind a controller watches to learn that the +// locations a project may place workloads at have changed. +func PlacementLocationGVK(source Source) (schema.GroupVersionKind, error) { + resolved, err := source.Resolve() + if err != nil { + return schema.GroupVersionKind{}, err + } + + if resolved == SourceNetworkServices { + return networkingv1alpha.GroupVersion.WithKind("LocationBinding"), nil + } + return locationsv1alpha1.GroupVersion.WithKind("Location"), nil +} + +// ServesPlacementLocationKind reports whether the control plane behind the +// mapper serves the kind the source watches for placement locations. +// +// Unlike EnsureServingLocationKind this answers rather than refuses. A cell +// serves the deployments it is asked about, so a missing serving location kind +// there is a misconfiguration worth failing on. Placement locations are read +// from many project control planes engaged one at a time, and a control plane +// that does not carry the kind is skipped rather than taking the whole manager +// down with it. +func ServesPlacementLocationKind(mapper apimeta.RESTMapper, source Source) (bool, error) { + gvk, err := PlacementLocationGVK(source) + if err != nil { + return false, err + } + + if _, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version); err != nil { + if kindNotInstalled(err) { + return false, nil + } + return false, fmt.Errorf("failed to determine whether %s is served: %w", gvk, err) + } + + return true, nil +} + // ServingLocationObject returns the object a controller watches to learn that // a cell has been told where it sits. func ServingLocationObject(source Source) (client.Object, error) { diff --git a/internal/locations/locations_test.go b/internal/locations/locations_test.go index e32a732b..7aa2986a 100644 --- a/internal/locations/locations_test.go +++ b/internal/locations/locations_test.go @@ -23,6 +23,10 @@ import ( const ( testCityCode = "DFW" testOtherCityCode = "ORD" + + testLocationORD = "ord" + testLocationDFWA = "dfw-a" + testLocationDFWB = "dfw-b" ) func testScheme(t *testing.T) *runtime.Scheme { @@ -95,7 +99,7 @@ func TestListPlacementLocations_NetworkServices(t *testing.T) { WithScheme(testScheme(t)). WithObjects( newBinding("dfw", testCityCode), - newBinding("ord", testOtherCityCode), + newBinding(testLocationORD, testOtherCityCode), // A binding with no city code contributes no placement city. &networkingv1alpha.LocationBinding{ObjectMeta: metav1.ObjectMeta{Name: "nowhere"}}, // The locations service must not be read when network services is @@ -117,7 +121,7 @@ func TestListPlacementLocations_Locations(t *testing.T) { WithScheme(testScheme(t)). WithObjects( newLocation("dfw", testCityCode), - newLocation("ord", testOtherCityCode), + newLocation(testLocationORD, testOtherCityCode), // The network services source must not be read when the locations // service is selected. newBinding("lhr", "LHR"), @@ -272,3 +276,102 @@ func TestServingLocationGVK(t *testing.T) { _, err = ServingLocationGVK("Nonsense") require.Error(t, err) } + +// TestSelect covers selection over topology: a selector matches Ready +// locations by their topology, the result is ordered by name, and an empty or +// malformed selector is refused rather than matching everything. +func TestSelect(t *testing.T) { + t.Parallel() + + region := "topology.datum.net/region" + found := []PlacementLocation{ + {Name: testLocationORD, Topology: map[string]string{TopologyCityCodeKey: testOtherCityCode, region: "us-central"}, Ready: true}, + {Name: testLocationDFWB, Topology: map[string]string{TopologyCityCodeKey: testCityCode, region: "us-south"}, Ready: true}, + {Name: testLocationDFWA, Topology: map[string]string{TopologyCityCodeKey: testCityCode, region: "us-south"}, Ready: true}, + {Name: "dfw-down", Topology: map[string]string{TopologyCityCodeKey: testCityCode}, Ready: false}, + {Name: "nowhere", Ready: true}, + } + + names := func(locations []PlacementLocation) []string { + out := make([]string, 0, len(locations)) + for _, location := range locations { + out = append(out, location.Name) + } + return out + } + + t.Run("match labels, sorted, Ready only", func(t *testing.T) { + t.Parallel() + matched, err := Select(found, &metav1.LabelSelector{ + MatchLabels: map[string]string{TopologyCityCodeKey: testCityCode}, + }) + require.NoError(t, err) + assert.Equal(t, []string{testLocationDFWA, testLocationDFWB}, names(matched)) + }) + + t.Run("match expressions", func(t *testing.T) { + t.Parallel() + matched, err := Select(found, &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: region, Operator: metav1.LabelSelectorOpIn, Values: []string{"us-central", "eu-west"}, + }}, + }) + require.NoError(t, err) + assert.Equal(t, []string{testLocationORD}, names(matched)) + }) + + t.Run("exists selects every location with the key", func(t *testing.T) { + t.Parallel() + matched, err := Select(found, &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: TopologyCityCodeKey, Operator: metav1.LabelSelectorOpExists, + }}, + }) + require.NoError(t, err) + assert.Equal(t, []string{testLocationDFWA, testLocationDFWB, testLocationORD}, names(matched)) + }) + + t.Run("no match is empty, not an error", func(t *testing.T) { + t.Parallel() + matched, err := Select(found, &metav1.LabelSelector{ + MatchLabels: map[string]string{TopologyCityCodeKey: "LHR"}, + }) + require.NoError(t, err) + assert.Empty(t, matched) + }) + + t.Run("empty selector is refused", func(t *testing.T) { + t.Parallel() + _, err := Select(found, &metav1.LabelSelector{}) + require.Error(t, err) + _, err = Select(found, nil) + require.Error(t, err) + }) + + t.Run("malformed selector is refused", func(t *testing.T) { + t.Parallel() + _, err := Select(found, &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: TopologyCityCodeKey, Operator: metav1.LabelSelectorOpIn, + }}, + }) + require.Error(t, err) + }) +} + +// TestPlacementLocationObject pins which kind each source watches for the +// locations a project may place at. +func TestPlacementLocationObject(t *testing.T) { + t.Parallel() + + obj, err := PlacementLocationObject("") + require.NoError(t, err) + assert.IsType(t, &networkingv1alpha.LocationBinding{}, obj) + + obj, err = PlacementLocationObject(SourceLocations) + require.NoError(t, err) + assert.IsType(t, &locationsv1alpha1.Location{}, obj) + + _, err = PlacementLocationObject("Nonsense") + require.Error(t, err) +} diff --git a/internal/validation/workload_validation.go b/internal/validation/workload_validation.go index 9d61fb43..a5315d55 100644 --- a/internal/validation/workload_validation.go +++ b/internal/validation/workload_validation.go @@ -4,9 +4,15 @@ import ( "context" "fmt" "slices" + "sort" + "strings" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" k8scorev1 "k8s.io/api/core/v1" apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" @@ -79,6 +85,10 @@ type WorkloadValidationOptions struct { Workload *computev1alpha.Workload ValidLocations []string + // LocationTopologies is the topology of every Ready location, keyed by + // name. A placement's locationSelector must match at least one of them. + LocationTopologies map[string]map[string]string + // RuntimeClasses is the catalog of execution tiers this control plane // publishes, read by the caller. The catalog is empty when runtime class // selection is disabled. When selection is enabled, an empty catalog means @@ -124,9 +134,15 @@ func validateWorkloadPlacement(placement computev1alpha.WorkloadPlacement, field } locationsPath := fieldPath.Child("locations") - if len(placement.Locations) == 0 { - allErrs = append(allErrs, field.Required(locationsPath, "")) - } else { + selectorPath := fieldPath.Child("locationSelector") + switch { + case len(placement.Locations) == 0 && placement.LocationSelector == nil: + allErrs = append(allErrs, field.Required(locationsPath, "one of locations or locationSelector must be set")) + case len(placement.Locations) > 0 && placement.LocationSelector != nil: + allErrs = append(allErrs, field.Forbidden(selectorPath, "may not be set together with locations")) + case placement.LocationSelector != nil: + allErrs = append(allErrs, validateLocationSelector(placement.LocationSelector, selectorPath, opts)...) + default: seen := sets.New[string]() for i, location := range placement.Locations { namePath := locationsPath.Index(i).Child("name") @@ -249,3 +265,40 @@ func validateMetricTarget(target computev1alpha.MetricTarget, fieldPath *field.P return allErrs } + +// validateLocationSelector checks a placement's selector the way the workload +// controller will evaluate it: well formed, non-empty, and matching at least +// one Ready location's topology. A selector that matches nothing is rejected +// for the same reason an unknown location name is: storing it would admit a +// placement that never runs anywhere. +func validateLocationSelector(selector *metav1.LabelSelector, fieldPath *field.Path, opts WorkloadValidationOptions) field.ErrorList { + allErrs := metav1validation.ValidateLabelSelector(selector, metav1validation.LabelSelectorValidationOptions{}, fieldPath) + if len(allErrs) > 0 { + return allErrs + } + + if len(selector.MatchLabels) == 0 && len(selector.MatchExpressions) == 0 { + return append(allErrs, field.Required(fieldPath, fmt.Sprintf( + "an empty selector is not treated as matching every location; select at least one topology key, such as %s", + locationsv1alpha1.TopologyCityCodeKey))) + } + + sel, err := metav1.LabelSelectorAsSelector(selector) + if err != nil { + return append(allErrs, field.Invalid(fieldPath, selector, err.Error())) + } + + for _, topology := range opts.LocationTopologies { + if sel.Matches(labels.Set(topology)) { + return allErrs + } + } + + names := make([]string, 0, len(opts.LocationTopologies)) + for name := range opts.LocationTopologies { + names = append(names, name) + } + sort.Strings(names) + return append(allErrs, field.Invalid(fieldPath, sel.String(), fmt.Sprintf( + "matches none of the Ready locations (%s)", strings.Join(names, ", ")))) +} diff --git a/internal/validation/workload_validation_test.go b/internal/validation/workload_validation_test.go index 9d562627..9dda765b 100644 --- a/internal/validation/workload_validation_test.go +++ b/internal/validation/workload_validation_test.go @@ -50,6 +50,90 @@ func TestValidateWorkloads(t *testing.T) { field.Required(field.NewPath("spec.placements"), ""), }, }, + "location selector by city code": { + workload: MakeSandboxWorkload( + "test", + func(w *computev1alpha.Workload) { + w.Spec.Placements[0].Locations = nil + w.Spec.Placements[0].LocationSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: testCityCodeDFW}, + } + }, + ), + expectedErrors: field.ErrorList{}, + }, + "location selector by expression": { + workload: MakeSandboxWorkload( + "test", + func(w *computev1alpha.Workload) { + w.Spec.Placements[0].Locations = nil + w.Spec.Placements[0].LocationSelector = &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: locationsv1alpha1.TopologyCityCodeKey, + Operator: metav1.LabelSelectorOpIn, + Values: []string{testCityCodeDFW, "ORD"}, + }}, + } + }, + ), + expectedErrors: field.ErrorList{}, + }, + "location selector matching no ready location": { + workload: MakeSandboxWorkload( + "test", + func(w *computev1alpha.Workload) { + w.Spec.Placements[0].Locations = nil + w.Spec.Placements[0].LocationSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: "LHR"}, + } + }, + ), + expectedErrors: field.ErrorList{ + field.Invalid(field.NewPath("spec.placements[0].locationSelector"), "", ""), + }, + }, + "empty location selector": { + workload: MakeSandboxWorkload( + "test", + func(w *computev1alpha.Workload) { + w.Spec.Placements[0].Locations = nil + w.Spec.Placements[0].LocationSelector = &metav1.LabelSelector{} + }, + ), + expectedErrors: field.ErrorList{ + field.Required(field.NewPath("spec.placements[0].locationSelector"), ""), + }, + }, + "malformed location selector": { + workload: MakeSandboxWorkload( + "test", + func(w *computev1alpha.Workload) { + w.Spec.Placements[0].Locations = nil + w.Spec.Placements[0].LocationSelector = &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: locationsv1alpha1.TopologyCityCodeKey, + Operator: metav1.LabelSelectorOpIn, + }}, + } + }, + ), + expectedErrors: field.ErrorList{ + field.Required(field.NewPath("spec.placements[0].locationSelector.matchExpressions[0].values"), ""), + }, + }, + "location selector together with locations": { + workload: MakeSandboxWorkload( + "test", + func(w *computev1alpha.Workload) { + w.Spec.Placements[0].LocationSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: testCityCodeDFW}, + } + }, + ), + expectedErrors: field.ErrorList{ + field.Forbidden(field.NewPath("spec.placements[0].locationSelector"), ""), + }, + }, "missing location": { workload: MakeSandboxWorkload( "test", @@ -619,6 +703,11 @@ func TestValidateWorkloads(t *testing.T) { if len(scenario.opts.ValidLocations) == 0 { scenario.opts.ValidLocations = []string{testCityCodeDFW} } + if scenario.opts.LocationTopologies == nil { + scenario.opts.LocationTopologies = map[string]map[string]string{ + testCityCodeDFW: {locationsv1alpha1.TopologyCityCodeKey: testCityCodeDFW}, + } + } t.Run(name, func(t *testing.T) { errs := ValidateWorkloadCreate(scenario.workload, scenario.opts) diff --git a/internal/webhook/v1alpha/workload_webhook.go b/internal/webhook/v1alpha/workload_webhook.go index f6aa1d5b..47b06ed0 100644 --- a/internal/webhook/v1alpha/workload_webhook.go +++ b/internal/webhook/v1alpha/workload_webhook.go @@ -47,14 +47,31 @@ type workloadWebhook struct { locationSource locations.Source } -// validLocations returns the names a placement may reference: the locations -// projected into the project control plane that are Ready to accept workloads. -func (r *workloadWebhook) validLocations(ctx context.Context, c client.Client) ([]string, error) { +// readyLocations describes the locations a placement may run at: the +// locations projected into the project control plane that are Ready to accept +// workloads, as the names a placement may list and the topology a selector is +// matched against. +type readyLocations struct { + names []string + topologies map[string]map[string]string +} + +func (r *workloadWebhook) readyLocations(ctx context.Context, c client.Client) (readyLocations, error) { placementLocations, err := locations.ListPlacementLocations(ctx, c, r.locationSource) if err != nil { - return nil, err + return readyLocations{}, err + } + + ready := readyLocations{ + names: sets.List(locations.ReadyNames(placementLocations)), + topologies: make(map[string]map[string]string), + } + for _, location := range placementLocations { + if location.Ready { + ready.topologies[location.Name] = location.Topology + } } - return sets.List(locations.ReadyNames(placementLocations)), nil + return ready, nil } var _ admission.Defaulter[*computev1alpha.Workload] = &workloadWebhook{} @@ -116,7 +133,7 @@ func (r *workloadWebhook) ValidateCreate(ctx context.Context, workload *computev // that means for the scheduling phase, since there would not currently be // sufficient context to know who created the workload and what locations // are valid candidates based on that. Maybe an annotation, or spec field? - validLocations, err := r.validLocations(ctx, clusterClient) + ready, err := r.readyLocations(ctx, clusterClient) if err != nil { return nil, err } @@ -127,12 +144,13 @@ func (r *workloadWebhook) ValidateCreate(ctx context.Context, workload *computev } opts := validation.WorkloadValidationOptions{ - Context: ctx, - Client: clusterClient, - AdmissionRequest: req, - Workload: workload, - ValidLocations: validLocations, - RuntimeClasses: runtimeClasses, + Context: ctx, + Client: clusterClient, + AdmissionRequest: req, + Workload: workload, + ValidLocations: ready.names, + LocationTopologies: ready.topologies, + RuntimeClasses: runtimeClasses, } if errs := validation.ValidateWorkloadCreate(workload, opts); len(errs) > 0 { @@ -158,7 +176,7 @@ func (r *workloadWebhook) ValidateUpdate(ctx context.Context, oldWorkload *compu return nil, err } - validLocations, err := r.validLocations(ctx, clusterClient) + ready, err := r.readyLocations(ctx, clusterClient) if err != nil { return nil, err } @@ -169,12 +187,13 @@ func (r *workloadWebhook) ValidateUpdate(ctx context.Context, oldWorkload *compu } opts := validation.WorkloadValidationOptions{ - Context: ctx, - Client: clusterClient, - AdmissionRequest: req, - Workload: newWorkload, - ValidLocations: validLocations, - RuntimeClasses: runtimeClasses, + Context: ctx, + Client: clusterClient, + AdmissionRequest: req, + Workload: newWorkload, + ValidLocations: ready.names, + LocationTopologies: ready.topologies, + RuntimeClasses: runtimeClasses, } if errs := validation.ValidateWorkloadUpdate(newWorkload, oldWorkload, opts); len(errs) > 0 { diff --git a/ui/consumer/src/adapter.ts b/ui/consumer/src/adapter.ts index 73412e40..5b98e5da 100644 --- a/ui/consumer/src/adapter.ts +++ b/ui/consumer/src/adapter.ts @@ -68,11 +68,13 @@ export const INSTANCE_LABELS = { interface RawWorkloadPlacement { name: string; locations?: Array<{ name: string }>; + locationSelector?: RawLabelSelector; scaleSettings?: { minReplicas?: number; maxReplicas?: number }; } interface RawWorkloadPlacementStatus { name?: string; + locations?: Array<{ name: string }>; conditions?: RawCondition[]; replicas?: number; currentReplicas?: number; @@ -179,6 +181,55 @@ function deriveUpdatedAt( return latest ?? undefined; } +interface RawLabelSelector { + matchLabels?: Record; + matchExpressions?: Array<{ key: string; operator: string; values?: string[] }>; +} + +/** Renders a label selector the way kubectl prints it, e.g. "city-code=DFW, region in (a,b)". */ +function formatLabelSelector(selector: RawLabelSelector): string { + const parts: string[] = []; + for (const [key, value] of Object.entries(selector.matchLabels ?? {})) { + parts.push(`${key}=${value}`); + } + for (const expr of selector.matchExpressions ?? []) { + const values = (expr.values ?? []).join(','); + switch (expr.operator) { + case 'In': + parts.push(`${expr.key} in (${values})`); + break; + case 'NotIn': + parts.push(`${expr.key} notin (${values})`); + break; + case 'Exists': + parts.push(expr.key); + break; + case 'DoesNotExist': + parts.push(`!${expr.key}`); + break; + default: + parts.push(`${expr.key} ${expr.operator} (${values})`); + } + } + return parts.join(', '); +} + +/** + * The locations a placement runs at: what the controller resolved when status + * is present, else what the spec names. A selector-based placement has no + * names in its spec, so status is the only place its locations appear. + */ +function placementLocations(p: RawWorkloadPlacement, status?: RawWorkloadPlacementStatus): string[] { + const resolved = (status?.locations ?? []).map((location) => location.name); + if (resolved.length > 0) return resolved; + return (p.locations ?? []).map((location) => location.name); +} + +function workloadLocations(placements: RawWorkloadPlacement[], statusPlacements: RawWorkloadPlacementStatus[]): string[] { + const statusByName = new Map(statusPlacements.filter((s) => !!s.name).map((s) => [s.name, s])); + return Array.from(new Set(placements.flatMap((p) => placementLocations(p, statusByName.get(p.name))))); +} + function toPlacementRegions( placements: RawWorkloadPlacement[], statusPlacements: RawWorkloadPlacementStatus[] @@ -209,7 +260,8 @@ function toPlacementRegions( return { name: p.name, - locations: (p.locations ?? []).map((location) => location.name), + locations: placementLocations(p, status), + locationSelector: p.locationSelector ? formatLabelSelector(p.locationSelector) : undefined, readyReplicas: ready, desiredReplicas: desired, health, @@ -243,7 +295,7 @@ export function toWorkload(raw: RawWorkload): Workload { runtimeType: runtime ? (runtime.sandbox ? 'Container sandbox' : 'Virtual machine') : undefined, tags: deriveTags(runtime), ports, - locations: Array.from(new Set(placements.flatMap((p) => (p.locations ?? []).map((location) => location.name)))), + locations: workloadLocations(placements, raw.status?.placements ?? []), resources: deriveResources(runtime), replicasPerRegion: deriveReplicasPerRegion(placements), conditions: conditions.map((c) => ({ diff --git a/ui/consumer/src/pages/workload-list.tsx b/ui/consumer/src/pages/workload-list.tsx index a86a84c3..0227f048 100644 --- a/ui/consumer/src/pages/workload-list.tsx +++ b/ui/consumer/src/pages/workload-list.tsx @@ -63,6 +63,7 @@ function statusLabel(workload: Workload): string { function regionLabel(region: WorkloadPlacementRegion): string { if (region.locations.length > 0) return region.locations.join(", "); + if (region.locationSelector) return region.locationSelector; return region.name; } diff --git a/ui/consumer/src/schema.ts b/ui/consumer/src/schema.ts index 395ef05c..fcfdc291 100644 --- a/ui/consumer/src/schema.ts +++ b/ui/consumer/src/schema.ts @@ -23,6 +23,8 @@ const workloadConditionSchema = z.object({ export const workloadPlacementRegionSchema = z.object({ name: z.string(), locations: z.array(z.string()).default([]), + /** Topology selector the placement resolves through, when it does not name locations. */ + locationSelector: z.string().optional(), readyReplicas: z.number(), desiredReplicas: z.number(), health: z.enum(['Available', 'Degraded', 'Unavailable', 'Unknown']), diff --git a/ui/provider/src/adapter.ts b/ui/provider/src/adapter.ts index 2dbe86a5..d3ca0885 100644 --- a/ui/provider/src/adapter.ts +++ b/ui/provider/src/adapter.ts @@ -75,11 +75,13 @@ export const INSTANCE_LABELS = { interface RawWorkloadPlacement { name: string; locations?: Array<{ name: string }>; + locationSelector?: RawLabelSelector; scaleSettings?: { minReplicas?: number; maxReplicas?: number }; } interface RawWorkloadPlacementStatus { name?: string; + locations?: Array<{ name: string }>; conditions?: RawCondition[]; replicas?: number; currentReplicas?: number; @@ -133,6 +135,55 @@ function deriveReplicasPerRegion(placements: RawWorkloadPlacement[]): number | u return mins.every((m) => m === first) ? first : undefined; } +interface RawLabelSelector { + matchLabels?: Record; + matchExpressions?: Array<{ key: string; operator: string; values?: string[] }>; +} + +/** Renders a label selector the way kubectl prints it, e.g. "city-code=DFW, region in (a,b)". */ +function formatLabelSelector(selector: RawLabelSelector): string { + const parts: string[] = []; + for (const [key, value] of Object.entries(selector.matchLabels ?? {})) { + parts.push(`${key}=${value}`); + } + for (const expr of selector.matchExpressions ?? []) { + const values = (expr.values ?? []).join(','); + switch (expr.operator) { + case 'In': + parts.push(`${expr.key} in (${values})`); + break; + case 'NotIn': + parts.push(`${expr.key} notin (${values})`); + break; + case 'Exists': + parts.push(expr.key); + break; + case 'DoesNotExist': + parts.push(`!${expr.key}`); + break; + default: + parts.push(`${expr.key} ${expr.operator} (${values})`); + } + } + return parts.join(', '); +} + +/** + * The locations a placement runs at: what the controller resolved when status + * is present, else what the spec names. A selector-based placement has no + * names in its spec, so status is the only place its locations appear. + */ +function placementLocations(p: RawWorkloadPlacement, status?: RawWorkloadPlacementStatus): string[] { + const resolved = (status?.locations ?? []).map((location) => location.name); + if (resolved.length > 0) return resolved; + return (p.locations ?? []).map((location) => location.name); +} + +function workloadLocations(placements: RawWorkloadPlacement[], statusPlacements: RawWorkloadPlacementStatus[]): string[] { + const statusByName = new Map(statusPlacements.filter((s) => !!s.name).map((s) => [s.name, s])); + return Array.from(new Set(placements.flatMap((p) => placementLocations(p, statusByName.get(p.name))))); +} + function toPlacements( placements: RawWorkloadPlacement[], statusPlacements: RawWorkloadPlacementStatus[] @@ -163,7 +214,8 @@ function toPlacements( return { name: p.name, - locations: (p.locations ?? []).map((location) => location.name), + locations: placementLocations(p, status), + locationSelector: p.locationSelector ? formatLabelSelector(p.locationSelector) : undefined, readyReplicas: ready, desiredReplicas: desired, currentReplicas: current, @@ -194,7 +246,7 @@ export function toWorkload(raw: RawWorkload): Workload { placements: toPlacements(placements, raw.status?.placements ?? []), conditions: toConditions(conditions), runtimeType: runtime ? (runtime.sandbox ? 'Container sandbox' : 'Virtual machine') : undefined, - locations: Array.from(new Set(placements.flatMap((p) => (p.locations ?? []).map((location) => location.name)))), + locations: workloadLocations(placements, raw.status?.placements ?? []), resources: deriveResources(runtime), replicasPerRegion: deriveReplicasPerRegion(placements), }; diff --git a/ui/provider/src/pages/workload-detail.tsx b/ui/provider/src/pages/workload-detail.tsx index bab21036..8d279f28 100644 --- a/ui/provider/src/pages/workload-detail.tsx +++ b/ui/provider/src/pages/workload-detail.tsx @@ -135,7 +135,7 @@ function PlacementsCard({ workload }: { workload: Workload }) {
    {p.name} - {p.locations.join(', ') || 'no locations'} + {p.locations.join(', ') || p.locationSelector || 'no locations'}
    diff --git a/ui/provider/src/schema.ts b/ui/provider/src/schema.ts index 5b567c06..1f1386a3 100644 --- a/ui/provider/src/schema.ts +++ b/ui/provider/src/schema.ts @@ -26,6 +26,8 @@ export type Condition = z.infer; export const workloadPlacementSchema = z.object({ name: z.string(), locations: z.array(z.string()).default([]), + /** Topology selector the placement resolves through, when it does not name locations. */ + locationSelector: z.string().optional(), readyReplicas: z.number(), desiredReplicas: z.number(), currentReplicas: z.number(), From 5dbb3086897dda8a8aca934565e6345dbb6f9349 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 15:55:56 -0500 Subject: [PATCH 03/12] feat(cli): make --city a location selector --city no longer resolves city codes to location names on the client. It emits a locationSelector on topology.datum.net/city-code, so the placement deploys to every Ready location in those cities and follows locations as they are added or removed. Admission already rejects a city that matches no Ready location, which replaces the client-side check, and --all-matching-locations has no purpose left. Co-Authored-By: Claude Fable 5.1 --- internal/cmd/compute/deploy/deploy.go | 88 ++++++++++++--------------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/internal/cmd/compute/deploy/deploy.go b/internal/cmd/compute/deploy/deploy.go index b0423017..d1b00e77 100644 --- a/internal/cmd/compute/deploy/deploy.go +++ b/internal/cmd/compute/deploy/deploy.go @@ -13,7 +13,6 @@ import ( "golang.org/x/term" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" - apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilyaml "k8s.io/apimachinery/pkg/util/yaml" @@ -29,17 +28,16 @@ import ( ) type options struct { - image string - build string - instanceType string - locations []string - locationSelector string - cities []string - allMatchingLocations bool - min int32 - port int32 - file string - yes bool + image string + build string + instanceType string + locations []string + locationSelector string + cities []string + min int32 + port int32 + file string + yes bool } func Command() *cobra.Command { @@ -70,8 +68,11 @@ a fix is applied — same as 'datumctl compute build --fix'.`, # Build from another directory datumctl compute deploy api --build=./api --image=ghcr.io/acme/api:1.4.2 --location=us-east-1,eu-west-1 - # Select locations by topology instead of naming them - datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --location-selector='topology.datum.net/city-code=DFW' + # Deploy to every location in one or more cities + datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --city=DFW,IAD + + # Select locations by any topology label + datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --location-selector='topology.datum.net/region=us-east-1' # Interactive mode datumctl compute deploy @@ -90,8 +91,7 @@ a fix is applied — same as 'datumctl compute build --fix'.`, cmd.Flags().StringVar(&opts.instanceType, "instance-type", "datumcloud/d1-standard-2", "Instance type (e.g. datumcloud/d1-standard-2)") cmd.Flags().StringSliceVar(&opts.locations, "location", nil, "One or more locations to deploy to (e.g. us-east-1,eu-west-1)") cmd.Flags().StringVar(&opts.locationSelector, "location-selector", "", "Select every location whose topology matches a label selector (e.g. 'topology.datum.net/city-code=DFW' or 'topology.datum.net/region in (us-east-1,eu-west-1)')") - cmd.Flags().StringSliceVar(&opts.cities, "city", nil, "Resolve a city code to an available location (e.g. IAD)") - cmd.Flags().BoolVar(&opts.allMatchingLocations, "all-matching-locations", false, "Deploy to every location matching --city") + cmd.Flags().StringSliceVar(&opts.cities, "city", nil, "Deploy to every location in these cities (e.g. DFW,IAD); shorthand for a --location-selector on topology.datum.net/city-code") cmd.Flags().Int32Var(&opts.min, "min", 1, "Minimum number of instances per location") cmd.Flags().Int32Var(&opts.port, "port", 0, "Port to expose on the workload (optional)") cmd.Flags().StringVarP(&opts.file, "file", "f", "", "Path to a workload manifest file") @@ -156,7 +156,7 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err } } if placementFlags == 0 { - return fmt.Errorf("--location is required (e.g. --location=us-east-1,eu-west-1); use --city to resolve a city code, or --location-selector to select locations by topology") + return fmt.Errorf("--location is required (e.g. --location=us-east-1,eu-west-1); or use --city to deploy to every location in a city, or --location-selector to select locations by topology") } if placementFlags > 1 { return fmt.Errorf("--location, --city, and --location-selector are mutually exclusive") @@ -169,6 +169,9 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err } locationSelector = parsed } + if len(opts.cities) > 0 { + locationSelector = citySelector(opts.cities) + } instanceType := opts.instanceType if instanceType == "" { instanceType = "datumcloud/d1-standard-2" @@ -182,13 +185,6 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err ctx := context.Background() out := cmd.OutOrStdout() locations := opts.locations - if len(opts.cities) > 0 { - locations, err = resolveCities(ctx, c, opts.cities, opts.allMatchingLocations) - if err != nil { - return err - } - fmt.Fprintf(out, "Resolved --city to locations: %s\n", strings.Join(locations, ", ")) - } if err := ensureNetwork(ctx, cmd, c, "default", project, opts); err != nil { return err @@ -511,32 +507,6 @@ func manifestDiff(existing, desired computev1alpha.Workload) []string { return lines } -func resolveCities(ctx context.Context, c client.Client, cities []string, all bool) ([]string, error) { - var list locationsv1alpha1.LocationList - if err := c.List(ctx, &list); err != nil { - return nil, fmt.Errorf("listing project locations: %w", err) - } - var resolved []string - for _, city := range cities { - var matches []string - for _, location := range list.Items { - ready := apimeta.FindStatusCondition(location.Status.Conditions, locationsv1alpha1.LocationConditionReady) - if location.Spec.Topology[locationsv1alpha1.TopologyCityCodeKey] != city || ready == nil || ready.Status != metav1.ConditionTrue { - continue - } - matches = append(matches, location.Name) - } - if len(matches) == 0 { - return nil, fmt.Errorf("city %q has no Ready locations; run 'datumctl get locations' to see available locations", city) - } - if len(matches) > 1 && !all { - return nil, fmt.Errorf("city %q matches multiple Ready locations (%s); use --location to choose one or --all-matching-locations", city, strings.Join(matches, ", ")) - } - resolved = append(resolved, matches...) - } - return resolved, nil -} - // describePlacementLocations says where a placement runs the way the CLI // prints it: the locations it names, or the selector it resolves through. func describePlacementLocations(p computev1alpha.WorkloadPlacement) string { @@ -549,3 +519,21 @@ func describePlacementLocations(p computev1alpha.WorkloadPlacement) string { } return fmt.Sprintf("locations=[%s]", strings.Join(names, ", ")) } + +// citySelector is the selector --city stands for: every location whose +// topology places it in one of the given cities. One city is a plain +// equality; several become an In expression. +func citySelector(cities []string) *metav1.LabelSelector { + if len(cities) == 1 { + return &metav1.LabelSelector{ + MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: cities[0]}, + } + } + return &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: locationsv1alpha1.TopologyCityCodeKey, + Operator: metav1.LabelSelectorOpIn, + Values: cities, + }}, + } +} From 44c13f80afd92169296cfd83aed5fe18b5f8db28 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 16:31:56 -0500 Subject: [PATCH 04/12] feat(cli): complete comma-separated locations and selector pairs --location and --city already completed, but only for the first element: the shell matches candidates against the whole value typed so far, so "--location=us-east-1,eu-" offered nothing. Candidates now carry the typed elements in front of them, already-chosen elements are not repeated, and no space is appended so a comma can follow. deploy offers only Ready locations and their city codes, since admission rejects anything else; list and describe filters keep offering every projected location. --location-selector completes the key=value pairs found in the topology of Ready locations, so the available topology keys can be discovered from the shell. Co-Authored-By: Claude Fable 5.1 --- internal/cmd/compute/deploy/deploy.go | 3 +- internal/cmd/compute/util/completion.go | 122 ++++++++++++++++--- internal/cmd/compute/util/completion_test.go | 98 +++++++++++++++ 3 files changed, 206 insertions(+), 17 deletions(-) create mode 100644 internal/cmd/compute/util/completion_test.go diff --git a/internal/cmd/compute/deploy/deploy.go b/internal/cmd/compute/deploy/deploy.go index d1b00e77..a5342b7d 100644 --- a/internal/cmd/compute/deploy/deploy.go +++ b/internal/cmd/compute/deploy/deploy.go @@ -96,7 +96,8 @@ a fix is applied — same as 'datumctl compute build --fix'.`, cmd.Flags().Int32Var(&opts.port, "port", 0, "Port to expose on the workload (optional)") cmd.Flags().StringVarP(&opts.file, "file", "f", "", "Path to a workload manifest file") cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false, "Skip confirmation prompts") - _ = cmd.RegisterFlagCompletionFunc("location", util.CompleteLocations) + _ = cmd.RegisterFlagCompletionFunc("location", util.CompletePlacementLocations) + _ = cmd.RegisterFlagCompletionFunc("location-selector", util.CompleteLocationSelector) _ = cmd.RegisterFlagCompletionFunc("city", util.CompleteCityCodes) return cmd diff --git a/internal/cmd/compute/util/completion.go b/internal/cmd/compute/util/completion.go index 7eed2b2f..f79b2455 100644 --- a/internal/cmd/compute/util/completion.go +++ b/internal/cmd/compute/util/completion.go @@ -3,8 +3,11 @@ package util import ( "context" "sort" + "strings" "github.com/spf13/cobra" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" computev1alpha "go.datum.net/compute/api/v1alpha" @@ -36,37 +39,124 @@ func CompleteInstanceNames(cmd *cobra.Command, args []string, _ string) ([]strin return names, cobra.ShellCompDirectiveNoFileComp } -// CompleteLocations returns the projected locations available to the project. -func CompleteLocations(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { +// CompleteLocations completes a --location flag with every location projected +// into the project, whether or not it is Ready. List and describe commands +// filter by location, and a location that is no longer Ready may still have +// deployments worth finding. +func CompleteLocations(cmd *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { list, ok := projectedLocations(cmd) if !ok { return nil, cobra.ShellCompDirectiveNoFileComp } + return completeCommaList(locationCandidates(list, false), toComplete) +} + +// CompletePlacementLocations completes a deploy-time --location flag with the +// locations a placement may name: those projected into the project that are +// Ready. Admission rejects any other, so they are not offered. +func CompletePlacementLocations(cmd *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { + list, ok := projectedLocations(cmd) + if !ok { + return nil, cobra.ShellCompDirectiveNoFileComp + } + return completeCommaList(locationCandidates(list, true), toComplete) +} + +// CompleteCityCodes completes --city with the city codes of the project's +// Ready locations. +func CompleteCityCodes(cmd *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { + list, ok := projectedLocations(cmd) + if !ok { + return nil, cobra.ShellCompDirectiveNoFileComp + } + return completeCommaList(cityCodeCandidates(list), toComplete) +} + +// CompleteLocationSelector completes --location-selector with the key=value +// pairs found in the topology of the project's Ready locations, so a user can +// discover which topology keys exist without reading each Location. +func CompleteLocationSelector(cmd *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { + list, ok := projectedLocations(cmd) + if !ok { + return nil, cobra.ShellCompDirectiveNoFileComp + } + return completeCommaList(selectorCandidates(list), toComplete) +} + +// locationCandidates returns location names, sorted, optionally only the +// Ready ones. +func locationCandidates(list locationsv1alpha1.LocationList, readyOnly bool) []string { names := make([]string, 0, len(list.Items)) for _, location := range list.Items { + if readyOnly && !locationIsReady(location) { + continue + } names = append(names, location.Name) } sort.Strings(names) - return names, cobra.ShellCompDirectiveNoFileComp + return names } -// CompleteCityCodes returns unique city codes from projected Locations. -func CompleteCityCodes(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { - list, ok := projectedLocations(cmd) - if !ok { - return nil, cobra.ShellCompDirectiveNoFileComp +// cityCodeCandidates returns the distinct city codes of Ready locations, +// sorted. +func cityCodeCandidates(list locationsv1alpha1.LocationList) []string { + codes := sets.New[string]() + for _, location := range list.Items { + if !locationIsReady(location) { + continue + } + if code := location.Spec.Topology[locationsv1alpha1.TopologyCityCodeKey]; code != "" { + codes.Insert(code) + } } - seen := make(map[string]bool) - var codes []string + return sets.List(codes) +} + +// selectorCandidates returns every distinct key=value pair in the topology of +// Ready locations, sorted, which is what a selector on those locations can +// match. +func selectorCandidates(list locationsv1alpha1.LocationList) []string { + pairs := sets.New[string]() for _, location := range list.Items { - code := location.Spec.Topology[locationsv1alpha1.TopologyCityCodeKey] - if code != "" && !seen[code] { - seen[code] = true - codes = append(codes, code) + if !locationIsReady(location) { + continue + } + for key, value := range location.Spec.Topology { + pairs.Insert(key + "=" + value) + } + } + return sets.List(pairs) +} + +func locationIsReady(location locationsv1alpha1.Location) bool { + return apimeta.IsStatusConditionTrue(location.Status.Conditions, locationsv1alpha1.LocationConditionReady) +} + +// completeCommaList completes the last element of a comma-separated flag +// value. The shell matches candidates against the whole value typed so far, +// so each candidate is returned with the already-typed elements in front of +// it; elements already present are not offered again. No space is appended, +// so the user can keep typing a comma for the next element. +func completeCommaList(candidates []string, toComplete string) ([]string, cobra.ShellCompDirective) { + prefix := "" + chosen := sets.New[string]() + if i := strings.LastIndex(toComplete, ","); i >= 0 { + prefix = toComplete[:i+1] + for _, element := range strings.Split(toComplete[:i], ",") { + if element != "" { + chosen.Insert(element) + } + } + } + + completions := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + if chosen.Has(candidate) { + continue } + completions = append(completions, prefix+candidate) } - sort.Strings(codes) - return codes, cobra.ShellCompDirectiveNoFileComp + return completions, cobra.ShellCompDirectiveNoFileComp | cobra.ShellCompDirectiveNoSpace } func projectedLocations(cmd *cobra.Command) (locationsv1alpha1.LocationList, bool) { diff --git a/internal/cmd/compute/util/completion_test.go b/internal/cmd/compute/util/completion_test.go new file mode 100644 index 00000000..3c336e46 --- /dev/null +++ b/internal/cmd/compute/util/completion_test.go @@ -0,0 +1,98 @@ +package util + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" +) + +const ( + completionTestRegionKey = "topology.datum.net/region" + completionTestCityDFW = "DFW" + completionTestDFWA = "dfw-a" + completionTestDFWB = "dfw-b" + completionTestORD = "ord" +) + +func completionTestLocation(name, city, region string, ready bool) locationsv1alpha1.Location { + location := locationsv1alpha1.Location{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: locationsv1alpha1.LocationSpec{ + Topology: map[string]string{ + locationsv1alpha1.TopologyCityCodeKey: city, + completionTestRegionKey: region, + }, + }, + } + if ready { + location.Status.Conditions = []metav1.Condition{{ + Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue, + }} + } + return location +} + +func completionTestList() locationsv1alpha1.LocationList { + return locationsv1alpha1.LocationList{Items: []locationsv1alpha1.Location{ + completionTestLocation(completionTestORD, "ORD", "us-central", true), + completionTestLocation(completionTestDFWA, completionTestCityDFW, "us-south", true), + completionTestLocation(completionTestDFWB, completionTestCityDFW, "us-south", true), + completionTestLocation("lhr", "LHR", "eu-west", false), + }} +} + +func TestLocationCandidates(t *testing.T) { + t.Parallel() + list := completionTestList() + assert.Equal(t, []string{completionTestDFWA, completionTestDFWB, "lhr", completionTestORD}, locationCandidates(list, false), + "list filters may name a location that is not Ready") + assert.Equal(t, []string{completionTestDFWA, completionTestDFWB, completionTestORD}, locationCandidates(list, true), + "a placement may only name a Ready location") +} + +func TestCityCodeCandidates(t *testing.T) { + t.Parallel() + assert.Equal(t, []string{completionTestCityDFW, "ORD"}, cityCodeCandidates(completionTestList()), + "city codes are deduplicated and only Ready locations count") +} + +func TestSelectorCandidates(t *testing.T) { + t.Parallel() + assert.Equal(t, []string{ + locationsv1alpha1.TopologyCityCodeKey + "=" + completionTestCityDFW, + locationsv1alpha1.TopologyCityCodeKey + "=ORD", + completionTestRegionKey + "=us-central", + completionTestRegionKey + "=us-south", + }, selectorCandidates(completionTestList()), + "every topology key=value pair of a Ready location is offered once") +} + +func TestCompleteCommaList(t *testing.T) { + t.Parallel() + candidates := []string{completionTestDFWA, completionTestDFWB, completionTestORD} + + tests := []struct { + name string + toComplete string + want []string + }{ + {"first element", "", []string{completionTestDFWA, completionTestDFWB, completionTestORD}}, + {"first element partial", "df", []string{completionTestDFWA, completionTestDFWB, completionTestORD}}, + {"second element keeps the first", completionTestDFWA + ",", []string{completionTestDFWA + "," + completionTestDFWB, completionTestDFWA + "," + completionTestORD}}, + {"second element partial", completionTestDFWA + ",o", []string{completionTestDFWA + "," + completionTestDFWB, completionTestDFWA + "," + completionTestORD}}, + {"third element skips both chosen", completionTestDFWA + "," + completionTestORD + ",", []string{completionTestDFWA + "," + completionTestORD + "," + completionTestDFWB}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, directive := completeCommaList(candidates, tt.toComplete) + assert.Equal(t, tt.want, got, "the shell filters by prefix, so candidates carry the typed elements") + assert.Equal(t, cobra.ShellCompDirectiveNoFileComp|cobra.ShellCompDirectiveNoSpace, directive) + }) + } +} From 3a081cf077fec42ecc12fe00608fa4d1f4323409 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 16:56:30 -0500 Subject: [PATCH 05/12] feat: place only where compute is available A location can be Ready for the platform generally yet have no compute cell behind it. The platform records where each service runs as a ServiceAvailability and mirrors those records into entitled projects, so placement now reads them: a location is placeable when it is Ready and a record for the compute service names it and reports Available. Admission, selector resolution, placement status, and shell completion all share that one predicate. A control plane that does not serve the kind enforces no availability gate, so placement there behaves as before the mirror existed. The workload controller watches the mirrored records, per control plane and only where the kind is served, so a placement starts or stops running at a location as compute availability there changes. The ClusterRole grants the watch. Co-Authored-By: Claude Fable 5.1 --- cmd/main_test.go | 1 + config/components/controller_rbac/role.yaml | 1 + internal/agent/catalog.go | 10 +- internal/cmd/compute/util/client.go | 7 ++ internal/cmd/compute/util/completion.go | 75 +++++++++---- internal/cmd/compute/util/completion_test.go | 19 +++- .../controller/instance_controller_test.go | 2 + internal/controller/location_source_test.go | 2 +- internal/controller/workload_controller.go | 77 +++++++++----- .../controller/workload_controller_test.go | 80 +++++++++++++- .../workloaddeployment_location_test.go | 2 + internal/locations/locations.go | 95 +++++++++++++++-- internal/locations/locations_envtest_test.go | 92 +++++++++++++++- internal/locations/locations_test.go | 100 +++++++++++++++++- internal/validation/workload_validation.go | 11 +- internal/webhook/v1alpha/workload_webhook.go | 10 +- 16 files changed, 500 insertions(+), 84 deletions(-) diff --git a/cmd/main_test.go b/cmd/main_test.go index db4d7189..df27d639 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -140,6 +140,7 @@ func TestControllerRoleGrantsPlacementLocationWatches(t *testing.T) { {"networking.datumapis.com", "servinglocations"}, {"locations.miloapis.com", "locations"}, {"locations.miloapis.com", "servinglocations"}, + {"services.miloapis.com", "serviceavailabilities"}, } { for _, verb := range []string{"get", "list", "watch"} { assert.Truef(t, granted(resource.group, resource.name, verb), diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index 5e9e0500..09a1d57e 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -137,6 +137,7 @@ rules: - apiGroups: - services.miloapis.com resources: + - serviceavailabilities - serviceconsumers - services verbs: diff --git a/internal/agent/catalog.go b/internal/agent/catalog.go index 57b37855..de6140ce 100644 --- a/internal/agent/catalog.go +++ b/internal/agent/catalog.go @@ -519,11 +519,11 @@ var catalog = []ReasonInfo{ ConditionTypes: []string{computev1alpha.WorkloadAvailable}, Actionability: ActionabilityUser, Explanation: "This placement resolves to no location, so nothing was created for it. Either none " + - "of the locations it names is ready, or its location selector matches none of the " + - "project's ready locations.", - Remediation: "Compare the placement against the project's locations and their topology " + - "(city code, region). Name a location that exists, or widen the selector until it " + - "matches one.", + "of the locations it names is ready with compute available, or its location selector " + + "matches none of the project's locations that are.", + Remediation: "Compare the placement against the project's locations, their topology " + + "(city code, region), and where compute is available. Name a location that exists and " + + "offers compute, or widen the selector until it matches one.", Skill: SkillWorkloadNotAvailable, }, diff --git a/internal/cmd/compute/util/client.go b/internal/cmd/compute/util/client.go index aa6fa536..24853e42 100644 --- a/internal/cmd/compute/util/client.go +++ b/internal/cmd/compute/util/client.go @@ -9,6 +9,7 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" quotav1alpha1 "go.miloapis.com/milo/pkg/apis/quota/v1alpha1" + servicesv1alpha1 "go.miloapis.com/service-catalog/api/v1alpha1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -56,6 +57,9 @@ func NewClient(project string) (client.Client, error) { if err := locationsv1alpha1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("registering locations scheme: %w", err) } + if err := servicesv1alpha1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("registering services scheme: %w", err) + } if err := quotav1alpha1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("registering quota scheme: %w", err) } @@ -85,6 +89,9 @@ func NewPlatformClient() (client.Client, error) { if err := locationsv1alpha1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("registering locations scheme: %w", err) } + if err := servicesv1alpha1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("registering services scheme: %w", err) + } if err := quotav1alpha1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("registering quota scheme: %w", err) } diff --git a/internal/cmd/compute/util/completion.go b/internal/cmd/compute/util/completion.go index f79b2455..92419e5b 100644 --- a/internal/cmd/compute/util/completion.go +++ b/internal/cmd/compute/util/completion.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/locations" "go.datum.net/datumctl/plugin" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" ) @@ -48,47 +49,63 @@ func CompleteLocations(cmd *cobra.Command, _ []string, toComplete string) ([]str if !ok { return nil, cobra.ShellCompDirectiveNoFileComp } - return completeCommaList(locationCandidates(list, false), toComplete) + return completeCommaList(locationCandidates(list, nil), toComplete) } // CompletePlacementLocations completes a deploy-time --location flag with the // locations a placement may name: those projected into the project that are -// Ready. Admission rejects any other, so they are not offered. +// Ready and where compute is available. Admission rejects any other, so they +// are not offered. func CompletePlacementLocations(cmd *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { list, ok := projectedLocations(cmd) if !ok { return nil, cobra.ShellCompDirectiveNoFileComp } - return completeCommaList(locationCandidates(list, true), toComplete) + return completeCommaList(locationCandidates(list, placeable(cmd)), toComplete) } -// CompleteCityCodes completes --city with the city codes of the project's -// Ready locations. +// CompleteCityCodes completes --city with the city codes of the locations a +// placement may run at. func CompleteCityCodes(cmd *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { list, ok := projectedLocations(cmd) if !ok { return nil, cobra.ShellCompDirectiveNoFileComp } - return completeCommaList(cityCodeCandidates(list), toComplete) + return completeCommaList(cityCodeCandidates(list, placeable(cmd)), toComplete) } // CompleteLocationSelector completes --location-selector with the key=value -// pairs found in the topology of the project's Ready locations, so a user can -// discover which topology keys exist without reading each Location. +// pairs found in the topology of the locations a placement may run at, so a +// user can discover which topology keys exist without reading each Location. func CompleteLocationSelector(cmd *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { list, ok := projectedLocations(cmd) if !ok { return nil, cobra.ShellCompDirectiveNoFileComp } - return completeCommaList(selectorCandidates(list), toComplete) + return completeCommaList(selectorCandidates(list, placeable(cmd)), toComplete) } -// locationCandidates returns location names, sorted, optionally only the -// Ready ones. -func locationCandidates(list locationsv1alpha1.LocationList, readyOnly bool) []string { +// placeableFilter decides which projected locations a placement may run at. +// A nil filter accepts every location. +type placeableFilter func(locationsv1alpha1.Location) bool + +// placeable returns the filter a placement is held to: the location is Ready +// and compute is available there. Availability is read from the +// ServiceAvailability records the platform mirrors into the project; a project +// that does not serve them enforces no availability gate. +func placeable(cmd *cobra.Command) placeableFilter { + available, enforced := availableLocations(cmd) + return func(location locationsv1alpha1.Location) bool { + return locationIsReady(location) && (!enforced || available.Has(location.Name)) + } +} + +// locationCandidates returns the names of the locations the filter accepts, +// sorted. +func locationCandidates(list locationsv1alpha1.LocationList, accept placeableFilter) []string { names := make([]string, 0, len(list.Items)) for _, location := range list.Items { - if readyOnly && !locationIsReady(location) { + if accept != nil && !accept(location) { continue } names = append(names, location.Name) @@ -97,12 +114,12 @@ func locationCandidates(list locationsv1alpha1.LocationList, readyOnly bool) []s return names } -// cityCodeCandidates returns the distinct city codes of Ready locations, -// sorted. -func cityCodeCandidates(list locationsv1alpha1.LocationList) []string { +// cityCodeCandidates returns the distinct city codes of the locations the +// filter accepts, sorted. +func cityCodeCandidates(list locationsv1alpha1.LocationList, accept placeableFilter) []string { codes := sets.New[string]() for _, location := range list.Items { - if !locationIsReady(location) { + if accept != nil && !accept(location) { continue } if code := location.Spec.Topology[locationsv1alpha1.TopologyCityCodeKey]; code != "" { @@ -113,12 +130,12 @@ func cityCodeCandidates(list locationsv1alpha1.LocationList) []string { } // selectorCandidates returns every distinct key=value pair in the topology of -// Ready locations, sorted, which is what a selector on those locations can -// match. -func selectorCandidates(list locationsv1alpha1.LocationList) []string { +// the locations the filter accepts, sorted, which is what a selector on those +// locations can match. +func selectorCandidates(list locationsv1alpha1.LocationList, accept placeableFilter) []string { pairs := sets.New[string]() for _, location := range list.Items { - if !locationIsReady(location) { + if accept != nil && !accept(location) { continue } for key, value := range location.Spec.Topology { @@ -159,6 +176,22 @@ func completeCommaList(candidates []string, toComplete string) ([]string, cobra. return completions, cobra.ShellCompDirectiveNoFileComp | cobra.ShellCompDirectiveNoSpace } +// availableLocations returns the locations where compute is available, and +// whether the project serves availability at all. On any failure it reports +// the gate as not enforced, so completion degrades to Ready locations rather +// than offering nothing. +func availableLocations(cmd *cobra.Command) (sets.Set[string], bool) { + c, err := NewClient(ProjectFromCmd(cmd)) + if err != nil { + return nil, false + } + available, enforced, err := locations.AvailableLocations(context.Background(), c) + if err != nil { + return nil, false + } + return available, enforced +} + func projectedLocations(cmd *cobra.Command) (locationsv1alpha1.LocationList, bool) { project := ProjectFromCmd(cmd) c, err := NewClient(project) diff --git a/internal/cmd/compute/util/completion_test.go b/internal/cmd/compute/util/completion_test.go index 3c336e46..7200cb6b 100644 --- a/internal/cmd/compute/util/completion_test.go +++ b/internal/cmd/compute/util/completion_test.go @@ -45,18 +45,29 @@ func completionTestList() locationsv1alpha1.LocationList { }} } +// readyOnly is the filter a placement is held to when no availability gate is +// enforced; onlyDFWA is the same with compute available in one location. +var ( + readyOnly placeableFilter = locationIsReady + onlyDFWA placeableFilter = func(location locationsv1alpha1.Location) bool { + return locationIsReady(location) && location.Name == completionTestDFWA + } +) + func TestLocationCandidates(t *testing.T) { t.Parallel() list := completionTestList() - assert.Equal(t, []string{completionTestDFWA, completionTestDFWB, "lhr", completionTestORD}, locationCandidates(list, false), + assert.Equal(t, []string{completionTestDFWA, completionTestDFWB, "lhr", completionTestORD}, locationCandidates(list, nil), "list filters may name a location that is not Ready") - assert.Equal(t, []string{completionTestDFWA, completionTestDFWB, completionTestORD}, locationCandidates(list, true), + assert.Equal(t, []string{completionTestDFWA, completionTestDFWB, completionTestORD}, locationCandidates(list, readyOnly), "a placement may only name a Ready location") + assert.Equal(t, []string{completionTestDFWA}, locationCandidates(list, onlyDFWA), + "the availability gate narrows what deploy offers") } func TestCityCodeCandidates(t *testing.T) { t.Parallel() - assert.Equal(t, []string{completionTestCityDFW, "ORD"}, cityCodeCandidates(completionTestList()), + assert.Equal(t, []string{completionTestCityDFW, "ORD"}, cityCodeCandidates(completionTestList(), readyOnly), "city codes are deduplicated and only Ready locations count") } @@ -67,7 +78,7 @@ func TestSelectorCandidates(t *testing.T) { locationsv1alpha1.TopologyCityCodeKey + "=ORD", completionTestRegionKey + "=us-central", completionTestRegionKey + "=us-south", - }, selectorCandidates(completionTestList()), + }, selectorCandidates(completionTestList(), readyOnly), "every topology key=value pair of a Ready location is offered once") } diff --git a/internal/controller/instance_controller_test.go b/internal/controller/instance_controller_test.go index 108511fe..3b488ca9 100644 --- a/internal/controller/instance_controller_test.go +++ b/internal/controller/instance_controller_test.go @@ -31,6 +31,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + servicesv1alpha1 "go.miloapis.com/service-catalog/api/v1alpha1" "go.datum.net/compute/internal/controller/instancecontrol" "go.datum.net/compute/internal/quota" @@ -76,6 +77,7 @@ func newTestScheme(t *testing.T) *runtime.Scheme { require.NoError(t, quotav1alpha1.AddToScheme(s)) require.NoError(t, corev1.AddToScheme(s)) require.NoError(t, locationsv1alpha1.AddToScheme(s)) + require.NoError(t, servicesv1alpha1.AddToScheme(s)) return s } diff --git a/internal/controller/location_source_test.go b/internal/controller/location_source_test.go index 9025841e..f46127e7 100644 --- a/internal/controller/location_source_test.go +++ b/internal/controller/location_source_test.go @@ -97,7 +97,7 @@ func TestGetDeploymentsForWorkload_LocationsSource(t *testing.T) { cl := fake.NewClientBuilder(). WithScheme(newLocationsServiceScheme()). - WithObjects(newLocationsServiceLocation(testLocationName, locSourceTestCityCode)). + WithObjects(newLocationsServiceLocation(testLocationName, locSourceTestCityCode), newTestComputeAvailability(testLocationName)). WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). Build() diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index 4e5d393c..76860df6 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -34,6 +34,7 @@ import ( "go.datum.net/compute/internal/locations" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + servicesv1alpha1 "go.miloapis.com/service-catalog/api/v1alpha1" ) const ( @@ -63,6 +64,7 @@ type WorkloadReconciler struct { // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloads/finalizers,verbs=update // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=locationbindings,verbs=get;list;watch +// +kubebuilder:rbac:groups=services.miloapis.com,resources=serviceavailabilities,verbs=get;list;watch // +kubebuilder:rbac:groups=locations.miloapis.com,resources=locations,verbs=get;list;watch func (r *WorkloadReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { @@ -342,15 +344,15 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( if len(sortedDeployments) == 0 { // Nothing was created for this placement: none of the locations it - // names is Ready, or its selector matched no Ready location. The - // user resolves this by changing the placement, so it outranks the - // generic no-deployments answer. + // names is Ready with compute available, or its selector matched + // no such location. The user resolves this by changing the + // placement, so it outranks the generic no-deployments answer. placementAvailableCondition.Reason = computev1alpha.WorkloadReasonNoMatchingLocations - placementAvailableCondition.Message = "No Ready location matches this placement, so it has nowhere to run" + placementAvailableCondition.Message = "No location that is Ready and offers compute matches this placement, so it has nowhere to run" if p := workloadBlockingReasonPriority(placementAvailableCondition.Reason); p > worstPriority { worstPriority = p worstReason = placementAvailableCondition.Reason - worstMessage = fmt.Sprintf("Placement %q matches no Ready location", placementName) + worstMessage = fmt.Sprintf("Placement %q matches no location that is Ready and offers compute", placementName) } } @@ -484,13 +486,13 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( return nil, nil, fmt.Errorf("no locations are registered with the system") } - readyLocations := locations.ReadyNames(placementLocations) + placeableLocations := locations.PlaceableNames(placementLocations) // Remember this: namespace, name, err := cache.SplitMetaNamespaceKey(key) for _, placement := range workload.Spec.Placements { // A placement that resolves to nothing is reported on its status by // reconcileWorkloadStatus rather than failing the workload here. - locationNames, err := resolvePlacementLocations(placement, placementLocations, readyLocations) + locationNames, err := resolvePlacementLocations(placement, placementLocations, placeableLocations) if err != nil { return nil, nil, fmt.Errorf("placement %q: %w", placement.Name, err) } @@ -540,14 +542,15 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( return desired, orphaned, nil } -// resolvePlacementLocations returns the names of the Ready locations a -// placement runs at, in a stable order: the locations it names that are Ready, -// or every Ready location its selector matches. A placement that names -// locations keeps their declared order; a selector yields locations by name. +// resolvePlacementLocations returns the names of the locations a placement +// runs at, in a stable order: the locations it names that are placeable, or +// every placeable location its selector matches. A location is placeable when +// it is Ready and compute is available there. A placement that names locations +// keeps their declared order; a selector yields locations by name. func resolvePlacementLocations( placement computev1alpha.WorkloadPlacement, available []locations.PlacementLocation, - ready sets.Set[string], + placeable sets.Set[string], ) ([]string, error) { if placement.LocationSelector != nil { matched, err := locations.Select(available, placement.LocationSelector) @@ -563,7 +566,7 @@ func resolvePlacementLocations( names := make([]string, 0, len(placement.Locations)) for _, ref := range placement.Locations { - if ready.Has(ref.Name) { + if placeable.Has(ref.Name) { names = append(names, ref.Name) } } @@ -593,24 +596,28 @@ func (r *WorkloadReconciler) SetupWithManager(mgr mcmanager.Manager) error { } // A placement that selects locations by topology gains and loses - // deployments as locations come and go, and one that names a location not - // yet Ready starts running when it becomes Ready. Neither has any other - // wake-up event, so every workload in the project is re-reconciled when - // its placement locations change. The kind watched follows the location - // source, which every reconcile already reads. + // deployments as locations come and go, one that names a location not yet + // Ready starts running when it becomes Ready, and any placement starts or + // stops running at a location as compute availability there changes. None + // of these has any other wake-up event, so every workload in the project + // is re-reconciled when its placement locations or their availability + // change. The location kind watched follows the location source, which + // every reconcile already reads. placementLocationObject, err := locations.PlacementLocationObject(r.LocationSource) if err != nil { return err } + enqueueAll := func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []mcreconcile.Request { + return enqueueAllWorkloads(ctx, cl.GetClient(), clusterName) + }) + } + b := mcbuilder.ControllerManagedBy(mgr). For(&computev1alpha.Workload{}, mcbuilder.WithEngageWithLocalCluster(false)). Owns(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)). - Watches(placementLocationObject, func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { - return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []mcreconcile.Request { - return enqueueAllWorkloads(ctx, cl.GetClient(), clusterName) - }) - }, + Watches(placementLocationObject, enqueueAll, // Workloads live in project control planes, never in the // management cluster this manager runs against, so the management // cluster is not watched and is not asked to serve the kind. The @@ -620,6 +627,12 @@ func (r *WorkloadReconciler) SetupWithManager(mgr mcmanager.Manager) error { // A control plane that does not serve the kind is skipped rather // than watched. See placementLocationClusterFilter. mcbuilder.WithClusterFilter(placementLocationClusterFilter(r.LocationSource)), + ). + Watches(&servicesv1alpha1.ServiceAvailability{}, enqueueAll, + mcbuilder.WithEngageWithLocalCluster(false), + // A control plane that does not mirror availability enforces no + // availability gate, and is skipped for the same reason as above. + mcbuilder.WithClusterFilter(servedKindClusterFilter("service availability", locations.ServesServiceAvailabilityKind)), ) if !r.NetworkingEnabled { @@ -748,17 +761,27 @@ func deploymentLocations(deployments []computev1alpha.WorkloadDeployment) []loca // it. A control plane that gains the kind later is picked up the next time it // is engaged. func placementLocationClusterFilter(source locations.Source) mcbuilder.ClusterFilterFunc { + return servedKindClusterFilter("placement location", func(mapper apimeta.RESTMapper) (bool, error) { + return locations.ServesPlacementLocationKind(mapper, source) + }) +} + +// servedKindClusterFilter keeps a watch off any control plane that does not +// serve its kind, as reported by serves against that control plane's mapper. +// The question is asked per control plane, since only the one being engaged +// can answer it; a control plane that gains the kind later is picked up the +// next time it is engaged. +func servedKindClusterFilter(what string, serves func(apimeta.RESTMapper) (bool, error)) mcbuilder.ClusterFilterFunc { return func(clusterName multicluster.ClusterName, cl cluster.Cluster) bool { logger := log.Log.WithName("workload").WithValues("cluster", clusterName) - served, err := locations.ServesPlacementLocationKind(cl.GetRESTMapper(), source) + served, err := serves(cl.GetRESTMapper()) if err != nil { - logger.Error(err, "failed to determine whether the cluster serves placement locations; not watching them") + logger.Error(err, "failed to determine whether the cluster serves a kind; not watching it", "kind", what) return false } if !served { - logger.Info("cluster does not serve the placement location kind; not watching placement locations", - "locationSource", source) + logger.Info("cluster does not serve the kind; not watching it", "kind", what) } return served } diff --git a/internal/controller/workload_controller_test.go b/internal/controller/workload_controller_test.go index b1addc0c..413a8f7d 100644 --- a/internal/controller/workload_controller_test.go +++ b/internal/controller/workload_controller_test.go @@ -16,10 +16,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/locations" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + servicesv1alpha1 "go.miloapis.com/service-catalog/api/v1alpha1" ) +// newTestComputeAvailability returns the record the platform mirrors into a +// project to say compute is deployed and validated at the location. +func newTestComputeAvailability(location string) *servicesv1alpha1.ServiceAvailability { + return &servicesv1alpha1.ServiceAvailability{ + ObjectMeta: metav1.ObjectMeta{Name: "compute--" + location}, + Spec: servicesv1alpha1.ServiceAvailabilitySpec{ + ServiceRef: servicesv1alpha1.ServiceRef{Name: locations.ComputeServiceName}, + LocationRef: servicesv1alpha1.LocationRef{Name: location}, + }, + Status: servicesv1alpha1.ServiceAvailabilityStatus{ + Conditions: []metav1.Condition{{Type: "Available", Status: metav1.ConditionTrue}}, + }, + } +} + // newTestLocationBinding builds the projection a project control plane holds // today for a location it may place workloads at. func newTestLocationBinding(name, cityCode string) *networkingv1alpha.LocationBinding { @@ -122,7 +139,7 @@ func TestGetDeploymentsForWorkload_InitializesReplicas(t *testing.T) { require.NoError(t, locationsv1alpha1.AddToScheme(s)) cl := fake.NewClientBuilder(). WithScheme(s). - WithObjects(location). + WithObjects(location, newTestComputeAvailability(testLocationName)). WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). Build() r := &WorkloadReconciler{} @@ -377,6 +394,9 @@ func TestGetDeploymentsForWorkload_LocationSelector(t *testing.T) { newTestLocationBinding("dfw-b", "DFW"), newTestLocationBinding("dfw-a", "DFW"), newTestLocationBinding(testOtherLocationName, "ORD"), + newTestComputeAvailability("dfw-a"), + newTestComputeAvailability("dfw-b"), + newTestComputeAvailability(testOtherLocationName), ). WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). Build() @@ -433,7 +453,11 @@ func TestGetDeploymentsForWorkload_LocationSelectorFollowsLocations(t *testing.T cl := fake.NewClientBuilder(). WithScheme(newNetworkingScheme()). - WithObjects(newTestLocationBinding(testLocationName, "DFW"), newTestLocationBinding(testOtherLocationName, "ORD"), stale). + WithObjects( + newTestLocationBinding(testLocationName, "DFW"), newTestLocationBinding(testOtherLocationName, "ORD"), + newTestComputeAvailability(testLocationName), newTestComputeAvailability(testOtherLocationName), + stale, + ). WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). Build() r := &WorkloadReconciler{} @@ -490,3 +514,55 @@ func TestReconcileWorkloadStatus_ReportsResolvedLocations(t *testing.T) { require.Len(t, workload.Status.Placements, 1) assert.Equal(t, []locationsv1alpha1.LocationReference{{Name: testLocationName}, {Name: testOtherLocationName}}, workload.Status.Placements[0].Locations) } + +// TestGetDeploymentsForWorkload_RequiresComputeAvailability verifies the +// availability gate: a Ready location the project may use, but where compute +// is not available, receives no deployment, whether named or selected. +func TestGetDeploymentsForWorkload_RequiresComputeAvailability(t *testing.T) { + t.Parallel() + + workload := &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: rdTestWorkloadName, + Namespace: testDefaultNamespace, + UID: types.UID("workload-uid"), + }, + Spec: computev1alpha.WorkloadSpec{ + Placements: []computev1alpha.WorkloadPlacement{ + { + Name: "named", + Locations: []locationsv1alpha1.LocationReference{{Name: testLocationName}, {Name: testOtherLocationName}}, + ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, + }, + { + Name: "selected", + LocationSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: networkingv1alpha.TopologyCityCodeKey, Operator: metav1.LabelSelectorOpExists, + }}, + }, + ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: 1}, + }, + }, + }, + } + + cl := fake.NewClientBuilder(). + WithScheme(newNetworkingScheme()). + WithObjects( + newTestLocationBinding(testLocationName, "DFW"), + newTestLocationBinding(testOtherLocationName, "ORD"), + // Compute is only available in one of the two locations. + newTestComputeAvailability(testLocationName), + ). + WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). + Build() + r := &WorkloadReconciler{} + + desired, _, err := r.getDeploymentsForWorkload(context.Background(), cl, workload) + require.NoError(t, err) + require.Len(t, desired, 2, "each placement lands only where compute is available") + for _, deployment := range desired { + assert.Equal(t, testLocationName, deployment.Spec.LocationRef.Name) + } +} diff --git a/internal/controller/workloaddeployment_location_test.go b/internal/controller/workloaddeployment_location_test.go index f9c5732c..497a1dd9 100644 --- a/internal/controller/workloaddeployment_location_test.go +++ b/internal/controller/workloaddeployment_location_test.go @@ -23,6 +23,7 @@ import ( "go.datum.net/compute/internal/locations" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + servicesv1alpha1 "go.miloapis.com/service-catalog/api/v1alpha1" "go.datum.net/compute/internal/controller/instancecontrol" ) @@ -44,6 +45,7 @@ func newNetworkingScheme() *runtime.Scheme { _ = computev1alpha.AddToScheme(s) _ = networkingv1alpha.AddToScheme(s) _ = locationsv1alpha1.AddToScheme(s) + _ = servicesv1alpha1.AddToScheme(s) return s } diff --git a/internal/locations/locations.go b/internal/locations/locations.go index 4b867357..9dd2a9c0 100644 --- a/internal/locations/locations.go +++ b/internal/locations/locations.go @@ -26,8 +26,14 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + servicesv1alpha1 "go.miloapis.com/service-catalog/api/v1alpha1" ) +// ComputeServiceName is the name the platform records compute availability +// under: a ServiceAvailability whose spec.serviceRef.name is this value says +// compute is deployed and validated at spec.locationRef.name. +const ComputeServiceName = "compute" + const ( // TopologyCityCodeKey is the topology key holding a location's city. TopologyCityCodeKey = locationsv1alpha1.TopologyCityCodeKey @@ -68,11 +74,25 @@ type PlacementLocation struct { Name string Topology map[string]string - // Ready reports whether the location accepts placements. A Location read + // Ready reports whether the location itself is serving. A Location read // from the locations service is Ready when its Ready condition is true. A // LocationBinding carries no readiness contract that compute reads, so // every binding is Ready. Ready bool + + // ServiceAvailable reports whether compute is deployed and validated at + // the location, read from the ServiceAvailability the platform mirrors + // into the project. A location can be Ready for the platform generally + // yet have no compute cell behind it; this is what tells the two apart. + // A control plane that does not serve the kind enforces no such gate, so + // every location there is available. + ServiceAvailable bool +} + +// Placeable reports whether a placement may run at the location: it is Ready +// and compute is available there. +func (l PlacementLocation) Placeable() bool { + return l.Ready && l.ServiceAvailable } // CityCode returns the city the location serves, and whether it declares one. @@ -114,7 +134,7 @@ func ListPlacementLocations(ctx context.Context, c client.Client, source Source) Ready: true, }) } - return found, nil + return markServiceAvailability(ctx, c, found) } var list locationsv1alpha1.LocationList @@ -133,9 +153,64 @@ func ListPlacementLocations(ctx context.Context, c client.Client, source Source) Ready: apimeta.IsStatusConditionTrue(location.Status.Conditions, locationsv1alpha1.LocationConditionReady), }) } + return markServiceAvailability(ctx, c, found) +} + +// markServiceAvailability sets ServiceAvailable on each location from the +// ServiceAvailability records the platform mirrors into the project. +// +// A location is available when a record for the compute service names it and +// reports Available. A control plane that does not serve the kind has no +// availability to consult, so every location is marked available and the +// Ready gate alone decides, which is what placement did before the platform +// began mirroring availability. +func markServiceAvailability(ctx context.Context, c client.Client, found []PlacementLocation) ([]PlacementLocation, error) { + available, enforced, err := AvailableLocations(ctx, c) + if err != nil { + return nil, err + } + for i := range found { + found[i].ServiceAvailable = !enforced || available.Has(found[i].Name) + } return found, nil } +// AvailableLocations returns the names of the locations where compute is +// available, and whether the control plane serves availability at all. When +// it does not, enforced is false and the set is empty. +func AvailableLocations(ctx context.Context, c client.Client) (available sets.Set[string], enforced bool, err error) { + var list servicesv1alpha1.ServiceAvailabilityList + if err := c.List(ctx, &list); err != nil { + if kindNotInstalled(err) { + return sets.Set[string]{}, false, nil + } + return nil, false, fmt.Errorf("failed to list service availabilities: %w", err) + } + + available = sets.Set[string]{} + for _, availability := range list.Items { + if availability.Spec.ServiceRef.Name != ComputeServiceName { + continue + } + if apimeta.IsStatusConditionTrue(availability.Status.Conditions, "Available") { + available.Insert(availability.Spec.LocationRef.Name) + } + } + return available, true, nil +} + +// ServiceAvailabilityGVK returns the kind a controller watches to learn that +// compute availability at a location changed. +func ServiceAvailabilityGVK() schema.GroupVersionKind { + return servicesv1alpha1.GroupVersion.WithKind("ServiceAvailability") +} + +// ServesServiceAvailabilityKind reports whether the control plane serves +// ServiceAvailability, so a watch on it is safe to register. +func ServesServiceAvailabilityKind(mapper apimeta.RESTMapper) (bool, error) { + return servesKind(mapper, ServiceAvailabilityGVK()) +} + // ListServingLocations returns the locations delivered to a cell. func ListServingLocations(ctx context.Context, c client.Client, source Source) ([]ServingLocation, error) { resolved, err := source.Resolve() @@ -195,7 +270,7 @@ func Select(found []PlacementLocation, selector *metav1.LabelSelector) ([]Placem var matched []PlacementLocation for _, location := range found { - if location.Ready && sel.Matches(labels.Set(location.Topology)) { + if location.Placeable() && sel.Matches(labels.Set(location.Topology)) { matched = append(matched, location) } } @@ -245,14 +320,19 @@ func ServesPlacementLocationKind(mapper apimeta.RESTMapper, source Source) (bool if err != nil { return false, err } + return servesKind(mapper, gvk) +} +// servesKind reports whether the control plane behind the mapper serves the +// kind. A kind that is not installed reads as not served; any other mapper +// failure is returned, since it says nothing about the kind. +func servesKind(mapper apimeta.RESTMapper, gvk schema.GroupVersionKind) (bool, error) { if _, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version); err != nil { if kindNotInstalled(err) { return false, nil } return false, fmt.Errorf("failed to determine whether %s is served: %w", gvk, err) } - return true, nil } @@ -324,11 +404,12 @@ func otherSource(source Source) Source { return SourceLocations } -// ReadyNames returns the names of the given locations that accept placements. -func ReadyNames(found []PlacementLocation) sets.Set[string] { +// PlaceableNames returns the names of the given locations a placement may run +// at: those that are Ready and where compute is available. +func PlaceableNames(found []PlacementLocation) sets.Set[string] { names := sets.Set[string]{} for _, location := range found { - if location.Ready { + if location.Placeable() { names.Insert(location.Name) } } diff --git a/internal/locations/locations_envtest_test.go b/internal/locations/locations_envtest_test.go index 02b7a4a2..c33b7ddc 100644 --- a/internal/locations/locations_envtest_test.go +++ b/internal/locations/locations_envtest_test.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + servicesv1alpha1 "go.miloapis.com/service-catalog/api/v1alpha1" ) // locationsCRDDir resolves the CRDs shipped by the locations module, so the @@ -37,6 +38,25 @@ func locationsCRDDir(t *testing.T) string { return dir } +// envtestLocationName is the one location every subtest below creates and +// reads back. +const envtestLocationName = "dfw" + +// serviceAvailabilityCRD resolves the ServiceAvailability CRD shipped by the +// service-catalog module, for the same reason as locationsCRDDir. +func serviceAvailabilityCRD(t *testing.T) string { + t.Helper() + + out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", "go.miloapis.com/service-catalog").Output() + require.NoError(t, err, "the service-catalog module must be resolvable") + + path := filepath.Join(strings.TrimSpace(string(out)), "config", "base", "crd", "bases", + "services.miloapis.com_serviceavailabilities.yaml") + _, err = os.Stat(path) + require.NoError(t, err) + return path +} + // TestLocationsSource_AgainstAPIServer is the runtime half of the typed switch. // Compiling against the locations types proves nothing about whether a client // can resolve the kinds, so this runs both states against a real API server: @@ -101,9 +121,9 @@ func TestLocationsSource_AgainstAPIServer(t *testing.T) { t.Run("CRDs installed resolve and read back", func(t *testing.T) { c := newClient() - require.NoError(t, c.Create(ctx, newLocation("dfw", testCityCode))) + require.NoError(t, c.Create(ctx, newLocation(envtestLocationName, testCityCode))) require.NoError(t, c.Create(ctx, &locationsv1alpha1.ServingLocation{ - ObjectMeta: metav1.ObjectMeta{Name: "dfw"}, + ObjectMeta: metav1.ObjectMeta{Name: envtestLocationName}, Spec: locationsv1alpha1.ServingLocationSpec{ Topology: map[string]string{TopologyCityCodeKey: testCityCode}, }, @@ -117,7 +137,73 @@ func TestLocationsSource_AgainstAPIServer(t *testing.T) { serving, err := ListServingLocations(ctx, c, SourceLocations) require.NoError(t, err) require.Len(t, serving, 1) - assert.Equal(t, "dfw", serving[0].Name) + assert.Equal(t, envtestLocationName, serving[0].Name) assert.Equal(t, testCityCode, serving[0].CityCode()) }) + + // markReady sets the Ready condition on the location through the status + // subresource, the way the locations controller does. + markReady := func(t *testing.T, c client.Client) { + t.Helper() + var location locationsv1alpha1.Location + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: envtestLocationName}, &location)) + location.Status.Conditions = []metav1.Condition{{ + Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue, + Reason: "Serving", LastTransitionTime: metav1.Now(), + }} + require.NoError(t, c.Status().Update(ctx, &location)) + } + + t.Run("availability CRD absent enforces no gate", func(t *testing.T) { + c := newClient() + markReady(t, c) + + // The absence must be recognisable for the same reason as above: a + // wrapping change would turn every project without the mirror into + // one where nothing can be placed. + var absent servicesv1alpha1.ServiceAvailabilityList + rawErr := c.List(ctx, &absent) + require.Error(t, rawErr, "the CRD really is absent, so the degrade is under test") + assert.True(t, kindNotInstalled(rawErr), "an absent CRD must stay recognisable as such: %T / %v", rawErr, rawErr) + + found, err := ListPlacementLocations(ctx, c, SourceLocations) + require.NoError(t, err) + require.Len(t, found, 1) + assert.True(t, found[0].Placeable(), "with no availability to consult, Ready alone decides") + }) + + _, err = envtest.InstallCRDs(cfg, envtest.CRDInstallOptions{ + Paths: []string{serviceAvailabilityCRD(t)}, + }) + require.NoError(t, err) + + t.Run("availability CRD installed gates on a compute record", func(t *testing.T) { + c := newClient() + + found, err := ListPlacementLocations(ctx, c, SourceLocations) + require.NoError(t, err) + require.Len(t, found, 1) + assert.False(t, found[0].Placeable(), "the kind is served but no record says compute runs here") + + availability := newComputeAvailability(envtestLocationName) + conditions := availability.Status.Conditions + availability.Status = servicesv1alpha1.ServiceAvailabilityStatus{} + require.NoError(t, c.Create(ctx, availability)) + for i := range conditions { + conditions[i].Reason = "Available" + conditions[i].LastTransitionTime = metav1.Now() + } + availability.Status.Conditions = conditions + require.NoError(t, c.Status().Update(ctx, availability)) + + found, err = ListPlacementLocations(ctx, c, SourceLocations) + require.NoError(t, err) + require.Len(t, found, 1) + assert.True(t, found[0].Placeable(), "an Available compute record opens the gate") + + available, enforced, err := AvailableLocations(ctx, c) + require.NoError(t, err) + assert.True(t, enforced) + assert.Equal(t, []string{envtestLocationName}, available.UnsortedList()) + }) } diff --git a/internal/locations/locations_test.go b/internal/locations/locations_test.go index 7aa2986a..29f2ef43 100644 --- a/internal/locations/locations_test.go +++ b/internal/locations/locations_test.go @@ -12,12 +12,14 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + servicesv1alpha1 "go.miloapis.com/service-catalog/api/v1alpha1" ) const ( @@ -35,9 +37,35 @@ func testScheme(t *testing.T) *runtime.Scheme { s := runtime.NewScheme() require.NoError(t, networkingv1alpha.AddToScheme(s)) require.NoError(t, locationsv1alpha1.AddToScheme(s)) + require.NoError(t, servicesv1alpha1.AddToScheme(s)) return s } +// newAvailability returns the mirrored record saying the named service is +// deployed at the named location, Available or not. +func newAvailability(service, location string, available bool) *servicesv1alpha1.ServiceAvailability { + status := metav1.ConditionFalse + if available { + status = metav1.ConditionTrue + } + return &servicesv1alpha1.ServiceAvailability{ + ObjectMeta: metav1.ObjectMeta{Name: service + "--" + location}, + Spec: servicesv1alpha1.ServiceAvailabilitySpec{ + ServiceRef: servicesv1alpha1.ServiceRef{Name: service}, + LocationRef: servicesv1alpha1.LocationRef{Name: location}, + }, + Status: servicesv1alpha1.ServiceAvailabilityStatus{ + Conditions: []metav1.Condition{{Type: "Available", Status: status}}, + }, + } +} + +// newComputeAvailability returns an Available record for compute at the +// location. +func newComputeAvailability(location string) *servicesv1alpha1.ServiceAvailability { + return newAvailability(ComputeServiceName, location, true) +} + func newBinding(name, cityCode string) *networkingv1alpha.LocationBinding { return &networkingv1alpha.LocationBinding{ ObjectMeta: metav1.ObjectMeta{Name: name}, @@ -58,6 +86,13 @@ func newLocation(name, cityCode string) *locationsv1alpha1.Location { } } +// newReadyLocation is newLocation with its Ready condition set. +func newReadyLocation(name, cityCode string) *locationsv1alpha1.Location { + location := newLocation(name, cityCode) + location.Status.Conditions = []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue}} + return location +} + // TestTopologyKeysAgreeAcrossSources guards the migration's central assumption: // a city code means the same thing whichever source served it. If the two // groups ever disagree, switching sources would silently repoint every @@ -285,11 +320,11 @@ func TestSelect(t *testing.T) { region := "topology.datum.net/region" found := []PlacementLocation{ - {Name: testLocationORD, Topology: map[string]string{TopologyCityCodeKey: testOtherCityCode, region: "us-central"}, Ready: true}, - {Name: testLocationDFWB, Topology: map[string]string{TopologyCityCodeKey: testCityCode, region: "us-south"}, Ready: true}, - {Name: testLocationDFWA, Topology: map[string]string{TopologyCityCodeKey: testCityCode, region: "us-south"}, Ready: true}, + {Name: testLocationORD, Topology: map[string]string{TopologyCityCodeKey: testOtherCityCode, region: "us-central"}, Ready: true, ServiceAvailable: true}, + {Name: testLocationDFWB, Topology: map[string]string{TopologyCityCodeKey: testCityCode, region: "us-south"}, Ready: true, ServiceAvailable: true}, + {Name: testLocationDFWA, Topology: map[string]string{TopologyCityCodeKey: testCityCode, region: "us-south"}, Ready: true, ServiceAvailable: true}, {Name: "dfw-down", Topology: map[string]string{TopologyCityCodeKey: testCityCode}, Ready: false}, - {Name: "nowhere", Ready: true}, + {Name: "nowhere", Ready: true, ServiceAvailable: true}, } names := func(locations []PlacementLocation) []string { @@ -375,3 +410,60 @@ func TestPlacementLocationObject(t *testing.T) { _, err = PlacementLocationObject("Nonsense") require.Error(t, err) } + +// TestListPlacementLocations_ServiceAvailability covers the availability gate +// under both sources: a location is placeable only when a record for compute +// names it and reports Available. A record for another service, or one that +// is not Available, leaves the location Ready but not placeable. +func TestListPlacementLocations_ServiceAvailability(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + source Source + objects []client.Object + }{ + { + name: "network services", + source: SourceNetworkServices, + objects: []client.Object{ + newBinding(testLocationDFWA, testCityCode), + newBinding(testLocationORD, testOtherCityCode), + newBinding("lhr", "LHR"), + }, + }, + { + name: "locations service", + source: SourceLocations, + objects: []client.Object{ + newReadyLocation(testLocationDFWA, testCityCode), + newReadyLocation(testLocationORD, testOtherCityCode), + newReadyLocation("lhr", "LHR"), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + objects := append(tt.objects, + newComputeAvailability(testLocationDFWA), + newAvailability(ComputeServiceName, testLocationORD, false), + newAvailability("networking-datumapis-com", "lhr", true), + ) + cl := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(objects...).Build() + + found, err := ListPlacementLocations(context.Background(), cl, tt.source) + require.NoError(t, err) + require.Len(t, found, 3, "availability narrows what is placeable, not what is listed") + + byName := map[string]PlacementLocation{} + for _, location := range found { + byName[location.Name] = location + } + assert.True(t, byName[testLocationDFWA].Placeable(), "an Available compute record makes the location placeable") + assert.False(t, byName[testLocationORD].Placeable(), "a compute record that is not Available does not") + assert.False(t, byName["lhr"].Placeable(), "another service's availability says nothing about compute") + assert.Equal(t, []string{testLocationDFWA}, sets.List(PlaceableNames(found))) + }) + } +} diff --git a/internal/validation/workload_validation.go b/internal/validation/workload_validation.go index a5315d55..f9c3a6bb 100644 --- a/internal/validation/workload_validation.go +++ b/internal/validation/workload_validation.go @@ -85,8 +85,9 @@ type WorkloadValidationOptions struct { Workload *computev1alpha.Workload ValidLocations []string - // LocationTopologies is the topology of every Ready location, keyed by - // name. A placement's locationSelector must match at least one of them. + // LocationTopologies is the topology of every location a placement may run + // at (Ready, with compute available), keyed by name. A placement's + // locationSelector must match at least one of them. LocationTopologies map[string]map[string]string // RuntimeClasses is the catalog of execution tiers this control plane @@ -267,8 +268,8 @@ func validateMetricTarget(target computev1alpha.MetricTarget, fieldPath *field.P } // validateLocationSelector checks a placement's selector the way the workload -// controller will evaluate it: well formed, non-empty, and matching at least -// one Ready location's topology. A selector that matches nothing is rejected +// controller will evaluate it: well formed, non-empty, and matching the +// topology of at least one location where compute is available. A selector that matches nothing is rejected // for the same reason an unknown location name is: storing it would admit a // placement that never runs anywhere. func validateLocationSelector(selector *metav1.LabelSelector, fieldPath *field.Path, opts WorkloadValidationOptions) field.ErrorList { @@ -300,5 +301,5 @@ func validateLocationSelector(selector *metav1.LabelSelector, fieldPath *field.P } sort.Strings(names) return append(allErrs, field.Invalid(fieldPath, sel.String(), fmt.Sprintf( - "matches none of the Ready locations (%s)", strings.Join(names, ", ")))) + "matches none of the locations where compute is available (%s)", strings.Join(names, ", ")))) } diff --git a/internal/webhook/v1alpha/workload_webhook.go b/internal/webhook/v1alpha/workload_webhook.go index 47b06ed0..5df116b7 100644 --- a/internal/webhook/v1alpha/workload_webhook.go +++ b/internal/webhook/v1alpha/workload_webhook.go @@ -48,9 +48,9 @@ type workloadWebhook struct { } // readyLocations describes the locations a placement may run at: the -// locations projected into the project control plane that are Ready to accept -// workloads, as the names a placement may list and the topology a selector is -// matched against. +// locations projected into the project control plane that are Ready and where +// compute is available, as the names a placement may list and the topology a +// selector is matched against. type readyLocations struct { names []string topologies map[string]map[string]string @@ -63,11 +63,11 @@ func (r *workloadWebhook) readyLocations(ctx context.Context, c client.Client) ( } ready := readyLocations{ - names: sets.List(locations.ReadyNames(placementLocations)), + names: sets.List(locations.PlaceableNames(placementLocations)), topologies: make(map[string]map[string]string), } for _, location := range placementLocations { - if location.Ready { + if location.Placeable() { ready.topologies[location.Name] = location.Topology } } From 0f1a584169fbf7298239e1077ff9959bd089fad4 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 18:03:22 -0500 Subject: [PATCH 06/12] feat: migrate stored city codes to a location selector Workloads stored before placement moved to locations still name city codes. Pruning the field on rollout would leave their placements resolving to nothing and tear down everything they run, so the field stays, deprecated, and is rewritten into the selector --city now emits: every location whose topology.datum.net/city-code is one of the codes. Admission rewrites it on any write, so nothing new is stored with it. The workload controller rewrites a stored workload the first time it reconciles it after the rollout and writes it back before deriving deployments; until that write succeeds the workload keeps what it runs. A placement that names city codes beside locations or a selector is rejected rather than guessed at. Deployments from that era carry no location, so the federator reads the city off their hub copy while finalizing them and removes the city-keyed PropagationPolicies once no deployment routed by that city remains. The CLI and portal show unmigrated placements by their cities in the meantime. Co-Authored-By: Claude Fable 5.1 --- api/v1alpha/workload_placement.go | 55 ++++++++++++ api/v1alpha/workload_placement_test.go | 67 +++++++++++++++ api/v1alpha/workload_types.go | 14 +++- api/v1alpha/zz_generated.deepcopy.go | 5 ++ .../compute.datumapis.com_workloads.yaml | 20 ++++- docs/api/workloads.md | 13 +++ internal/cmd/compute/deploy/deploy.go | 24 ++---- internal/cmd/compute/workloads/workloads.go | 4 + internal/controller/workload_controller.go | 15 ++++ .../workloaddeployment_federator.go | 59 +++++++++++++ .../workloaddeployment_federator_test.go | 84 +++++++++++++++++++ internal/validation/workload_validation.go | 7 ++ .../validation/workload_validation_test.go | 11 +++ internal/webhook/v1alpha/workload_webhook.go | 6 ++ .../webhook/v1alpha/workload_webhook_test.go | 22 +++++ ui/consumer/src/adapter.ts | 8 +- ui/provider/src/adapter.ts | 8 +- 17 files changed, 397 insertions(+), 25 deletions(-) create mode 100644 api/v1alpha/workload_placement.go create mode 100644 api/v1alpha/workload_placement_test.go diff --git a/api/v1alpha/workload_placement.go b/api/v1alpha/workload_placement.go new file mode 100644 index 00000000..d54c1ce3 --- /dev/null +++ b/api/v1alpha/workload_placement.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" +) + +// CityCodeSelector returns the location selector that places at every +// location in the given cities: an equality for one city, an In expression +// for several. It is what the deprecated cityCodes field and the CLI's --city +// flag both stand for, so a placement written either way resolves the same. +func CityCodeSelector(cityCodes []string) *metav1.LabelSelector { + if len(cityCodes) == 1 { + return &metav1.LabelSelector{ + MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: cityCodes[0]}, + } + } + return &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: locationsv1alpha1.TopologyCityCodeKey, + Operator: metav1.LabelSelectorOpIn, + Values: append([]string(nil), cityCodes...), + }}, + } +} + +// MigrateCityCodes rewrites a placement that still names city codes into the +// equivalent locationSelector and clears the deprecated field. It reports +// whether anything changed. A placement that already names locations or a +// selector is left alone, including one that also carries city codes, which +// validation rejects rather than guessing which the author meant. +func (p *WorkloadPlacement) MigrateCityCodes() bool { + if len(p.CityCodes) == 0 || len(p.Locations) > 0 || p.LocationSelector != nil { + return false + } + p.LocationSelector = CityCodeSelector(p.CityCodes) + p.CityCodes = nil + return true +} + +// MigrateCityCodes rewrites every placement that still names city codes. It +// reports whether the spec changed, so a caller that persisted the workload +// knows to write it back. +func (w *Workload) MigrateCityCodes() bool { + migrated := false + for i := range w.Spec.Placements { + if w.Spec.Placements[i].MigrateCityCodes() { + migrated = true + } + } + return migrated +} diff --git a/api/v1alpha/workload_placement_test.go b/api/v1alpha/workload_placement_test.go new file mode 100644 index 00000000..622cb307 --- /dev/null +++ b/api/v1alpha/workload_placement_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" +) + +const ( + placementTestCityDFW = "DFW" + placementTestCityIAD = "IAD" +) + +func TestCityCodeSelector(t *testing.T) { + t.Parallel() + + assert.Equal(t, &metav1.LabelSelector{ + MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: placementTestCityDFW}, + }, CityCodeSelector([]string{placementTestCityDFW}), "one city is a plain equality") + + assert.Equal(t, &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: locationsv1alpha1.TopologyCityCodeKey, + Operator: metav1.LabelSelectorOpIn, + Values: []string{placementTestCityDFW, placementTestCityIAD}, + }}, + }, CityCodeSelector([]string{placementTestCityDFW, placementTestCityIAD}), "several cities are an In expression") +} + +// TestMigrateCityCodes covers the shim for workloads stored before placement +// moved to locations: a placement that only names city codes becomes the +// equivalent selector, and anything else is left for validation to judge. +func TestMigrateCityCodes(t *testing.T) { + t.Parallel() + + t.Run("city codes alone are rewritten", func(t *testing.T) { + t.Parallel() + w := &Workload{Spec: WorkloadSpec{Placements: []WorkloadPlacement{ + {Name: "a", CityCodes: []string{placementTestCityDFW, placementTestCityIAD}}, + {Name: "b", Locations: []locationsv1alpha1.LocationReference{{Name: "us-east-1"}}}, + }}} + + require.True(t, w.MigrateCityCodes()) + assert.Nil(t, w.Spec.Placements[0].CityCodes, "the deprecated field is cleared") + assert.Equal(t, CityCodeSelector([]string{placementTestCityDFW, placementTestCityIAD}), w.Spec.Placements[0].LocationSelector) + assert.Empty(t, w.Spec.Placements[1].LocationSelector, "a placement naming locations is untouched") + assert.False(t, w.MigrateCityCodes(), "a second pass finds nothing to do") + }) + + t.Run("city codes beside locations or a selector are left for validation", func(t *testing.T) { + t.Parallel() + w := &Workload{Spec: WorkloadSpec{Placements: []WorkloadPlacement{ + {Name: "a", CityCodes: []string{placementTestCityDFW}, Locations: []locationsv1alpha1.LocationReference{{Name: "us-east-1"}}}, + {Name: "b", CityCodes: []string{placementTestCityDFW}, LocationSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"k": "v"}}}, + }}} + + assert.False(t, w.MigrateCityCodes()) + assert.Equal(t, []string{placementTestCityDFW}, w.Spec.Placements[0].CityCodes) + assert.Equal(t, []string{placementTestCityDFW}, w.Spec.Placements[1].CityCodes) + }) +} diff --git a/api/v1alpha/workload_types.go b/api/v1alpha/workload_types.go index 09012f70..5444e3c0 100644 --- a/api/v1alpha/workload_types.go +++ b/api/v1alpha/workload_types.go @@ -131,7 +131,7 @@ type WorkloadList struct { Items []Workload `json:"items"` } -// +kubebuilder:validation:XValidation:message="exactly one of locations or locationSelector must be set",rule="has(self.locations) != has(self.locationSelector)" +// +kubebuilder:validation:XValidation:message="exactly one of locations, locationSelector, or cityCodes must be set",rule="(has(self.locations) ? 1 : 0) + (has(self.locationSelector) ? 1 : 0) + (has(self.cityCodes) ? 1 : 0) == 1" type WorkloadPlacement struct { // The name of the placement // @@ -156,6 +156,18 @@ type WorkloadPlacement struct { // +kubebuilder:validation:Optional LocationSelector *metav1.LabelSelector `json:"locationSelector,omitempty"` + // The city codes this placement was written against before placement + // moved to locations. This field is deprecated and kept only so workloads + // stored before that change keep running: admission and the workload + // controller rewrite it into a locationSelector on + // topology.datum.net/city-code, which places at every location in those + // cities, and clear it. New workloads set locations or locationSelector + // instead. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinItems=1 + CityCodes []string `json:"cityCodes,omitempty"` + // Scale settings such as minimum and maximum replica counts. // // +kubebuilder:validation:Required diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index 3c54b3bf..93044fa6 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -1361,6 +1361,11 @@ func (in *WorkloadPlacement) DeepCopyInto(out *WorkloadPlacement) { *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } + if in.CityCodes != nil { + in, out := &in.CityCodes, &out.CityCodes + *out = make([]string, len(*in)) + copy(*out, *in) + } in.ScaleSettings.DeepCopyInto(&out.ScaleSettings) } diff --git a/config/base/crd/bases/compute.datumapis.com_workloads.yaml b/config/base/crd/bases/compute.datumapis.com_workloads.yaml index 465f6189..998e6644 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloads.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloads.yaml @@ -71,6 +71,19 @@ spec: will live in, such as in a city, or region. items: properties: + cityCodes: + description: |- + The city codes this placement was written against before placement + moved to locations. This field is deprecated and kept only so workloads + stored before that change keep running: admission and the workload + controller rewrite it into a locationSelector on + topology.datum.net/city-code, which places at every location in those + cities, and clear it. New workloads set locations or locationSelector + instead. + items: + type: string + minItems: 1 + type: array locationSelector: description: |- A selector over the topology of the locations available to the project, @@ -213,9 +226,10 @@ spec: - scaleSettings type: object x-kubernetes-validations: - - message: exactly one of locations or locationSelector must be - set - rule: has(self.locations) != has(self.locationSelector) + - message: exactly one of locations, locationSelector, or cityCodes + must be set + rule: '(has(self.locations) ? 1 : 0) + (has(self.locationSelector) + ? 1 : 0) + (has(self.cityCodes) ? 1 : 0) == 1' minItems: 1 type: array template: diff --git a/docs/api/workloads.md b/docs/api/workloads.md index 13668af5..791f445d 100644 --- a/docs/api/workloads.md +++ b/docs/api/workloads.md @@ -132,6 +132,19 @@ will live in, such as in a city, or region.
    Scale settings such as minimum and maximum replica counts.
    true + + cityCodes + []string + + The city codes this placement was written against before placement +moved to locations. This field is deprecated and kept only so workloads +stored before that change keep running: admission and the workload +controller rewrite it into a locationSelector on +topology.datum.net/city-code, which places at every location in those +cities, and clear it. New workloads set locations or locationSelector +instead.
    + + false locationSelector object diff --git a/internal/cmd/compute/deploy/deploy.go b/internal/cmd/compute/deploy/deploy.go index a5342b7d..2c3925d4 100644 --- a/internal/cmd/compute/deploy/deploy.go +++ b/internal/cmd/compute/deploy/deploy.go @@ -171,7 +171,7 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err locationSelector = parsed } if len(opts.cities) > 0 { - locationSelector = citySelector(opts.cities) + locationSelector = computev1alpha.CityCodeSelector(opts.cities) } instanceType := opts.instanceType if instanceType == "" { @@ -514,27 +514,13 @@ func describePlacementLocations(p computev1alpha.WorkloadPlacement) string { if p.LocationSelector != nil { return fmt.Sprintf("selector=[%s]", metav1.FormatLabelSelector(p.LocationSelector)) } + if len(p.CityCodes) > 0 { + // Stored before placement moved to locations and not yet rewritten. + return fmt.Sprintf("cities=[%s]", strings.Join(p.CityCodes, ", ")) + } names := make([]string, 0, len(p.Locations)) for _, ref := range p.Locations { names = append(names, ref.Name) } return fmt.Sprintf("locations=[%s]", strings.Join(names, ", ")) } - -// citySelector is the selector --city stands for: every location whose -// topology places it in one of the given cities. One city is a plain -// equality; several become an In expression. -func citySelector(cities []string) *metav1.LabelSelector { - if len(cities) == 1 { - return &metav1.LabelSelector{ - MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: cities[0]}, - } - } - return &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{{ - Key: locationsv1alpha1.TopologyCityCodeKey, - Operator: metav1.LabelSelectorOpIn, - Values: cities, - }}, - } -} diff --git a/internal/cmd/compute/workloads/workloads.go b/internal/cmd/compute/workloads/workloads.go index cf29dd9d..4093ed3b 100644 --- a/internal/cmd/compute/workloads/workloads.go +++ b/internal/cmd/compute/workloads/workloads.go @@ -478,6 +478,10 @@ func placementLocationsSummary(p computev1alpha.WorkloadPlacement) string { if p.LocationSelector != nil { return "selector: " + metav1.FormatLabelSelector(p.LocationSelector) } + if len(p.CityCodes) > 0 { + // Stored before placement moved to locations and not yet rewritten. + return "cities: " + strings.Join(p.CityCodes, ", ") + } names := make([]string, 0, len(p.Locations)) for _, ref := range p.Locations { names = append(names, ref.Name) diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index 76860df6..74e54d1c 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -110,6 +110,21 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ logger.Info("reconciling workload") defer logger.Info("reconcile complete") + // A workload stored before placement moved to locations still names city + // codes. It is rewritten to the equivalent selector and written back, and + // nothing is placed from it until that write succeeds: deriving + // deployments from the old spec would resolve to nothing and tear down + // what is running. The write passes through admission, which rejects a + // city with no placeable location; the workload then keeps its existing + // deployments and the error is retried. + if workload.MigrateCityCodes() { + logger.Info("migrating placement city codes to a location selector") + if err := cl.GetClient().Update(ctx, &workload); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to migrate placement city codes: %w", err) + } + return ctrl.Result{}, nil + } + // TODO(jreese) perform extra validation on the workload now that it's been // created. // diff --git a/internal/controller/workloaddeployment_federator.go b/internal/controller/workloaddeployment_federator.go index 88746b53..3cecd4ba 100644 --- a/internal/controller/workloaddeployment_federator.go +++ b/internal/controller/workloaddeployment_federator.go @@ -240,11 +240,27 @@ func (r *WorkloadDeploymentFederator) Finalize(ctx context.Context, obj client.O Namespace: downstreamNS, }, } + + // A deployment from before placement moved to locations has no location + // to key its PropagationPolicy by; the city it was routed by survives only + // as a label on its hub copy, so it is read before that copy goes. + legacyCity := "" + if deployment.Spec.LocationRef.Name == "" { + if err := r.FederationClient.Get(ctx, client.ObjectKeyFromObject(kd), kd); client.IgnoreNotFound(err) != nil { + return finalizer.Result{}, fmt.Errorf("failed to read downstream deployment %s/%s: %w", downstreamNS, deployment.Name, err) + } + legacyCity = kd.Labels[networkingv1alpha.TopologyCityCodeKey] + } + if err := r.FederationClient.Delete(ctx, kd); client.IgnoreNotFound(err) != nil { return finalizer.Result{}, fmt.Errorf("failed to delete downstream deployment %s/%s: %w", downstreamNS, deployment.Name, err) } logger.Info("deleted downstream WorkloadDeployment", "downstreamNamespace", downstreamNS) + if deployment.Spec.LocationRef.Name == "" { + return finalizer.Result{}, r.cleanupLegacyPropagationPolicies(ctx, downstreamNS, legacyCity) + } + if err := r.cleanupPropagationPolicyIfUnused(ctx, downstreamNS, deployment.Spec.LocationRef.Name, r.propagationRuntimeClass(deployment)); err != nil { return finalizer.Result{}, err } @@ -252,6 +268,49 @@ func (r *WorkloadDeploymentFederator) Finalize(ctx context.Context, obj client.O return finalizer.Result{}, nil } +// cleanupLegacyPropagationPolicies removes the PropagationPolicies a +// pre-location federator keyed by city, once no hub deployment routed by that +// city remains. Those policies were named city- and +// city--class-, and selected on the topology.datum.net/city-code +// label, which is how the remaining users are counted. A deployment whose hub +// copy is already gone leaves no city to clean up by, and is logged. +func (r *WorkloadDeploymentFederator) cleanupLegacyPropagationPolicies(ctx context.Context, downstreamNS, city string) error { + logger := log.FromContext(ctx) + if city == "" { + logger.Info("legacy deployment carries no city on its hub copy; leaving its PropagationPolicy for a later cleanup", "downstreamNamespace", downstreamNS) + return nil + } + + var remaining computev1alpha.WorkloadDeploymentList + if err := r.FederationClient.List(ctx, &remaining, + client.InNamespace(downstreamNS), + client.MatchingLabels{networkingv1alpha.TopologyCityCodeKey: city}, + ); err != nil { + return fmt.Errorf("failed to list remaining legacy downstream deployments for city %q: %w", city, err) + } + if len(remaining.Items) > 0 { + return nil + } + + var policies karmadapolicyv1alpha1.PropagationPolicyList + if err := r.FederationClient.List(ctx, &policies, client.InNamespace(downstreamNS)); err != nil { + return fmt.Errorf("failed to list PropagationPolicies in %s: %w", downstreamNS, err) + } + + prefix := "city-" + sanitizePolicyNameSegment(city) + for i := range policies.Items { + policy := &policies.Items[i] + if policy.Name != prefix && !strings.HasPrefix(policy.Name, prefix+"-class-") { + continue + } + if err := r.FederationClient.Delete(ctx, policy); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete legacy PropagationPolicy %s/%s: %w", downstreamNS, policy.Name, err) + } + logger.Info("deleted legacy PropagationPolicy (no more deployments for city)", "policy", policy.Name, "city", city, "downstreamNamespace", downstreamNS) + } + return nil +} + // recordFederationNamespace stamps the resolved hub namespace onto the project // WorkloadDeployment. It patches only when the value changes, so it adds no // write traffic in the steady state. diff --git a/internal/controller/workloaddeployment_federator_test.go b/internal/controller/workloaddeployment_federator_test.go index 662b0698..02097c5c 100644 --- a/internal/controller/workloaddeployment_federator_test.go +++ b/internal/controller/workloaddeployment_federator_test.go @@ -21,6 +21,7 @@ import ( mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.miloapis.com/milo/pkg/downstreamclient" ) @@ -767,3 +768,86 @@ func TestWorkloadDeploymentFederator_FinalizeHoldsWhenUnresolvable(t *testing.T) _, err := r.Finalize(ctx, wd) require.Error(t, err, "an unresolvable hub namespace must hold the finalizer") } + +// TestWorkloadDeploymentFederator_FinalizesLegacyDeployment covers deleting a +// deployment stored before placement moved to locations. It has no location +// to key a PropagationPolicy by; the city it was routed by is read off its +// hub copy, and the city-keyed policies from that era are removed once no hub +// deployment routed by the city remains. Policies keyed by location are not +// touched. +func TestWorkloadDeploymentFederator_FinalizesLegacyDeployment(t *testing.T) { + t.Parallel() + + const legacyCity = "LAX" + withoutLocation := func(wd *computev1alpha.WorkloadDeployment) { + wd.Spec.LocationRef = locationsv1alpha1.LocationReference{} + } + + tests := []struct { + name string + siblingLabels map[string]string + wantCityPPs bool + }{ + { + name: "last legacy deployment for the city removes its policies", + wantCityPPs: false, + }, + { + name: "another legacy deployment routed by the city keeps them", + siblingLabels: map[string]string{networkingv1alpha.TopologyCityCodeKey: legacyCity}, + wantCityPPs: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + wd := testWorkloadDeployment(withFinalizer, withDeletionTimestamp, withoutLocation) + projectClient := newProjectFakeClient(testProjectNamespace(), wd) + + hubCopy := &computev1alpha.WorkloadDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: testWDName, + Namespace: testKarmadaNSStr, + Labels: map[string]string{networkingv1alpha.TopologyCityCodeKey: legacyCity}, + }, + } + policy := func(name string) *karmadapolicyv1alpha1.PropagationPolicy { + return &karmadapolicyv1alpha1.PropagationPolicy{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testKarmadaNSStr}} + } + karmadaObjs := []client.Object{ + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testKarmadaNSStr}}, + hubCopy, + policy("city-lax"), + policy("city-lax-class-basalt"), + policy("city-sea"), + policy(propagationPolicyNameFor(testFederatorLocation, "")), + } + if tt.siblingLabels != nil { + karmadaObjs = append(karmadaObjs, &computev1alpha.WorkloadDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "legacy-sibling", Namespace: testKarmadaNSStr, Labels: tt.siblingLabels}, + }) + } + karmadaClient := newKarmadaFakeClient(karmadaObjs...) + + r := newTestFederator(projectClient, karmadaClient) + _, err := r.Reconcile(context.Background(), reconcileRequest()) + require.NoError(t, err, "a deployment without a location must still finalize") + + ctx := context.Background() + var gone computev1alpha.WorkloadDeployment + err = karmadaClient.Get(ctx, types.NamespacedName{Name: testWDName, Namespace: testKarmadaNSStr}, &gone) + assert.True(t, apierrors.IsNotFound(err), "the hub copy is deleted") + + exists := func(name string) bool { + var pp karmadapolicyv1alpha1.PropagationPolicy + return karmadaClient.Get(ctx, types.NamespacedName{Name: name, Namespace: testKarmadaNSStr}, &pp) == nil + } + assert.Equal(t, tt.wantCityPPs, exists("city-lax")) + assert.Equal(t, tt.wantCityPPs, exists("city-lax-class-basalt")) + assert.True(t, exists("city-sea"), "another city's policies are not touched") + assert.True(t, exists(propagationPolicyNameFor(testFederatorLocation, "")), "location-keyed policies are not touched") + }) + } +} diff --git a/internal/validation/workload_validation.go b/internal/validation/workload_validation.go index f9c3a6bb..4f12c417 100644 --- a/internal/validation/workload_validation.go +++ b/internal/validation/workload_validation.go @@ -137,6 +137,13 @@ func validateWorkloadPlacement(placement computev1alpha.WorkloadPlacement, field locationsPath := fieldPath.Child("locations") selectorPath := fieldPath.Child("locationSelector") switch { + case len(placement.CityCodes) > 0: + // Admission rewrites a lone cityCodes into a locationSelector before + // validation runs, so reaching this means it was set together with + // locations or a selector, or defaulting was bypassed. Either way the + // author has to say which they meant. + allErrs = append(allErrs, field.Forbidden(fieldPath.Child("cityCodes"), + "deprecated: place with locations or a locationSelector on "+locationsv1alpha1.TopologyCityCodeKey+"; a placement that only names city codes is rewritten on admission")) case len(placement.Locations) == 0 && placement.LocationSelector == nil: allErrs = append(allErrs, field.Required(locationsPath, "one of locations or locationSelector must be set")) case len(placement.Locations) > 0 && placement.LocationSelector != nil: diff --git a/internal/validation/workload_validation_test.go b/internal/validation/workload_validation_test.go index 9dda765b..a2c070ff 100644 --- a/internal/validation/workload_validation_test.go +++ b/internal/validation/workload_validation_test.go @@ -121,6 +121,17 @@ func TestValidateWorkloads(t *testing.T) { field.Required(field.NewPath("spec.placements[0].locationSelector.matchExpressions[0].values"), ""), }, }, + "city codes are deprecated": { + workload: MakeSandboxWorkload( + "test", + func(w *computev1alpha.Workload) { + w.Spec.Placements[0].CityCodes = []string{testCityCodeDFW} + }, + ), + expectedErrors: field.ErrorList{ + field.Forbidden(field.NewPath("spec.placements[0].cityCodes"), ""), + }, + }, "location selector together with locations": { workload: MakeSandboxWorkload( "test", diff --git a/internal/webhook/v1alpha/workload_webhook.go b/internal/webhook/v1alpha/workload_webhook.go index 5df116b7..ccefd0c2 100644 --- a/internal/webhook/v1alpha/workload_webhook.go +++ b/internal/webhook/v1alpha/workload_webhook.go @@ -79,6 +79,12 @@ var _ admission.Validator[*computev1alpha.Workload] = &workloadWebhook{} // Default implements admission.Defaulter so a mutating webhook will be registered for the type. func (r *workloadWebhook) Default(ctx context.Context, workload *computev1alpha.Workload) error { + // A manifest written before placement moved to locations still names + // city codes. It is rewritten here so what is stored is what the + // controller places by, and so the stored object never carries the + // deprecated field. + workload.MigrateCityCodes() + // With the gate off there is only one runtime class, so the field stays // empty rather than recording a class name the platform does not yet honor. if features.FeatureGate.Enabled(features.RuntimeClasses) { diff --git a/internal/webhook/v1alpha/workload_webhook_test.go b/internal/webhook/v1alpha/workload_webhook_test.go index 0083faac..72ffcedd 100644 --- a/internal/webhook/v1alpha/workload_webhook_test.go +++ b/internal/webhook/v1alpha/workload_webhook_test.go @@ -64,6 +64,28 @@ func TestWorkloadWebhookDefaultGateOff(t *testing.T) { } } +// TestWorkloadWebhookDefaultMigratesCityCodes covers the shim for manifests +// and stored objects written before placement moved to locations: a placement +// that only names city codes is stored as the equivalent selector. +func TestWorkloadWebhookDefaultMigratesCityCodes(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, features.MutableFeatureGate, features.RuntimeClasses, false) + + workload := &computev1alpha.Workload{} + workload.Spec.Placements = []computev1alpha.WorkloadPlacement{{Name: "default", CityCodes: []string{"DFW"}}} + + if err := (&workloadWebhook{}).Default(context.Background(), workload); err != nil { + t.Fatalf("Default: %v", err) + } + + placement := workload.Spec.Placements[0] + if placement.CityCodes != nil { + t.Errorf("cityCodes = %v, want cleared", placement.CityCodes) + } + if placement.LocationSelector == nil || placement.LocationSelector.MatchLabels["topology.datum.net/city-code"] != "DFW" { + t.Errorf("locationSelector = %v, want a city-code selector for DFW", placement.LocationSelector) + } +} + // TestDefaultRuntimeClass covers which class a workload that selected none // records. The catalog's default marker decides it, and a catalog with no // default leaves the field empty for validation to reject. diff --git a/ui/consumer/src/adapter.ts b/ui/consumer/src/adapter.ts index 5b98e5da..87d17d25 100644 --- a/ui/consumer/src/adapter.ts +++ b/ui/consumer/src/adapter.ts @@ -69,6 +69,8 @@ interface RawWorkloadPlacement { name: string; locations?: Array<{ name: string }>; locationSelector?: RawLabelSelector; + /** Deprecated: stored before placement moved to locations and not yet rewritten. */ + cityCodes?: string[]; scaleSettings?: { minReplicas?: number; maxReplicas?: number }; } @@ -261,7 +263,11 @@ function toPlacementRegions( return { name: p.name, locations: placementLocations(p, status), - locationSelector: p.locationSelector ? formatLabelSelector(p.locationSelector) : undefined, + locationSelector: p.locationSelector + ? formatLabelSelector(p.locationSelector) + : p.cityCodes && p.cityCodes.length > 0 + ? `topology.datum.net/city-code in (${p.cityCodes.join(',')})` + : undefined, readyReplicas: ready, desiredReplicas: desired, health, diff --git a/ui/provider/src/adapter.ts b/ui/provider/src/adapter.ts index d3ca0885..e0fc8618 100644 --- a/ui/provider/src/adapter.ts +++ b/ui/provider/src/adapter.ts @@ -76,6 +76,8 @@ interface RawWorkloadPlacement { name: string; locations?: Array<{ name: string }>; locationSelector?: RawLabelSelector; + /** Deprecated: stored before placement moved to locations and not yet rewritten. */ + cityCodes?: string[]; scaleSettings?: { minReplicas?: number; maxReplicas?: number }; } @@ -215,7 +217,11 @@ function toPlacements( return { name: p.name, locations: placementLocations(p, status), - locationSelector: p.locationSelector ? formatLabelSelector(p.locationSelector) : undefined, + locationSelector: p.locationSelector + ? formatLabelSelector(p.locationSelector) + : p.cityCodes && p.cityCodes.length > 0 + ? `topology.datum.net/city-code in (${p.cityCodes.join(',')})` + : undefined, readyReplicas: ready, desiredReplicas: desired, currentReplicas: current, From 2be5c89a5d7d8f615514b2e4e4b788e0a60d881e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 18:31:21 -0500 Subject: [PATCH 07/12] feat(cli): publish workloads on a URL with --http-port Deploying a workload with an HTTP port now ends with a URL. Previously --port opened a port and gave the developer nothing to point a browser at. --http-port declares the workload an HTTP service and publishes it on a Datum-managed HTTPS URL, backed by a NetworkService and an HTTPProxy that load-balance across every city the workload runs in. --no-http removes it. --port is removed and hard-errors rather than aliasing to --http-port: silently aliasing would publish every existing workload on the next upgrade. Adds `compute open` (heroku/fly/railway convention) and a `compute domains` group for listing domains, showing one workload's URL in detail with per-city backend health, and attaching custom hostnames. Surfaces the URL in `compute workloads` and includes it in `compute destroy`. Co-Authored-By: Claude Opus 5 --- docs/enhancements/datumctl-compute-domains.md | 291 +++++++++ go.mod | 14 +- go.sum | 4 + internal/cmd/compute/deploy/deploy.go | 358 +++++++++- internal/cmd/compute/deploy/deploy_test.go | 475 ++++++++++++++ internal/cmd/compute/deploy/nohttp_test.go | 434 ++++++++++++ internal/cmd/compute/destroy/destroy.go | 273 +++++++- internal/cmd/compute/destroy/destroy_test.go | 324 +++++++++ internal/cmd/compute/destroy/leftover_test.go | 201 ++++++ internal/cmd/compute/domains/add.go | 577 ++++++++++++++++ internal/cmd/compute/domains/add_test.go | 616 ++++++++++++++++++ internal/cmd/compute/domains/domains.go | 255 ++++++++ internal/cmd/compute/domains/domains_test.go | 449 +++++++++++++ internal/cmd/compute/domains/hostname.go | 100 +++ internal/cmd/compute/domains/hostname_test.go | 108 +++ internal/cmd/compute/domains/remove.go | 193 ++++++ internal/cmd/compute/domains/remove_test.go | 208 ++++++ .../cmd/compute/domains/wait_failure_test.go | 113 ++++ internal/cmd/compute/domains/write_test.go | 204 ++++++ internal/cmd/compute/open/browser.go | 83 +++ internal/cmd/compute/open/browser_test.go | 91 +++ internal/cmd/compute/open/custom_test.go | 133 ++++ internal/cmd/compute/open/open.go | 104 +++ internal/cmd/compute/open/open_test.go | 283 ++++++++ internal/cmd/compute/root.go | 4 + internal/cmd/compute/url/lookup.go | 475 ++++++++++++++ internal/cmd/compute/url/lookup_test.go | 444 +++++++++++++ internal/cmd/compute/url/publish.go | 405 ++++++++++++ .../cmd/compute/url/publish_failure_test.go | 367 +++++++++++ internal/cmd/compute/url/publish_test.go | 233 +++++++ internal/cmd/compute/url/reason.go | 73 +++ internal/cmd/compute/url/reason_test.go | 46 ++ internal/cmd/compute/url/render.go | 174 +++++ internal/cmd/compute/url/render_test.go | 189 ++++++ internal/cmd/compute/url/resources.go | 156 +++++ internal/cmd/compute/url/resources_test.go | 227 +++++++ internal/cmd/compute/url/state_test.go | 309 +++++++++ internal/cmd/compute/workloads/filter_test.go | 176 +++++ internal/cmd/compute/workloads/workloads.go | 468 +++++++++---- .../cmd/compute/workloads/workloads_test.go | 571 ++++++++++++++++ 40 files changed, 10029 insertions(+), 179 deletions(-) create mode 100644 docs/enhancements/datumctl-compute-domains.md create mode 100644 internal/cmd/compute/deploy/deploy_test.go create mode 100644 internal/cmd/compute/deploy/nohttp_test.go create mode 100644 internal/cmd/compute/destroy/destroy_test.go create mode 100644 internal/cmd/compute/destroy/leftover_test.go create mode 100644 internal/cmd/compute/domains/add.go create mode 100644 internal/cmd/compute/domains/add_test.go create mode 100644 internal/cmd/compute/domains/domains.go create mode 100644 internal/cmd/compute/domains/domains_test.go create mode 100644 internal/cmd/compute/domains/hostname.go create mode 100644 internal/cmd/compute/domains/hostname_test.go create mode 100644 internal/cmd/compute/domains/remove.go create mode 100644 internal/cmd/compute/domains/remove_test.go create mode 100644 internal/cmd/compute/domains/wait_failure_test.go create mode 100644 internal/cmd/compute/domains/write_test.go create mode 100644 internal/cmd/compute/open/browser.go create mode 100644 internal/cmd/compute/open/browser_test.go create mode 100644 internal/cmd/compute/open/custom_test.go create mode 100644 internal/cmd/compute/open/open.go create mode 100644 internal/cmd/compute/open/open_test.go create mode 100644 internal/cmd/compute/url/lookup.go create mode 100644 internal/cmd/compute/url/lookup_test.go create mode 100644 internal/cmd/compute/url/publish.go create mode 100644 internal/cmd/compute/url/publish_failure_test.go create mode 100644 internal/cmd/compute/url/publish_test.go create mode 100644 internal/cmd/compute/url/reason.go create mode 100644 internal/cmd/compute/url/reason_test.go create mode 100644 internal/cmd/compute/url/render.go create mode 100644 internal/cmd/compute/url/render_test.go create mode 100644 internal/cmd/compute/url/resources.go create mode 100644 internal/cmd/compute/url/resources_test.go create mode 100644 internal/cmd/compute/url/state_test.go create mode 100644 internal/cmd/compute/workloads/filter_test.go create mode 100644 internal/cmd/compute/workloads/workloads_test.go diff --git a/docs/enhancements/datumctl-compute-domains.md b/docs/enhancements/datumctl-compute-domains.md new file mode 100644 index 00000000..a5faee81 --- /dev/null +++ b/docs/enhancements/datumctl-compute-domains.md @@ -0,0 +1,291 @@ +# `datumctl compute` — URLs and Domains + +**Status:** Draft +**Companion:** [`datumctl-compute-dx.md`](./datumctl-compute-dx.md) (the DX arc this extends), [`../scoping/alb-workload-exposure.md`](../scoping/alb-workload-exposure.md) (technical scoping) + +--- + +## Summary + +`datumctl compute deploy --port=8080` runs a container across multiple cities and gives the developer nothing to point a browser at. This proposes closing that gap by adopting the conventions every comparable platform already uses, rather than inventing a Datum-specific vocabulary for it. + +Three breaking changes, stated up front: + +1. **`--port` becomes `--http-port`.** The name declares the contract: this is an HTTP service, and it gets a URL. +2. **There is no `expose` / `unexpose` verb.** Custom hostnames live under `domains`, matching Heroku, Vercel, and Railway. +3. **The managed URL is automatic and permanent** for any workload that declares an HTTP port. It is not a resource the developer creates or deletes. + +Plus one addition that costs almost nothing and that four of five comparable CLIs ship: **`datumctl compute open`**. + +--- + +## What the industry actually does + +| Platform | How you declare "this is a web service" | Platform URL | Custom domain | Open in browser | +|---|---|---|---|---| +| **Heroku** | `web:` process type in `Procfile` | automatic | `heroku domains:add` | `heroku open` | +| **Fly.io** | `[http_service]` block in `fly.toml` | automatic | `fly certs add` | `fly open` | +| **Railway** | port detection, then Generate Domain | on request | `railway domain` | `railway open` | +| **Render** | service type: Web Service vs Private Service | automatic | dashboard | — | +| **Cloud Run** | everything is HTTP | automatic | `gcloud run domain-mappings create` | — | +| **Kubernetes** | `Service` + `Ingress`/`Gateway` | none | — | — | + +Three patterns hold across all of them, and they are the whole basis of this proposal. + +**1. The declared *role* determines exposure — never a port number.** Heroku routes the `web:` process and not `worker:`. Render makes you pick Web Service or Private Service. Fly has an `[http_service]` block whose port field is literally named `internal_port`. Not one of these platforms treats "a port exists" as "publish it." + +This is the answer to whether `--port` is intuitive: it isn't, and neither is adding `--public` next to it. `--port` says *what is listening*. Every platform above makes the developer say *what the thing is*. `--http-port` does that in one flag — you cannot type `--http-port 5432` for a Postgres and be surprised by the result, because the flag name states the contract. + +**2. The platform URL is free, automatic, and not a managed object.** Nobody runs `heroku expose`. The URL exists because the service is a web service. This kills `expose` and `unexpose` as concepts: to stop serving, you stop declaring an HTTP port, exactly as you would delete an `[http_service]` block on Fly. + +**3. "Domain" is the standard word — "expose" is not.** `heroku domains:add`, `vercel domains add`, `railway domain`. The one platform that says "expose" is Kubernetes, where `kubectl expose` defaults to a **ClusterIP** — internal. Borrowing that verb for a public-internet action would be borrowing the least familiar option *and* the one that means the opposite thing. + +Datum Compute is general-purpose — sandboxes, VMs, workers, multi-city placement — so it cannot follow Cloud Run and assume everything is HTTP. It sits closest to Fly and Render, and it should follow them: **make the role explicit, then make everything after it automatic.** + +--- + +## Product principles + +**1. Declaring an HTTP port is declaring a web service.** One flag, one meaning, no confirmation flag stacked on top of it. + +**2. The URL is free, so it is automatic.** A managed URL costs the developer nothing, which is what licenses issuing one without asking. Every platform in the table above makes the same trade. + +**3. The URL is the deliverable.** Last line of output, on its own, copy-pasteable. + +**4. Never name the machinery.** The developer sees "URL," "backends," "certificate." Never `NetworkService`, `HTTPProxy`, or a condition reason. Per the DX doc, raw platform state stays behind `describe` and `-o yaml`. + +**5. Never invent resources that don't exist.** No `endpoint/api created` for a kind nobody can `datumctl get`. Plain language in normal output; `-o yaml` shows the real objects. + +**6. One workload, one URL.** Path routing, multiple hostnames, and multi-backend fan-out are manifest territory. + +--- + +## Workflows + +### 1. Deploy and get a URL + +``` +$ datumctl compute deploy api \ + --image=ghcr.io/acme/api:1.4.2 \ + --city=DFW,IAD \ + --min=2 \ + --http-port=8080 + +Resolving workload "api" in project acme-prod... + Placement "default": cities=[DFW, IAD], min=2 + HTTP service: port 8080 → Datum-managed URL + +Apply? (Y/n): y + workload/api created +Saved workload.yaml + +Waiting for rollout. Ctrl-C to detach (rollout continues in background). + + PLACEMENT CITY UPDATED READY OLD PHASE + default DFW 2 0 0 Pending + default IAD 2 0 0 Pending + default DFW 2 2 0 Done + default IAD 2 2 0 Done + +Rollout complete in 47s. + +Publishing... + Backends 4 healthy across DFW, IAD + Edge programmed + Certificate issued + + https://a1b2c3d4.datumproxy.net +``` + +The URL objects are created alongside the workload, not after the rollout, so backends register as instances come up and the URL is ready within a second or two of the last city reaching `Done`. Creating them afterward would add a visible stall to every deploy. + +A workload with no `--http-port` deploys exactly as it does today and says so, so the developer is never left guessing: + +``` +Rollout complete in 47s. + + No HTTP port declared — this workload is not reachable from the internet. + To publish it: datumctl compute deploy api --http-port 8080 +``` + +### 2. Open it + +``` +$ datumctl compute open api +Opening https://a1b2c3d4.datumproxy.net +``` + +`--url` prints without opening, for piping. This is `heroku open` / `fly open` / `railway open`, and it is the single most-used command in that family after deploy. + +### 3. Find the URL again + +There is no `status` command in this CLI today (the DX doc proposes one; it was never built), so +the URL surfaces in the two places a developer already looks — the list view: + +``` +$ datumctl compute workloads + + NAME CITIES READY IMAGE URL + api DFW, IAD 4/4 ghcr.io/acme/api:1.4.2 https://api.example.com + worker DFW 1/1 ghcr.io/acme/worker:2.0 — +``` + +and as a field for scripting, so `| jq -r .url` works: + +``` +$ datumctl compute workloads -o json | jq -r '.[] | select(.name=="api") | .url' +$ datumctl compute open api --url # the single-workload shortcut +``` + +When a `status` command does land, the URL belongs in its header — but nothing here should wait on it. + +### 4. Diagnose a URL that isn't working + +Passing a workload to `domains` gives the per-URL detail view. This is what makes multi-city legible — the thing a developer is paying Datum for and currently cannot see: + +``` +$ datumctl compute domains api + +URL https://api.example.com + https://a1b2c3d4.datumproxy.net +Backend port 8080/tcp + +Health Degraded — 2 of 4 backends healthy + + CITY BACKENDS HEALTHY SERVING + DFW 2 2 yes + IAD 2 0 no + + IAD: no healthy backends — instances are running but not passing health checks. + Traffic is being served from DFW only. + + Next steps: + Check instances: datumctl compute instances --workload=api --city=IAD + Check logs: datumctl compute logs api --city=IAD +``` + +Per the DX doc's rule, the CLI renders whatever reason and message the server emits rather than branching on reason strings, so new platform conditions surface without a CLI release. + +### 5. Add a custom domain *(phase 2)* + +Modeled directly on `heroku domains:add` and `fly certs add`: print the records, then poll. + +``` +$ datumctl compute domains add api api.example.com + +Verifying example.com... + + Add these DNS records: + + TYPE NAME VALUE + TXT _datum.example.com datum-verify=8f3a91c2b7 + CNAME api.example.com a1b2c3d4.datumproxy.net + +Waiting for DNS. Ctrl-C to detach — run 'datumctl compute domains' to check. + + Domain verified ✓ + DNS record programmed ✓ + Certificate issued ✓ + + https://api.example.com +``` + +``` +$ datumctl compute domains + + WORKLOAD DOMAIN STATUS CERTIFICATE + api api.example.com active valid + api a1b2c3d4.datumproxy.net active valid (Datum-managed) + web www.acme.io pending waiting for DNS +``` + +``` +$ datumctl compute domains remove api api.example.com +``` + +The managed URL always appears in this list and cannot be removed. It is a stable fallback and a useful health-check target. + +### 6. Stop serving + +There is no `unexpose`, for the same reason Heroku has no such command. Remove the HTTP port: + +``` +$ datumctl compute deploy api --image=... --city=DFW,IAD --no-http + + HTTP service: removed — https://a1b2c3d4.datumproxy.net will stop responding +Apply? (Y/n): +``` + +And `destroy` states the consequence rather than silently dropping it: + +``` +$ datumctl compute destroy api + +Workload: api +Placements: 1 Cities: DFW, IAD +Min replicas: 2 +URLs: https://api.example.com, https://a1b2c3d4.datumproxy.net + +This will delete the workload, all its instances, and its URLs. Continue? (y/N): +``` + +--- + +## Command reference + +New: + +``` +datumctl compute open Open the workload's URL in a browser (--url to print) +datumctl compute domains List domains across the project +datumctl compute domains Per-URL detail: hostnames, certificates, per-city backends +datumctl compute domains add Attach a custom hostname (phase 2) +datumctl compute domains remove Detach a custom hostname (phase 2) +``` + +Changed: + +``` +datumctl compute deploy --port → --http-port (breaking); --no-http removes it +datumctl compute workloads gains a URL column and a `url` field in json/yaml output +datumctl compute destroy lists URLs in its summary and deletes them +``` + +### On breaking `--port` + +`--port` is removed, not silently aliased. Silently mapping it to `--http-port` would publish every existing CI workload on the next plugin upgrade — precisely the surprise this design exists to avoid. For one release it remains registered and **errors**: + +``` +Error: --port has been replaced by --http-port, which publishes the workload on a +public HTTPS URL. Use --http-port 8080 to publish, or --no-http to keep it internal. +``` + +That is a loud, one-line migration, and the plugin is activation-gated and alpha — the audience for this break is small and reachable. + +--- + +## What this deliberately does not do + +- **No path or header routing.** One backend at `/`. Anyone needing `/api` and `/admin` split across workloads writes an `HTTPProxy` manifest. +- **No TCP or UDP exposure.** HTTP/HTTPS only, matching the platform's proxy. +- **No plaintext HTTP.** Always `https://`. No `--insecure`. +- **No backend TLS.** The edge reaches instances over plaintext inside the network. A container terminating TLS itself will not work, and the CLI must say so clearly rather than producing a broken URL. +- **No domain deletion on `destroy`.** A verified domain is a project asset that outlives any one workload. + +--- + +## Phasing + +**Phase 1 — the URL.** `--http-port`, automatic managed URL, `open`, URL in `status` / `workloads` / `destroy`, `--url` diagnostics, the `--port` migration error. This is the entire product promise. + +**Phase 2 — domains.** `domains` list/add/remove, verification UX, certificate and DNS progress. + +**Phase 3 — driven by usage.** Multiple ports, health-check configuration, request metrics on the URL view. + +--- + +## Open questions + +1. **What should the managed hostname look like?** `.datumproxy.net` is stable and collision-free but unmemorable and awkward to share. Heroku moved off `.herokuapp.com` to random-word names for exactly the squatting reason; Fly uses `.fly.dev` and accepts the collision namespace. This is a platform decision the CLI inherits, but it shapes the first-run experience more than anything else in this document, and PR 411 is still a draft — worth pushing on now. + +2. **How does `deploy -f workload.yaml` declare an HTTP service?** The manifest path has no `--http-port` equivalent. Fly's answer is that the manifest *is* the declaration (`[http_service]`), which suggests the right answer here is a field on the workload spec rather than CLI-only sugar. That is an API conversation with the compute team, not a CLI one, and until it happens the flag and manifest paths do not converge the way the DX doc claims they do. diff --git a/go.mod b/go.mod index c95bfd7c..3e1095c7 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,14 @@ require ( github.com/onsi/gomega v1.42.1 github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.11.1 - // Pinned to network-services-operator main: the latest tag (v0.26.0) - // predates the Prepared condition this gate reads. Re-pin to a tagged - // release once one carries it. - go.datum.net/network-services-operator v0.26.1-0.20260821014231-aceb24b1b569 + // UNMERGED: pinned to the head of network-services-operator#411 + // (branch proto/network-service), which adds the NetworkService API and the + // networkService HTTPProxy backend that `datumctl compute --http-port` needs. + // That PR is a draft and its branch may be force-pushed or deleted, which + // would break `go mod download` here. Re-pin to main as soon as it merges. + // The pre-411 pin was chosen for the Prepared condition, which this commit + // also carries. + go.datum.net/network-services-operator v0.26.5-0.20260909170421-bcd1965a821c // Pinned by pseudo-version to the commit deployed to staging, which is the // same one network-services-operator pins. The module publishes no tag yet. go.miloapis.com/locations v0.0.0-20260825185141-507ac2cbd48c @@ -116,7 +120,7 @@ require ( github.com/go-openapi/jsonreference v0.21.6 // indirect github.com/go-openapi/swag v0.25.4 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/google/cel-go v0.26.0 // indirect + github.com/google/cel-go v0.27.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/go.sum b/go.sum index d63b9ff2..799a9c28 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/cel-go v0.27.0 h1:e7ih85+4qVrBuqQWTW4FKSqZYokVuc3HnhH5keboFTo= +github.com/google/cel-go v0.27.0/go.mod h1:tTJ11FWqnhw5KKpnWpvW9CJC3Y9GK4EIS0WXnBbebzw= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -350,6 +352,8 @@ go.datum.net/datumctl v0.17.1-0.20260710003126-296c2fcbbd67 h1:Mhgt688CeTh2hX7ql go.datum.net/datumctl v0.17.1-0.20260710003126-296c2fcbbd67/go.mod h1:6skEjcE7aT8VPf/HVamA/BB6Dc9IISA6c/DdYKhqWNc= go.datum.net/network-services-operator v0.26.1-0.20260821014231-aceb24b1b569 h1:14vajo15fGGEAmdystQEE261rFH4AqER9s1Vg7POUKo= go.datum.net/network-services-operator v0.26.1-0.20260821014231-aceb24b1b569/go.mod h1:A7JNOuc+e6j/KkUVCcZ7Z2odvf6JFMQlX0Zo1Awj2TY= +go.datum.net/network-services-operator v0.26.5-0.20260909170421-bcd1965a821c h1:nHFLUbR5xi2hvYuV7DtH00SPuve/uyL95Fi0ASSjHY8= +go.datum.net/network-services-operator v0.26.5-0.20260909170421-bcd1965a821c/go.mod h1:9nuuBWdrkdnIBMaWJsWM3j4CcbKJIF1GDuCmAjdpIHo= go.miloapis.com/locations v0.0.0-20260825185141-507ac2cbd48c h1:+BQirT3wYCgv7H2lEZtAH+dpMiWu9j/Wa+c8s4jJUIA= go.miloapis.com/locations v0.0.0-20260825185141-507ac2cbd48c/go.mod h1:gzfAfHhSMwl/N68k/uSNYXOKK3IOBJCdXYaBgCE3gdE= go.miloapis.com/milo v0.32.0 h1:TkNIQu/37d+SEquLJ5+GmdisSl+K2RT7eEC4idg6RIs= diff --git a/internal/cmd/compute/deploy/deploy.go b/internal/cmd/compute/deploy/deploy.go index 2c3925d4..d2bfb644 100644 --- a/internal/cmd/compute/deploy/deploy.go +++ b/internal/cmd/compute/deploy/deploy.go @@ -4,7 +4,9 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" + "io" "os" "os/signal" "strings" @@ -20,6 +22,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/compute/internal/cmd/compute/build" + "go.datum.net/compute/internal/cmd/compute/url" "go.datum.net/compute/internal/cmd/compute/util" "go.datum.net/compute/internal/cmd/compute/watch" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" @@ -27,6 +30,24 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + // httpPortName is the name given to the container port --http-port + // declares, and the name the URL's backend reference points at. + httpPortName = "http" + + // planLabelWidth is the label column of the plan summary printed before + // the Apply prompt, so every line in it starts its value at the same + // column. + planLabelWidth = 20 +) + +// errPortRenamed is the one-release migration for --port. It is an error and +// not an alias on purpose: silently mapping it to --http-port would publish +// every existing workload on the internet at the next plugin upgrade. +var errPortRenamed = errors.New( + "--port has been replaced by --http-port, which publishes the workload on a public HTTPS URL. " + + "Use --http-port 8080 to publish, or --no-http to keep it internal") + type options struct { image string build string @@ -35,12 +56,22 @@ type options struct { locationSelector string cities []string min int32 + httpPort int32 + noHTTP bool port int32 file string yes bool } +// Command returns the deploy command. func Command() *cobra.Command { + cmd, _ := command() + return cmd +} + +// command builds the deploy command and hands back the options it writes into, +// so flag validation can be exercised without a control plane. +func command() (*cobra.Command, *options) { opts := &options{} cmd := &cobra.Command{ @@ -57,10 +88,21 @@ Dockerfile discovery, no build-arg/target overrides — use 'datumctl compute bu directly if you need those) and pushes to --image, which the deployed workload then pins by digest rather than the tag you gave it. It also analyzes and auto-fixes common compatibility issues, rewriting the Dockerfile in place when -a fix is applied — same as 'datumctl compute build --fix'.`, +a fix is applied — same as 'datumctl compute build --fix'. + +Use --http-port to declare that the workload is an HTTP service. Declaring one +publishes the workload on a Datum-managed HTTPS URL, printed as the last line +of a successful deploy. Omitting --http-port on an existing workload leaves its +HTTP service as it is; --no-http removes it and stops serving.`, Args: cobra.MaximumNArgs(1), Example: ` # Deploy with flags - datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --location=us-east-1,eu-west-1 --min=2 --port=8080 + datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --location=us-east-1,eu-west-1 --min=2 --http-port=8080 + + # Deploy an internal workload (no URL) + datumctl compute deploy worker --image=ghcr.io/acme/worker:2.0 --location=us-east-1 + + # Stop serving: remove the HTTP service and its URL + datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --location=us-east-1,eu-west-1 --no-http # Build and deploy in one step (builds ., pushes to --image, deploys that digest) datumctl compute deploy api --build --image=ghcr.io/acme/api:1.4.2 --location=us-east-1,eu-west-1 @@ -93,17 +135,62 @@ a fix is applied — same as 'datumctl compute build --fix'.`, cmd.Flags().StringVar(&opts.locationSelector, "location-selector", "", "Select every location whose topology matches a label selector (e.g. 'topology.datum.net/city-code=DFW' or 'topology.datum.net/region in (us-east-1,eu-west-1)')") cmd.Flags().StringSliceVar(&opts.cities, "city", nil, "Deploy to every location in these cities (e.g. DFW,IAD); shorthand for a --location-selector on topology.datum.net/city-code") cmd.Flags().Int32Var(&opts.min, "min", 1, "Minimum number of instances per location") - cmd.Flags().Int32Var(&opts.port, "port", 0, "Port to expose on the workload (optional)") + cmd.Flags().Int32Var(&opts.httpPort, "http-port", 0, "Port the container serves HTTP on; publishes the workload on a Datum-managed HTTPS URL") + cmd.Flags().BoolVar(&opts.noHTTP, "no-http", false, "Remove the workload's HTTP service, and with it its URL") cmd.Flags().StringVarP(&opts.file, "file", "f", "", "Path to a workload manifest file") cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false, "Skip confirmation prompts") _ = cmd.RegisterFlagCompletionFunc("location", util.CompletePlacementLocations) _ = cmd.RegisterFlagCompletionFunc("location-selector", util.CompleteLocationSelector) _ = cmd.RegisterFlagCompletionFunc("city", util.CompleteCityCodes) - return cmd + // --port stays registered for one release so that using it produces the + // migration error rather than "unknown flag". It is hidden rather than + // deprecated: cobra's deprecation only warns and proceeds, and printing a + // warning above the error that follows says the same thing twice. + cmd.Flags().Int32Var(&opts.port, "port", 0, "Removed: use --http-port") + _ = cmd.Flags().MarkHidden("port") + + return cmd, opts +} + +// validateFlags rejects flag combinations before the command builds a client +// or creates anything, so an upgrade that trips the --port break costs a +// message and not a workload. +func validateFlags(cmd *cobra.Command, opts *options) error { + if cmd.Flags().Changed("port") { + return errPortRenamed + } + + httpPortSet := cmd.Flags().Changed("http-port") + + if httpPortSet && opts.noHTTP { + return fmt.Errorf("--http-port and --no-http cannot be combined — pass --http-port to publish the workload, or --no-http to stop serving it") + } + + if opts.file != "" { + switch { + case httpPortSet: + // TODO: a manifest has no way to declare an HTTP service yet. + // Resolving that is an API conversation (a field on the workload + // spec), not CLI sugar layered on top of -f. + return fmt.Errorf("--http-port cannot be combined with -f: a manifest declares its own ports, and declaring an HTTP service in a manifest is not supported yet") + case opts.noHTTP: + return fmt.Errorf("--no-http cannot be combined with -f: remove the URL with 'datumctl compute destroy', or deploy with flags") + } + } + + if httpPortSet && (opts.httpPort < 1 || opts.httpPort > 65535) { + return fmt.Errorf("--http-port must be between 1 and 65535, got %d", opts.httpPort) + } + + return nil } func runDeploy(cmd *cobra.Command, args []string, opts *options) error { + if err := validateFlags(cmd, opts); err != nil { + return err + } + // Determine path. if opts.file != "" { if opts.build != "" { @@ -209,16 +296,34 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err } } + // Resolve the HTTP service this deploy declares: + // + // --http-port N declares (or changes) it + // --no-http removes it, and the URL with it + // neither keeps what the workload already declares + // + // Carrying the existing port forward matters: without it, a routine image + // bump would drop the port from the spec and take a live URL down without + // anyone saying so. --no-http is the only way to stop serving. + httpPort := opts.httpPort + if httpPort == 0 && !opts.noHTTP { + httpPort = declaredHTTPPort(&workload) + } + // Build spec. tcp := corev1.ProtocolTCP container := computev1alpha.SandboxContainer{ Name: "app", Image: opts.image, } - if opts.port > 0 { - container.Ports = []computev1alpha.NamedPort{ - {Name: "http", Port: opts.port, Protocol: &tcp}, + portName := "" + if httpPort > 0 { + httpNamedPort := computev1alpha.NamedPort{Name: httpPortName, Port: httpPort, Protocol: &tcp} + portName, err = url.PortName(httpNamedPort) + if err != nil { + return err } + container.Ports = []computev1alpha.NamedPort{httpNamedPort} } locationRefs := make([]locationsv1alpha1.LocationReference, 0, len(locations)) @@ -258,7 +363,10 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err Placements: []computev1alpha.WorkloadPlacement{placement}, } - fmt.Fprintf(out, " Placement \"default\": %s, min=%d\n", describePlacementLocations(placement), opts.min) + fmt.Fprintln(out, planLine(`Placement "default"`, + fmt.Sprintf("%s, min=%d", describePlacementLocations(placement), opts.min))) + + removedURL := planHTTPService(ctx, out, c, workloadName, httpPort, opts, creating) // Prompt unless --yes or non-interactive. if !opts.yes && term.IsTerminal(int(os.Stdin.Fd())) { @@ -287,6 +395,21 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err fmt.Fprintf(out, " workload/%s updated\n", workloadName) } + if opts.noHTTP { + if err := removeHTTPService(ctx, out, c, workloadName, removedURL, creating); err != nil { + return err + } + } + + // The URL goes in alongside the workload, not after the rollout: backends + // register as instances come up, so the URL answers moments after the last + // city is Done rather than starting from scratch once it is. + // + // A failure is carried to publish rather than returned here. The workload + // is applied and rolling out, and a user is owed that table before being + // told the URL did not go up. + publishErr := declareURL(ctx, c, &workload, portName, httpPort) + // Save workload.yaml. if err := saveWorkloadYAML(workloadName, &workload); err != nil { fmt.Fprintf(out, " warning: could not save workload.yaml: %v\n", err) @@ -298,7 +421,198 @@ func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) err watchCtx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) defer cancel() - return watch.Rollout(watchCtx, c, out, project, workload.UID) + if err := watch.Rollout(watchCtx, c, out, project, workload.UID); err != nil { + return err + } + + return publish(watchCtx, out, c, &workload, httpPort, opts, publishErr) +} + +// planHTTPService prints the HTTP line of the plan summary and returns the URL +// that --no-http is about to take down, if there is one. +// +// The HTTP service is part of the plan, not a surprise after the fact: what +// gets published — or what stops answering — is stated before the prompt that +// creates it. +func planHTTPService(ctx context.Context, out io.Writer, c client.Client, workloadName string, httpPort int32, opts *options, creating bool) string { + if httpPort > 0 { + fmt.Fprintln(out, planLine("HTTP service", fmt.Sprintf("port %d → Datum-managed URL", httpPort))) + // Said here, once, because a container that terminates TLS itself + // answers nothing and the only symptom is a URL that does not work. + fmt.Fprintln(out, planNote("Datum terminates TLS; serve plain HTTP on this port.")) + return "" + } + if !opts.noHTTP || creating { + return "" + } + + // A lookup failure here is not worth failing a deploy over: the line just + // loses the hostname it would have named, and Unpublish reports any real + // problem with the control plane a moment later. + removedURL := "" + if info, err := url.ForWorkload(ctx, c, workloadName); err == nil && info != nil { + removedURL = info.URL + } + + if removedURL == "" { + fmt.Fprintln(out, planLine("HTTP service", "removed")) + return "" + } + fmt.Fprintln(out, planLine("HTTP service", fmt.Sprintf("removed — %s will stop responding", removedURL))) + return removedURL +} + +// removeHTTPService takes the URL down. It runs as soon as the workload stops +// declaring the port rather than at the end of the rollout: the workload and +// what answers for it have to agree. +func removeHTTPService(ctx context.Context, out io.Writer, c client.Client, workloadName, removedURL string, creating bool) error { + if err := url.Unpublish(ctx, c, workloadName); err != nil { + return err + } + switch { + case removedURL != "": + fmt.Fprintf(out, " HTTP service removed — %s no longer responds\n", removedURL) + case !creating: + _, _ = fmt.Fprintln(out, " HTTP service removed") + } + return nil +} + +// declareURL writes the objects that put the workload on its URL. It runs +// alongside the workload write, before the rollout: backends then register as +// instances come up, and the URL is ready within a second or two of the last +// city reaching Done. Declaring them after the rollout would add a visible +// stall to every deploy. +// +// It prints nothing. Nothing has happened yet that a user needs to read, and +// the rollout table comes next; publishing reports itself once the rollout is +// over and there is progress to show. +func declareURL(ctx context.Context, c client.Client, w *computev1alpha.Workload, portName string, port int32) error { + if port <= 0 { + return nil + } + + hostnames, err := existingHostnames(ctx, c, w.Name) + if err != nil { + return err + } + return url.Declare(ctx, c, w, portName, port, hostnames) +} + +// notReachable states the dead end this whole feature exists to close: a +// workload with no HTTP port is not on the internet, and no developer should +// have to work that out for themselves. +func notReachable(out io.Writer, workloadName string) { + fmt.Fprintf(out, "\n No HTTP port declared — this workload is not reachable from the internet.\n") + fmt.Fprintf(out, " To publish it: datumctl compute deploy %s --http-port 8080\n", workloadName) +} + +// publish waits for the URL declared before the rollout and prints it as the +// last line of the deploy, or explains why there is no URL to print. +// +// declareErr is whatever declareURL reported. It is carried this far rather +// than failing the deploy on the spot so that a user still gets the rollout +// table for a workload that is, after all, being deployed. +// +// It runs after the rollout, so the workload is already up: a failure here is +// a failure to publish, never a failure to deploy, and it says so before the +// error is returned. A user whose workload is running must not read a bare +// "Error:" as "the deploy failed". +func publish(ctx context.Context, out io.Writer, c client.Client, w *computev1alpha.Workload, port int32, opts *options, declareErr error) error { + if port <= 0 { + // --no-http was just told, line by line, that the URL is gone. Telling + // the same user to publish is answering a question nobody asked. + if !opts.noHTTP { + notReachable(out, w.Name) + } + return nil + } + + _, _ = fmt.Fprintln(out, "\nPublishing...") + + // The objects went in before the rollout, so there is nothing left to do + // here but watch — including for a user who detached, whose URL is already + // declared and coming up without them. + var info *url.Info + err := declareErr + if err == nil { + info, err = url.Wait(ctx, out, c, w.Name) + } + if err != nil { + fmt.Fprintf(out, "\n The rollout succeeded — the workload is deployed and running.\n") + fmt.Fprintf(out, " Only publishing its URL failed. Retry with:\n") + fmt.Fprintf(out, " datumctl compute deploy %s --image %s --http-port %d\n", w.Name, opts.image, port) + return fmt.Errorf("publishing URL for workload %q: %w", w.Name, err) + } + + // A nil Info is a detach, not a failure: url.Wait has already said how to + // pick the URL up again. + if info == nil || info.URL == "" { + return nil + } + + fmt.Fprintf(out, "\n %s\n", info.URL) + return nil +} + +// existingHostnames returns the custom hostnames already attached to the +// workload's URL, so republishing carries them forward. +// +// Publishing rewrites the proxy spec wholesale. Without this, every redeploy +// of a workload would silently detach the hostnames 'datumctl compute domains +// add' put there, and the custom domain would stop answering — an unpublish +// the user never asked for. Detaching a hostname is 'domains remove' and +// nothing else. +// +// It fails closed. A workload that has never been published has no hostnames +// and that is a nil with no error, but a control plane that cannot be read is +// an error the caller must stop on: the two calls use different verbs on the +// same object — a List here, a Get in the apply — so a control plane that +// refuses one and answers the other would otherwise rewrite spec.Hostnames to +// nothing and detach every custom domain while the deploy reported success. +func existingHostnames(ctx context.Context, c client.Client, workloadName string) ([]string, error) { + info, err := url.ForWorkload(ctx, c, workloadName) + if err != nil { + return nil, fmt.Errorf("reading the domains attached to %q: %w", workloadName, err) + } + if info == nil { + return nil, nil + } + return info.CustomHostnames, nil +} + +// planLine renders one line of the plan summary printed before the Apply +// prompt, with every value starting in the same column. +func planLine(label, value string) string { + return fmt.Sprintf(" %-*s %s", planLabelWidth, label+":", value) +} + +// planNote renders a continuation of the plan line above it, aligned under +// that line's value rather than carrying a label of its own. +func planNote(text string) string { + return fmt.Sprintf(" %-*s %s", planLabelWidth, "", text) +} + +// declaredHTTPPort returns the HTTP port a workload already declares, or 0. +// The port named "http" wins; failing that, the first declared port is the one +// the URL was built on, since that is what a flag-driven deploy writes. +func declaredHTTPPort(w *computev1alpha.Workload) int32 { + sandbox := w.Spec.Template.Spec.Runtime.Sandbox + if sandbox == nil { + return 0 + } + first := int32(0) + for _, container := range sandbox.Containers { + for _, p := range container.Ports { + if p.Name == httpPortName { + return p.Port + } + if first == 0 { + first = p.Port + } + } + } + return first } // deployFromFile implements Path C: deploy from a manifest file. @@ -387,7 +701,31 @@ func deployFromFile(cmd *cobra.Command, opts *options) error { watchCtx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) defer cancel() - return watch.Rollout(watchCtx, c, out, project, workload.UID) + + if err := watch.Rollout(watchCtx, c, out, project, workload.UID); err != nil { + return err + } + + reportManifestReachability(out, &workload) + return nil +} + +// reportManifestReachability closes the dead end for the manifest path. +// +// TODO: the manifest path publishes nothing. A workload manifest has no way to +// declare "this is an HTTP service" — the flag path's --http-port has no +// equivalent field — and inferring one from a container port would publish +// workloads whose authors never asked for a URL. Resolving it means a field on +// the workload spec, which is an API decision, not a CLI one. +// +// The note, though, is not publishing. A workload nothing can reach is the +// same dead end however it was deployed, and a developer who reads it after a +// flag deploy but not after a -f deploy is a developer who concludes the URL +// is somewhere they have not looked. +func reportManifestReachability(out io.Writer, w *computev1alpha.Workload) { + if declaredHTTPPort(w) == 0 { + notReachable(out, w.Name) + } } // saveWorkloadYAML marshals the workload and writes it to workload.yaml in the diff --git a/internal/cmd/compute/deploy/deploy_test.go b/internal/cmd/compute/deploy/deploy_test.go new file mode 100644 index 00000000..90f69a16 --- /dev/null +++ b/internal/cmd/compute/deploy/deploy_test.go @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "bytes" + "context" + "errors" + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + testWorkload = "api" + testCanonical = "a1b2c3d4.datumproxy.net" + testImage = "ghcr.io/acme/api:1" + + imageFlag = "--image=" + testImage + httpPortFlag = "--http-port=8080" + noHTTPFlag = "--no-http" + + wantPortRange = "between 1 and 65535" +) + +// TestValidateFlags is the migration matrix: which flag combinations the +// command refuses, and — the point of the whole break — that --port is one of +// them rather than a silent alias for --http-port. +func TestValidateFlags(t *testing.T) { + tests := []struct { + name string + args []string + wantErr string + }{ + { + name: "no http flags is the old behaviour", + args: []string{testWorkload, imageFlag}, + }, + { + name: "http-port publishes", + args: []string{testWorkload, imageFlag, httpPortFlag}, + }, + { + name: "no-http alone", + args: []string{testWorkload, imageFlag, noHTTPFlag}, + }, + { + name: "port is removed, not aliased", + args: []string{testWorkload, imageFlag, "--port=8080"}, + wantErr: "--port has been replaced by --http-port", + }, + { + name: "port zero still errors", + args: []string{testWorkload, imageFlag, "--port=0"}, + wantErr: "--port has been replaced by --http-port", + }, + { + name: "http-port and no-http conflict", + args: []string{testWorkload, imageFlag, httpPortFlag, noHTTPFlag}, + wantErr: "cannot be combined", + }, + { + name: "http-port with a manifest", + args: []string{"-f", "workload.yaml", httpPortFlag}, + wantErr: "--http-port cannot be combined with -f", + }, + { + name: "no-http with a manifest", + args: []string{"-f", "workload.yaml", noHTTPFlag}, + wantErr: "--no-http cannot be combined with -f", + }, + { + name: "http-port below range", + args: []string{testWorkload, imageFlag, "--http-port=0"}, + wantErr: wantPortRange, + }, + { + name: "http-port above range", + args: []string{testWorkload, imageFlag, "--http-port=70000"}, + wantErr: wantPortRange, + }, + { + name: "http-port negative", + args: []string{testWorkload, imageFlag, "--http-port=-1"}, + wantErr: wantPortRange, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cmd, opts := command() + if err := cmd.Flags().Parse(tc.args); err != nil { + t.Fatalf("parsing flags: %v", err) + } + + err := validateFlags(cmd, opts) + switch { + case tc.wantErr == "" && err != nil: + t.Fatalf("want no error, got %v", err) + case tc.wantErr != "" && err == nil: + t.Fatalf("want error containing %q, got nil", tc.wantErr) + case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): + t.Fatalf("want error containing %q, got %v", tc.wantErr, err) + } + }) + } +} + +// TestPortErrorReachesTheUser runs the command the way a user does. Validation +// that only exists in a helper is validation an upgrade path can skip. +func TestPortErrorReachesTheUser(t *testing.T) { + cmd := Command() + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + cmd.SetArgs([]string{testWorkload, imageFlag, "--port=8080"}) + + err := cmd.Execute() + if err == nil { + t.Fatal("deploying with --port must fail, not publish the workload") + } + if !errors.Is(err, errPortRenamed) { + t.Fatalf("want the migration error, got %v", err) + } + for _, want := range []string{"--http-port 8080", noHTTPFlag} { + if !strings.Contains(err.Error(), want) { + t.Errorf("migration error does not mention %q: %v", want, err.Error()) + } + } +} + +// TestPortFlagStaysRegistered guards the difference between "errors" and +// "unknown flag": the migration message only lands if the flag still parses. +func TestPortFlagStaysRegistered(t *testing.T) { + cmd := Command() + + port := cmd.Flags().Lookup("port") + if port == nil { + t.Fatal("--port must stay registered for one release so it can error") + } + if !port.Hidden { + t.Error("--port must not be advertised in help") + } + if cmd.Flags().Lookup("http-port") == nil { + t.Error("--http-port must be registered") + } + if cmd.Flags().Lookup("no-http") == nil { + t.Error("--no-http must be registered") + } + if !strings.Contains(cmd.Example, httpPortFlag) { + t.Error("the example block must show --http-port") + } + if strings.Contains(cmd.Example, "--port=8080") { + t.Error("the example block must not show --port") + } +} + +// TestDeclaredHTTPPort covers the carry-forward rule: a deploy that does not +// mention a port must not silently take a live URL down. +func TestDeclaredHTTPPort(t *testing.T) { + tcp := corev1.ProtocolTCP + tests := []struct { + name string + workload computev1alpha.Workload + want int32 + }{ + { + name: "no runtime", + workload: computev1alpha.Workload{}, + }, + { + name: "no ports", + workload: workloadWithPorts(), + }, + { + name: "the http port", + workload: workloadWithPorts(computev1alpha.NamedPort{Name: "http", Port: 8080, Protocol: &tcp}), + want: 8080, + }, + { + name: "http wins over an earlier port", + workload: workloadWithPorts( + computev1alpha.NamedPort{Name: "metrics", Port: 9090}, + computev1alpha.NamedPort{Name: "http", Port: 8080}, + ), + want: 8080, + }, + { + name: "falls back to the first port", + workload: workloadWithPorts(computev1alpha.NamedPort{Name: "web", Port: 3000}), + want: 3000, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := declaredHTTPPort(&tc.workload); got != tc.want { + t.Fatalf("declaredHTTPPort = %d, want %d", got, tc.want) + } + }) + } +} + +// TestPlanLine pins the plan summary's alignment: the HTTP service line has to +// line up under the placement line the developer reads it with. +func TestPlanLine(t *testing.T) { + placement := planLine(`Placement "default"`, "cities=[DFW, IAD], min=2") + http := planLine("HTTP service", "port 8080 → Datum-managed URL") + + if want := ` Placement "default": cities=[DFW, IAD], min=2`; placement != want { + t.Errorf("placement line = %q, want %q", placement, want) + } + if want := " HTTP service: port 8080 → Datum-managed URL"; http != want { + t.Errorf("http line = %q, want %q", http, want) + } + if strings.Index(placement, "cities") != strings.Index(http, "port 8080") { + t.Errorf("plan values do not start in the same column:\n%s\n%s", placement, http) + } +} + +// TestPublishWithoutHTTPPortSaysSo covers the dead end this feature exists to +// close: a workload with no HTTP port must never leave the developer guessing +// why there is nothing to open. +func TestPublishWithoutHTTPPortSaysSo(t *testing.T) { + var out bytes.Buffer + + // A nil client is deliberate: with no port there is nothing to publish, so + // nothing may be read or written. + if err := publish(context.Background(), &out, nil, workload(), 0, &options{}, nil); err != nil { + t.Fatalf("publish without a port must not fail: %v", err) + } + + got := out.String() + for _, want := range []string{ + "No HTTP port declared — this workload is not reachable from the internet.", + "To publish it: datumctl compute deploy api --http-port 8080", + } { + if !strings.Contains(got, want) { + t.Errorf("output missing %q:\n%s", want, got) + } + } + if strings.Contains(got, "Publishing") { + t.Errorf("nothing was published, so nothing should say so:\n%s", got) + } +} + +// TestPublishPrintsTheURLLast covers the product promise: the URL is the +// deliverable, on its own line, at the end. +func TestPublishPrintsTheURLLast(t *testing.T) { + w := workload() + c := newFakeClient(t, liveProxy(w), url.BuildNetworkService(w, "http", 8080)) + + var out bytes.Buffer + if err := publish(context.Background(), &out, c, w, 8080, &options{}, nil); err != nil { + t.Fatalf("publish: %v", err) + } + + got := out.String() + if !strings.Contains(got, "Publishing...") { + t.Errorf("output missing the publishing heading:\n%s", got) + } + + lines := strings.Split(strings.TrimRight(got, "\n"), "\n") + last := lines[len(lines)-1] + if want := " https://" + testCanonical; last != want { + t.Errorf("last line = %q, want %q\nfull output:\n%s", last, want, got) + } +} + +// TestPublishFailureStillReportsTheRollout is the rule that a healthy workload +// never reads as a failed deploy: the URL is declared before the rollout, so a +// failure to declare it is carried past the rollout table and reported as what +// it is — a failure to publish something that did deploy. +func TestPublishFailureStillReportsTheRollout(t *testing.T) { + boom := errors.New("connection refused") + c := interceptor.NewClient(newFakeClient(t), interceptor.Funcs{ + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + return boom + }, + }) + + declareErr := declareURL(context.Background(), c, workload(), "http", 8080) + if declareErr == nil { + t.Fatal("declaring a URL against a control plane that refuses writes must fail") + } + + var out bytes.Buffer + err := publish(context.Background(), &out, c, workload(), 8080, + &options{image: testImage}, declareErr) + + if err == nil { + t.Fatal("a failure to publish must be reported as an error") + } + if !errors.Is(err, boom) { + t.Errorf("the underlying failure must not be swallowed: %v", err) + } + + got := out.String() + for _, want := range []string{ + "The rollout succeeded — the workload is deployed and running.", + "Only publishing its URL failed.", + "datumctl compute deploy api --image " + testImage + " --http-port 8080", + } { + if !strings.Contains(got, want) { + t.Errorf("output missing %q:\n%s", want, got) + } + } +} + +// TestDeclareURLWritesTheObjectsBeforeTheRollout: declaring an HTTP port +// declares a URL, and the objects go in while the workload does — silently, +// because the rollout table is the next thing the user reads. +func TestDeclareURLWritesTheObjectsBeforeTheRollout(t *testing.T) { + c := newFakeClient(t) + + if err := declareURL(context.Background(), c, workload(), "http", 8080); err != nil { + t.Fatalf("declareURL: %v", err) + } + + key := types.NamespacedName{Namespace: util.ResourceNamespace, Name: url.ResourceName(testWorkload)} + if err := c.Get(context.Background(), key, &networkingv1alpha.NetworkService{}); err != nil { + t.Errorf("backends were not declared: %v", err) + } + if err := c.Get(context.Background(), key, &networkingv1alpha.HTTPProxy{}); err != nil { + t.Errorf("the URL was not declared: %v", err) + } +} + +// A workload with no HTTP port declares no URL, so nothing may be written for +// it — least of all a proxy publishing a workload the user kept internal. +func TestDeclareURLWritesNothingWithoutAPort(t *testing.T) { + // A nil client is the assertion: any write at all would panic. + if err := declareURL(context.Background(), nil, workload(), "", 0); err != nil { + t.Fatalf("declaring nothing must not fail: %v", err) + } +} + +// TestPublishAfterTheRolloutOnlyWaits is the ordering the spec is explicit +// about: the URL objects are created alongside the workload so that backends +// register as instances come up, and the URL answers within a second or two of +// the last city reaching Done. Publishing after the rollout therefore has +// nothing left to write — a write here is proof the objects were not declared +// earlier, and proof of the stall the spec says never to add. +func TestPublishAfterTheRolloutOnlyWaits(t *testing.T) { + w := workload() + + // The declared backends are deliberately out of date, so an apply running + // at this point would have to update them and be caught doing it. + declared := newFakeClient(t, liveProxy(w), url.BuildNetworkService(w, "http", 9090)) + c := interceptor.NewClient(declared, interceptor.Funcs{ + Create: func(_ context.Context, _ client.WithWatch, obj client.Object, _ ...client.CreateOption) error { + t.Errorf("publishing wrote %T after the rollout — the URL objects belong alongside the workload", obj) + return nil + }, + Update: func(_ context.Context, _ client.WithWatch, obj client.Object, _ ...client.UpdateOption) error { + t.Errorf("publishing wrote %T after the rollout — the URL objects belong alongside the workload", obj) + return nil + }, + }) + + var out bytes.Buffer + if err := publish(context.Background(), &out, c, w, 8080, &options{}, nil); err != nil { + t.Fatalf("publish: %v", err) + } + if !strings.Contains(out.String(), "https://"+testCanonical) { + t.Errorf("publishing must still wait for and print the URL:\n%s", out.String()) + } +} + +// Ctrl-C during the rollout detaches. The URL is already declared, so +// publishing has nothing to write and only stops watching — and a detach is +// never an error. +func TestPublishDetachedIsNotAnError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + var out bytes.Buffer + if err := publish(ctx, &out, newFakeClient(t), workload(), 8080, &options{}, nil); err != nil { + t.Fatalf("detaching is never an error: %v", err) + } + if strings.Contains(out.String(), "https://") { + t.Errorf("no URL is known yet, so none may be printed:\n%s", out.String()) + } + if !strings.Contains(out.String(), "Detached.") { + t.Errorf("a detach must say publishing carries on:\n%s", out.String()) + } +} + +// TestPublishKeepsCustomHostnames is the cross-command contract between +// `domains add` and `deploy`: publishing rewrites the proxy spec, so a +// redeploy must carry forward the hostnames the user attached. Dropping them +// would unpublish a live custom domain on the next image bump, which only +// `domains remove` is allowed to do. +func TestPublishKeepsCustomHostnames(t *testing.T) { + w := workload() + existing := liveProxy(w) + existing.Spec.Hostnames = []gatewayv1.Hostname{"api.example.com"} + c := newFakeClient(t, existing, url.BuildNetworkService(w, "http", 8080)) + + if err := declareURL(context.Background(), c, w, "http", 8080); err != nil { + t.Fatalf("declareURL: %v", err) + } + + var got networkingv1alpha.HTTPProxy + key := types.NamespacedName{Namespace: util.ResourceNamespace, Name: url.ResourceName(testWorkload)} + if err := c.Get(context.Background(), key, &got); err != nil { + t.Fatalf("reading the proxy back: %v", err) + } + + want := []gatewayv1.Hostname{"api.example.com"} + if !reflect.DeepEqual(got.Spec.Hostnames, want) { + t.Errorf("hostnames after redeploy = %v, want %v — a redeploy must not detach a custom domain", got.Spec.Hostnames, want) + } +} + +// --- fixtures --- + +func workload() *computev1alpha.Workload { + return &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: testWorkload, + Namespace: util.ResourceNamespace, + UID: types.UID("11111111-2222-3333-4444-555555555555"), + }, + } +} + +func workloadWithPorts(ports ...computev1alpha.NamedPort) computev1alpha.Workload { + w := workload() + w.Spec.Template.Spec.Runtime.Sandbox = &computev1alpha.SandboxRuntime{ + Containers: []computev1alpha.SandboxContainer{{Name: "app", Image: "ghcr.io/acme/api:1", Ports: ports}}, + } + return *w +} + +// liveProxy is the proxy as the platform reports it once the URL answers. +func liveProxy(w *computev1alpha.Workload) *networkingv1alpha.HTTPProxy { + p := url.BuildHTTPProxy(w, "http", nil) + p.Status.CanonicalHostname = testCanonical + p.Status.Conditions = []metav1.Condition{ + {Type: networkingv1alpha.HTTPProxyConditionAccepted, Status: metav1.ConditionTrue, Reason: "Accepted"}, + {Type: networkingv1alpha.HTTPProxyConditionProgrammed, Status: metav1.ConditionTrue, Reason: "Programmed"}, + {Type: networkingv1alpha.HTTPProxyConditionCertificatesReady, Status: metav1.ConditionTrue, Reason: "AllCertificatesReady"}, + } + return p +} + +func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch { + t.Helper() + s := runtime.NewScheme() + if err := computev1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering compute scheme: %v", err) + } + if err := networkingv1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering networking scheme: %v", err) + } + return fake.NewClientBuilder(). + WithScheme(s). + WithStatusSubresource(&networkingv1alpha.HTTPProxy{}, &networkingv1alpha.NetworkService{}). + WithObjects(objs...). + Build() +} diff --git a/internal/cmd/compute/deploy/nohttp_test.go b/internal/cmd/compute/deploy/nohttp_test.go new file mode 100644 index 00000000..dfe0079b --- /dev/null +++ b/internal/cmd/compute/deploy/nohttp_test.go @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const testCustom = "api.example.com" + +func publishedClient(t *testing.T, hostnames ...string) client.WithWatch { + t.Helper() + w := workload() + proxy := liveProxy(w) + for _, h := range hostnames { + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, gatewayv1.Hostname(h)) + // A custom hostname only serves on the strength of its own entry here. + // Without one the platform has not checked it yet, and a hostname + // nobody has verified is not the URL to name in a plan. + proxy.Status.HostnameStatuses = append(proxy.Status.HostnameStatuses, + networkingv1alpha.HostnameStatus{ + Hostname: h, + Conditions: []metav1.Condition{ + {Type: networkingv1alpha.HostnameConditionVerified, Status: metav1.ConditionTrue, Reason: "Verified"}, + {Type: networkingv1alpha.HostnameConditionDNSRecordProgrammed, Status: metav1.ConditionTrue, Reason: "RecordCreated"}, + {Type: networkingv1alpha.HostnameConditionCertificateReady, Status: metav1.ConditionTrue, Reason: "CertificateIssued"}, + }, + }) + } + return newFakeClient(t, proxy, url.BuildNetworkService(w, "http", 8080)) +} + +func published(t *testing.T, c client.Client, obj client.Object) bool { + t.Helper() + return c.Get(context.Background(), client.ObjectKey{ + Namespace: util.ResourceNamespace, Name: url.ResourceName(testWorkload), + }, obj) == nil +} + +// TestPlanHTTPServiceStatesTheConsequence: --no-http takes a URL down, and the +// plan summary printed before the Apply prompt has to say which URL, by name. +// A user cannot consent to something the prompt does not mention. +func TestPlanHTTPServiceStatesTheConsequence(t *testing.T) { + tests := []struct { + name string + client func(t *testing.T) client.WithWatch + httpPort int32 + opts *options + creating bool + wantLine string + wantMissing string + wantRemoving string + }{ + { + name: "publishing names the port and says a URL is coming", + client: func(t *testing.T) client.WithWatch { return newFakeClient(t) }, + httpPort: 8080, + opts: &options{httpPort: 8080}, + wantLine: "HTTP service: port 8080 → Datum-managed URL", + }, + { + name: "removing names the URL that stops answering", + client: func(t *testing.T) client.WithWatch { return publishedClient(t) }, + opts: &options{noHTTP: true}, + wantLine: "HTTP service: removed — https://" + testCanonical + " will stop responding", + wantRemoving: "https://" + testCanonical, + }, + { + name: "the custom hostname is the one named, since it is the one people use", + client: func(t *testing.T) client.WithWatch { return publishedClient(t, testCustom) }, + opts: &options{noHTTP: true}, + wantLine: "https://" + testCustom + " will stop responding", + wantRemoving: "https://" + testCustom, + }, + { + name: "removing something that was never published says so plainly", + client: func(t *testing.T) client.WithWatch { return newFakeClient(t) }, + opts: &options{noHTTP: true}, + wantLine: "HTTP service: removed", + }, + { + name: "a workload being created has no URL to lose", + client: func(t *testing.T) client.WithWatch { return newFakeClient(t) }, + opts: &options{noHTTP: true}, + creating: true, + wantMissing: "HTTP service", + }, + { + name: "no HTTP flags at all is not an HTTP plan", + client: func(t *testing.T) client.WithWatch { return publishedClient(t) }, + opts: &options{}, + wantMissing: "HTTP service", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + got := planHTTPService(context.Background(), &out, tc.client(t), testWorkload, tc.httpPort, tc.opts, tc.creating) + + if tc.wantLine != "" && !strings.Contains(out.String(), tc.wantLine) { + t.Errorf("plan missing %q:\n%s", tc.wantLine, out.String()) + } + if tc.wantMissing != "" && strings.Contains(out.String(), tc.wantMissing) { + t.Errorf("plan should not mention %q:\n%s", tc.wantMissing, out.String()) + } + if got != tc.wantRemoving { + t.Errorf("removed URL = %q, want %q", got, tc.wantRemoving) + } + }) + } +} + +// A control plane that cannot be read must not stop a deploy at the plan +// stage: the line loses the hostname it would have named and nothing else. +func TestPlanHTTPServiceSurvivesAnUnreadableControlPlane(t *testing.T) { + c := interceptor.NewClient(publishedClient(t), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errors.New("connection refused") + }, + }) + + var out bytes.Buffer + got := planHTTPService(context.Background(), &out, c, testWorkload, 0, &options{noHTTP: true}, false) + + if got != "" { + t.Errorf("removed URL = %q, want none: nothing was read", got) + } + if !strings.Contains(out.String(), "HTTP service: removed") { + t.Errorf("the plan must still state the removal:\n%s", out.String()) + } + if strings.Contains(out.String(), "will stop responding") { + t.Errorf("no URL is known, so none may be named:\n%s", out.String()) + } +} + +// TestRemoveHTTPServiceTakesBothObjectsDown is the whole point of --no-http: +// the workload stops declaring the port, and what answered for it goes with +// it. Leaving either object behind leaves a URL routing to a workload that no +// longer serves it. +func TestRemoveHTTPServiceTakesBothObjectsDown(t *testing.T) { + c := publishedClient(t, testCustom) + + var out bytes.Buffer + removedURL := "https://" + testCustom + if err := removeHTTPService(context.Background(), &out, c, testWorkload, removedURL, false); err != nil { + t.Fatalf("removeHTTPService: %v", err) + } + + if published(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("the URL survived --no-http") + } + if published(t, c, &networkingv1alpha.NetworkService{}) { + t.Error("the URL backends survived --no-http") + } + if want := "HTTP service removed — " + removedURL + " no longer responds"; !strings.Contains(out.String(), want) { + t.Errorf("output missing %q:\n%s", want, out.String()) + } +} + +// Removing an HTTP service that was never there is a no-op with nothing to +// report — but only on a create. On an existing workload the user asked for +// something, so they are told it happened. +func TestRemoveHTTPServiceSaysNothingOnACreate(t *testing.T) { + c := newFakeClient(t) + + var creating bytes.Buffer + if err := removeHTTPService(context.Background(), &creating, c, testWorkload, "", true); err != nil { + t.Fatalf("removeHTTPService: %v", err) + } + if creating.Len() != 0 { + t.Errorf("a new workload never had an HTTP service, so nothing may be reported:\n%s", creating.String()) + } + + var existing bytes.Buffer + if err := removeHTTPService(context.Background(), &existing, c, testWorkload, "", false); err != nil { + t.Fatalf("removeHTTPService: %v", err) + } + if !strings.Contains(existing.String(), "HTTP service removed") { + t.Errorf("an existing workload's removal must be reported:\n%s", existing.String()) + } +} + +// A URL that could not be taken down is a failed deploy, not a warning: the +// workload has already been updated to stop serving, so a URL still routing to +// it is a live inconsistency the user has to know about. +func TestRemoveHTTPServiceReportsAFailure(t *testing.T) { + boom := errors.New("forbidden") + c := interceptor.NewClient(publishedClient(t), interceptor.Funcs{ + DeleteAllOf: func(context.Context, client.WithWatch, client.Object, ...client.DeleteAllOfOption) error { + return boom + }, + }) + + var out bytes.Buffer + err := removeHTTPService(context.Background(), &out, c, testWorkload, "https://"+testCanonical, false) + if err == nil { + t.Fatal("a URL that could not be removed must fail the deploy") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure", err) + } + if strings.Contains(out.String(), "no longer responds") { + t.Errorf("nothing was removed, so nothing may claim it was:\n%s", out.String()) + } +} + +// TestPublishKeepsCustomHostnamesWhenTheSpecChanges is the carry-forward test +// with the short circuit removed: changing the port forces the apply through +// the update path that rewrites the proxy spec, which is where a dropped +// hostname would actually be lost. Republishing an unchanged workload writes +// nothing at all, so it cannot prove this on its own. +func TestPublishKeepsCustomHostnamesWhenTheSpecChanges(t *testing.T) { + c := publishedClient(t, testCustom, "www.example.com") + + if err := declareURL(context.Background(), c, workload(), "http", 9090); err != nil { + t.Fatalf("declareURL: %v", err) + } + + var proxy networkingv1alpha.HTTPProxy + key := client.ObjectKey{Namespace: util.ResourceNamespace, Name: url.ResourceName(testWorkload)} + if err := c.Get(context.Background(), key, &proxy); err != nil { + t.Fatalf("reading the proxy back: %v", err) + } + + got := make([]string, 0, len(proxy.Spec.Hostnames)) + for _, h := range proxy.Spec.Hostnames { + got = append(got, string(h)) + } + if want := testCustom + "," + "www.example.com"; strings.Join(got, ",") != want { + t.Errorf("hostnames after a port change = %v, want %q — a redeploy must not detach a custom domain", got, want) + } + + // And the port change did land, so the test is exercising the update path. + var svc networkingv1alpha.NetworkService + if err := c.Get(context.Background(), key, &svc); err != nil { + t.Fatalf("reading the backends back: %v", err) + } + if svc.Spec.Ports[0].Port != 9090 { + t.Fatalf("port = %d, want the new port — the update path was not exercised", svc.Spec.Ports[0].Port) + } +} + +// TestExistingHostnames pins what the carry-forward reads, including the two +// cases where it deliberately reports none. +func TestExistingHostnames(t *testing.T) { + t.Run("attached hostnames are carried forward in declared order", func(t *testing.T) { + c := publishedClient(t, testCustom, "www.example.com") + got, err := existingHostnames(context.Background(), c, testWorkload) + if err != nil { + t.Fatalf("existingHostnames: %v", err) + } + if want := testCustom + ",www.example.com"; strings.Join(got, ",") != want { + t.Errorf("existingHostnames = %v, want %q", got, want) + } + }) + + t.Run("a workload that was never published has none", func(t *testing.T) { + got, err := existingHostnames(context.Background(), newFakeClient(t), testWorkload) + if err != nil { + t.Fatalf("a workload with no URL is not an error: %v", err) + } + if got != nil { + t.Errorf("existingHostnames = %v, want nil", got) + } + }) + + t.Run("an unreadable control plane fails closed", func(t *testing.T) { + boom := errors.New("connection refused") + c := interceptor.NewClient(publishedClient(t, testCustom), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return boom + }, + }) + got, err := existingHostnames(context.Background(), c, testWorkload) + if err == nil { + t.Fatal("a control plane that cannot be read must not report \"no custom domains\"") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure", err) + } + if got != nil { + t.Errorf("existingHostnames = %v, want nil alongside the error", got) + } + }) +} + +// TestDeclareURLDoesNotDetachHostnamesWhenTheLookupFails is the hole left in +// the carry-forward: existingHostnames reported none when it could not read the +// URL, and declaring then rewrote the proxy spec with none. The two calls use +// different verbs on the same object — a List for the carry-forward, a Get for +// the apply — so a control plane that answers one and refuses the other would +// detach every custom domain on the next deploy, while the deploy reported +// success. +// +// A missing list permission on httpproxies is the everyday shape of that. +func TestDeclareURLDoesNotDetachHostnamesWhenTheLookupFails(t *testing.T) { + boom := errors.New("httpproxies.networking.datumapis.com is forbidden") + c := interceptor.NewClient(publishedClient(t, testCustom), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return boom + }, + }) + + err := declareURL(context.Background(), c, workload(), "http", 8080) + if err == nil { + t.Fatal("a deploy that could not read the attached domains must fail before it rewrites them") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure", err) + } + + var proxy networkingv1alpha.HTTPProxy + key := client.ObjectKey{Namespace: util.ResourceNamespace, Name: url.ResourceName(testWorkload)} + if err := c.Get(context.Background(), key, &proxy); err != nil { + t.Fatalf("reading the proxy back: %v", err) + } + if len(proxy.Spec.Hostnames) != 1 || string(proxy.Spec.Hostnames[0]) != testCustom { + t.Errorf("hostnames = %v, want %q kept — a deploy that could not read the URL must not detach a domain", + proxy.Spec.Hostnames, testCustom) + } +} + +// And the failure reaches the user as a failure to publish, after the rollout +// table, rather than as a silent success with a detached domain. +func TestPublishReportsACarryForwardFailure(t *testing.T) { + boom := errors.New("httpproxies.networking.datumapis.com is forbidden") + c := interceptor.NewClient(publishedClient(t, testCustom), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return boom + }, + }) + + declareErr := declareURL(context.Background(), c, workload(), "http", 8080) + + var out bytes.Buffer + err := publish(context.Background(), &out, c, workload(), 8080, &options{image: testImage}, declareErr) + if err == nil { + t.Fatal("a URL that was never declared must not be reported as published") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure", err) + } + if !strings.Contains(out.String(), "The rollout succeeded") { + t.Errorf("a running workload must not read as a failed deploy:\n%s", out.String()) + } +} + +// TestPublishSaysNothingMoreAfterNoHTTP: --no-http has just told the user, by +// name, that their URL is gone. Following that with "this workload is not +// reachable from the internet — to publish it..." answers a question nobody +// asked, and reads as though the removal were a mistake. +func TestPublishSaysNothingMoreAfterNoHTTP(t *testing.T) { + var out bytes.Buffer + if err := publish(context.Background(), &out, nil, workload(), 0, &options{noHTTP: true}, nil); err != nil { + t.Fatalf("publish: %v", err) + } + if out.Len() != 0 { + t.Errorf("--no-http was already reported; nothing more may be said:\n%s", out.String()) + } +} + +// TestManifestDeployReportsTheDeadEnd: the spec guarantees the not-reachable +// note for any workload with no HTTP port, and a manifest deploy is no +// exception. Printing it only on the flag path leaves a -f user to conclude +// the URL is somewhere they have not looked. +func TestManifestDeployReportsTheDeadEnd(t *testing.T) { + t.Run("no port declared", func(t *testing.T) { + w := workloadWithPorts() + + var out bytes.Buffer + reportManifestReachability(&out, &w) + + for _, want := range []string{ + "No HTTP port declared — this workload is not reachable from the internet.", + "To publish it: datumctl compute deploy api --http-port 8080", + } { + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q:\n%s", want, out.String()) + } + } + }) + + t.Run("a declared http port is not a dead end", func(t *testing.T) { + w := workloadWithPorts(computev1alpha.NamedPort{Name: "http", Port: 8080}) + + var out bytes.Buffer + reportManifestReachability(&out, &w) + + if out.Len() != 0 { + t.Errorf("a workload that declares an HTTP port is not unreachable:\n%s", out.String()) + } + }) +} + +// TestPlanWarnsThatTheEdgeSpeaksPlaintext: the edge reaches instances over +// plaintext inside the network, so a container terminating TLS itself answers +// nothing. The only symptom is a URL that does not work, which is why this is +// said before the Apply prompt rather than left to be discovered. +func TestPlanWarnsThatTheEdgeSpeaksPlaintext(t *testing.T) { + var out bytes.Buffer + planHTTPService(context.Background(), &out, newFakeClient(t), testWorkload, 8080, &options{httpPort: 8080}, true) + + got := out.String() + if !strings.Contains(got, "Datum terminates TLS; serve plain HTTP on this port.") { + t.Errorf("the plan must say the edge reaches the container over plaintext:\n%s", got) + } + for _, machinery := range []string{"NetworkService", "HTTPProxy"} { + if strings.Contains(got, machinery) { + t.Errorf("the plan names the machinery %q:\n%s", machinery, got) + } + } + + // And it is not said to a workload that publishes nothing. + var internal bytes.Buffer + planHTTPService(context.Background(), &internal, newFakeClient(t), testWorkload, 0, &options{}, false) + if strings.Contains(internal.String(), "TLS") { + t.Errorf("nothing is being published, so TLS is not the user's problem:\n%s", internal.String()) + } +} diff --git a/internal/cmd/compute/destroy/destroy.go b/internal/cmd/compute/destroy/destroy.go index bc5b1b00..d7d8d55d 100644 --- a/internal/cmd/compute/destroy/destroy.go +++ b/internal/cmd/compute/destroy/destroy.go @@ -4,25 +4,52 @@ import ( "bufio" "context" "fmt" + "io" "os" "strings" "github.com/spf13/cobra" "golang.org/x/term" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" ) +// destroyPrompt states every consequence of the command, including the one a +// user is most likely to have forgotten: the workload's URL stops answering. +const destroyPrompt = "This will delete the workload, all its instances, and its URLs. Continue? (y/N): " + +// leftoverPrompt is for the second run of a destroy whose first run deleted +// the workload but could not delete its URL. +const leftoverPrompt = "This will delete the URLs left behind by %s. Continue? (y/N): " + +// summaryLabel keeps the summary block's values in one column. +const summaryLabel = 14 + +// leftoverBackendsDescription names what is left when a partial delete removed +// the URL and not the backends behind it. There is no hostname left to show, +// and the machinery is never named, so this is the plainest true thing to say. +const leftoverBackendsDescription = "URL backends from an unfinished destroy" + func Command() *cobra.Command { var yes bool cmd := &cobra.Command{ Use: "destroy ", - Short: "Delete a workload and all its instances", - Args: cobra.ExactArgs(1), + Short: "Delete a workload, all its instances, and its URLs", + Long: `Delete a workload and everything that serves it: its instances and the URLs it +answers on. + +Custom domains are not deleted. A verified domain is a project asset that +outlives any one workload.`, + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runDestroy(cmd, args, yes) }, @@ -42,18 +69,131 @@ func runDestroy(cmd *cobra.Command, args []string, yes bool) error { return err } - ctx := context.Background() - workloadName := args[0] + return destroyWorkload(context.Background(), cmd.OutOrStdout(), cmd.ErrOrStderr(), c, project, args[0], yes) +} +// destroyWorkload deletes the workload and then the objects that put it on a +// URL. The client is a parameter so the whole flow is testable against a fake +// one. +func destroyWorkload(ctx context.Context, out, errOut io.Writer, c client.Client, project, workloadName string, yes bool) error { var workload computev1alpha.Workload - if err := c.Get(ctx, types.NamespacedName{Namespace: util.ResourceNamespace, Name: workloadName}, &workload); err != nil { - if k8serrors.IsNotFound(err) { - return fmt.Errorf("workload %q not found in project %s", workloadName, project) - } + err := c.Get(ctx, types.NamespacedName{Namespace: util.ResourceNamespace, Name: workloadName}, &workload) + switch { + case err == nil: + case k8serrors.IsNotFound(err): + // The workload is gone. Its URL may not be — a previous destroy can + // have deleted the workload and failed on the URL, and this is the + // command that run told the user to repeat. + return destroyLeftoverURLs(ctx, out, errOut, c, project, workloadName, yes) + default: return fmt.Errorf("getting workload: %w", err) } - // Summarize placements. + // The URL is read before anything is deleted: the summary has to be able + // to name what will stop answering. + info, urlErr := url.ForWorkload(ctx, c, workloadName) + if urlErr != nil { + fmt.Fprintf(errOut, "Warning: could not read URLs for %q: %v\n", workloadName, urlErr) + } + + printSummary(out, &workload, info) + + confirmed, err := confirm(out, yes, destroyPrompt) + if err != nil { + return err + } + if !confirmed { + fmt.Fprintln(out, "Aborted.") + return nil + } + + if err := c.Delete(ctx, &workload); err != nil { + return fmt.Errorf("deleting workload: %w", err) + } + fmt.Fprintf(out, "workload/%s deleted.\n", workloadName) + + // The URL objects are deleted explicitly rather than left to + // owner-reference garbage collection, which a project control plane does + // not guarantee. A failure here fails the command: the destroy was asked to + // stop the URLs answering and it did not, and a script reading exit 0 would + // carry on believing otherwise. The message still says the workload is + // gone, because it is. + if err := url.Unpublish(ctx, c, workloadName); err != nil { + reportLeftoverURLs(errOut, workloadName, info) + return err + } + + return nil +} + +// destroyLeftoverURLs handles a workload that is already gone. When it left no +// URL behind there is nothing to do and the workload really is missing; when +// it did, this cleans it up rather than making the user reach for the API. +func destroyLeftoverURLs(ctx context.Context, out, errOut io.Writer, c client.Client, project, workloadName string, yes bool) error { + info, err := url.ForWorkload(ctx, c, workloadName) + if err != nil { + fmt.Fprintf(errOut, "Warning: could not read URLs for %q: %v\n", workloadName, err) + } + + // A partial delete can remove the URL and leave its backends: the lookup + // keys on the URL, so it reports nothing while there is still something + // there. Asking for the backends directly is what makes the leftovers of + // every partial delete reachable — without it the user is left holding + // objects no command can remove. + backends, backendsErr := leftoverBackends(ctx, c, workloadName) + + // Fail closed. Telling a user there is nothing to clean up is a claim, and a + // read that failed is not evidence for it — the leftovers this command + // exists to remove would be exactly what went unseen. Same choice as + // deploy's existingHostnames, which fails closed rather than detaching + // domains it could not read. + if backendsErr != nil { + return fmt.Errorf("checking for leftover URL resources of %q: %w", workloadName, backendsErr) + } + if err != nil && !backends { + return fmt.Errorf("checking for leftover URLs of %q: %w", workloadName, err) + } + + if info == nil && !backends { + return fmt.Errorf("workload %q not found in project %s", workloadName, project) + } + + // With the URL itself already gone there is no hostname left to name, so + // the summary and the closing line both fall back to what does remain. + urls := hostnameURLs(info) + + fmt.Fprintf(out, "%-*s %s (already deleted)\n", summaryLabel, "Workload:", workloadName) + if len(urls) > 0 { + printURLs(out, info) + } else { + fmt.Fprintf(out, "%-*s %s\n", summaryLabel, "Leftovers:", leftoverBackendsDescription) + } + fmt.Fprintln(out) + + confirmed, err := confirm(out, yes, fmt.Sprintf(leftoverPrompt, workloadName)) + if err != nil { + return err + } + if !confirmed { + fmt.Fprintln(out, "Aborted.") + return nil + } + + // Nothing else is being deleted here, so a failure is the command failing. + if err := url.Unpublish(ctx, c, workloadName); err != nil { + return err + } + + if len(urls) > 0 { + fmt.Fprintf(out, "URLs for %s deleted.\n", workloadName) + } else { + fmt.Fprintf(out, "Leftover URL backends for %s deleted.\n", workloadName) + } + return nil +} + +// printSummary states what is about to be deleted, in the user's terms. +func printSummary(out io.Writer, workload *computev1alpha.Workload, info *url.Info) { var allLocations []string var totalMin int32 for _, p := range workload.Spec.Placements { @@ -63,32 +203,99 @@ func runDestroy(cmd *cobra.Command, args []string, yes bool) error { totalMin += p.ScaleSettings.MinReplicas } - out := cmd.OutOrStdout() - fmt.Fprintf(out, "Workload: %s\nPlacements: %d Locations: %s\nMin replicas: %d\n\n", - workloadName, - len(workload.Spec.Placements), - strings.Join(allLocations, ", "), - totalMin, - ) - - // Prompt unless --yes or non-interactive. - if !yes && term.IsTerminal(int(os.Stdin.Fd())) { - _, _ = fmt.Fprint(out, "This will delete workload and all its instances. Continue? (y/N): ") - line, err := bufio.NewReader(os.Stdin).ReadString('\n') - if err != nil { - return fmt.Errorf("reading confirmation: %w", err) - } - line = strings.TrimSpace(line) - if line != "y" && line != "Y" { - _, _ = fmt.Fprintln(out, "Aborted.") - return nil - } + fmt.Fprintf(out, "%-*s %s\n", summaryLabel, "Workload:", workload.Name) + fmt.Fprintf(out, "%-*s %d Locations: %s\n", summaryLabel, "Placements:", + len(workload.Spec.Placements), strings.Join(allLocations, ", ")) + fmt.Fprintf(out, "%-*s %d\n", summaryLabel, "Min replicas:", totalMin) + printURLs(out, info) + fmt.Fprintln(out) +} + +// leftoverBackends reports whether the URL backends published for a workload +// are still in the project. It is asked only about a workload that is already +// gone, where anything still labelled with its name is debris from a destroy +// that did not finish. +// +// A control plane that does not serve the kind at all has nothing left over: +// that is an empty project, not a failure to report. +func leftoverBackends(ctx context.Context, c client.Client, workloadName string) (bool, error) { + var services networkingv1alpha.NetworkServiceList + err := c.List(ctx, &services, + client.InNamespace(util.ResourceNamespace), + client.MatchingLabels{computev1alpha.WorkloadNameLabel: workloadName}) + switch { + case err == nil: + return len(services.Items) > 0, nil + case notServed(err): + return false, nil + default: + return false, err } +} - if err := c.Delete(ctx, &workload); err != nil { - return fmt.Errorf("deleting workload: %w", err) +// notServed reports whether a list error means the control plane does not +// serve this kind, rather than that the read failed. +func notServed(err error) bool { + return k8serrors.IsNotFound(err) || + meta.IsNoMatchError(err) || + runtime.IsNotRegisteredError(err) +} + +// printURLs adds the URLs line, when there is one. A workload with no HTTP +// port has no line at all rather than a line saying so: the summary lists what +// is being deleted. +func printURLs(out io.Writer, info *url.Info) { + urls := hostnameURLs(info) + if len(urls) == 0 { + return } + fmt.Fprintf(out, "%-*s %s\n", summaryLabel, "URLs:", strings.Join(urls, ", ")) +} - fmt.Fprintf(out, "workload/%s deleted.\n", workloadName) - return nil +// hostnameURLs lists every URL the workload answers on, custom hostnames +// first, the platform-managed one last, as the url package orders them. +func hostnameURLs(info *url.Info) []string { + if info == nil { + return nil + } + urls := make([]string, 0, len(info.Hostnames)) + for _, h := range info.Hostnames { + urls = append(urls, h.URL) + } + if len(urls) == 0 && info.URL != "" { + urls = append(urls, info.URL) + } + return urls +} + +// reportLeftoverURLs explains a URL that outlived its workload. It names +// exactly what is still answering and the command that finishes the job, +// because a URL still serving traffic for a workload the user believes they +// deleted is the worst possible way to be quiet. +// +// The cause is not repeated here: the caller returns it, and it is printed as +// the command's error immediately after this block. +func reportLeftoverURLs(errOut io.Writer, workloadName string, info *url.Info) { + fmt.Fprintf(errOut, "\nThe workload was deleted, but its URLs were not.\n") + for _, u := range hostnameURLs(info) { + fmt.Fprintf(errOut, " %s may keep answering.\n", u) + } + fmt.Fprintf(errOut, " Run 'datumctl compute destroy %s' again to remove them.\n", workloadName) +} + +// confirm asks the question and reports whether the user said yes. --yes skips +// it; so does a non-interactive run, which is the behaviour this command has +// always had. +func confirm(out io.Writer, yes bool, question string) (bool, error) { + if yes || !term.IsTerminal(int(os.Stdin.Fd())) { + return true, nil + } + + fmt.Fprint(out, question) + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + if err != nil { + return false, fmt.Errorf("reading confirmation: %w", err) + } + line = strings.TrimSpace(line) + return line == "y" || line == "Y", nil } diff --git a/internal/cmd/compute/destroy/destroy_test.go b/internal/cmd/compute/destroy/destroy_test.go new file mode 100644 index 00000000..8c40cb84 --- /dev/null +++ b/internal/cmd/compute/destroy/destroy_test.go @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package destroy + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + testProject = "acme-prod" + testWorkload = "api" + testCanonical = "a1b2c3d4.datumproxy.net" + testCustom = "api.example.com" +) + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := computev1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering compute scheme: %v", err) + } + if err := networkingv1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering networking scheme: %v", err) + } + return s +} + +func testWorkloadObject() *computev1alpha.Workload { + minReplicas := int32(2) + return &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: testWorkload, + Namespace: util.ResourceNamespace, + UID: types.UID("uid-api"), + }, + Spec: computev1alpha.WorkloadSpec{ + Placements: []computev1alpha.WorkloadPlacement{{ + Name: "default", + Locations: []locationsv1alpha1.LocationReference{{Name: "us-east-1"}, {Name: "eu-west-1"}}, + ScaleSettings: computev1alpha.HorizontalScaleSettings{MinReplicas: minReplicas}, + }}, + }, + } +} + +// publishedProxy is the proxy the platform reports for a live URL. +func publishedProxy(customHostnames ...string) *networkingv1alpha.HTTPProxy { + hostnames := make([]gatewayv1.Hostname, 0, len(customHostnames)) + for _, h := range customHostnames { + hostnames = append(hostnames, gatewayv1.Hostname(h)) + } + return &networkingv1alpha.HTTPProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: testWorkload, + Namespace: util.ResourceNamespace, + Labels: map[string]string{computev1alpha.WorkloadNameLabel: testWorkload}, + }, + Spec: networkingv1alpha.HTTPProxySpec{Hostnames: hostnames}, + Status: networkingv1alpha.HTTPProxyStatus{ + CanonicalHostname: testCanonical, + Conditions: []metav1.Condition{ + {Type: networkingv1alpha.HTTPProxyConditionProgrammed, Status: metav1.ConditionTrue, Reason: "Programmed"}, + {Type: networkingv1alpha.HTTPProxyConditionCertificatesReady, Status: metav1.ConditionTrue, Reason: "Issued"}, + }, + }, + } +} + +func publishedService() *networkingv1alpha.NetworkService { + return &networkingv1alpha.NetworkService{ + ObjectMeta: metav1.ObjectMeta{ + Name: testWorkload, + Namespace: util.ResourceNamespace, + Labels: map[string]string{computev1alpha.WorkloadNameLabel: testWorkload}, + }, + Spec: networkingv1alpha.NetworkServiceSpec{ + Ports: []networkingv1alpha.NetworkServicePort{{Name: "http", Port: 8080}}, + }, + } +} + +func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch { + t.Helper() + return fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(objs...).Build() +} + +func exists(t *testing.T, c client.Client, obj client.Object) bool { + t.Helper() + err := c.Get(context.Background(), types.NamespacedName{Namespace: util.ResourceNamespace, Name: testWorkload}, obj) + if err == nil { + return true + } + if k8serrors.IsNotFound(err) { + return false + } + t.Fatalf("get: %v", err) + return false +} + +// TestDestroyPrompt pins the sentence the user is asked to agree to. It has to +// name the URLs: a workload's URL is the part of a destroy a user is most +// likely not to have thought about. +func TestDestroyPrompt(t *testing.T) { + const want = "This will delete the workload, all its instances, and its URLs. Continue? (y/N): " + if destroyPrompt != want { + t.Errorf("destroyPrompt = %q, want %q", destroyPrompt, want) + } +} + +func TestDestroySummary(t *testing.T) { + tests := []struct { + name string + objs []client.Object + wantLines []string + wantMissing []string + }{ + { + name: "published workload lists every URL it answers on", + objs: []client.Object{testWorkloadObject(), publishedProxy(testCustom), publishedService()}, + wantLines: []string{ + "Workload: " + testWorkload, + "Placements: 1 Locations: us-east-1, eu-west-1", + "Min replicas: 2", + "URLs: https://" + testCustom + ", https://" + testCanonical, + "workload/api deleted.", + }, + }, + { + name: "workload with no URL has no URLs line", + objs: []client.Object{testWorkloadObject()}, + wantLines: []string{"Workload: " + testWorkload, "workload/api deleted."}, + wantMissing: []string{"URLs:"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t, tc.objs...) + + if err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true); err != nil { + t.Fatalf("destroyWorkload: %v", err) + } + + got := out.String() + for _, want := range tc.wantLines { + if !strings.Contains(got, want) { + t.Errorf("output missing %q:\n%s", want, got) + } + } + for _, missing := range tc.wantMissing { + if strings.Contains(got, missing) { + t.Errorf("output should not contain %q:\n%s", missing, got) + } + } + if errOut.Len() != 0 { + t.Errorf("unexpected stderr: %s", errOut.String()) + } + }) + } +} + +// TestDestroyUnpublishes: the URL objects are deleted explicitly, not left to +// owner-reference garbage collection. +func TestDestroyUnpublishes(t *testing.T) { + c := newFakeClient(t, testWorkloadObject(), publishedProxy(testCustom), publishedService()) + + var out, errOut bytes.Buffer + if err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true); err != nil { + t.Fatalf("destroyWorkload: %v", err) + } + + if exists(t, c, &computev1alpha.Workload{}) { + t.Error("workload survived destroy") + } + if exists(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("URL survived destroy") + } + if exists(t, c, &networkingv1alpha.NetworkService{}) { + t.Error("URL backends survived destroy") + } +} + +// TestDestroyLeftoverURLFails: a URL that could not be deleted names exactly +// what is still answering and how to finish the job — and fails the command. +// The workload really is gone, but a script that reads exit 0 here would carry +// on believing the URLs stopped answering when they may not have. +func TestDestroyLeftoverURLFails(t *testing.T) { + boom := errors.New("forbidden") + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(testWorkloadObject(), publishedProxy(testCustom), publishedService()). + WithInterceptorFuncs(interceptor.Funcs{ + DeleteAllOf: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { + if _, ok := obj.(*networkingv1alpha.HTTPProxy); ok { + return boom + } + return cl.DeleteAllOf(ctx, obj, opts...) + }, + }). + Build() + + var out, errOut bytes.Buffer + err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true) + if err == nil { + t.Fatal("URLs that outlived the workload must fail the command") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure", err) + } + + if !strings.Contains(out.String(), "workload/api deleted.") { + t.Errorf("stdout should still report the workload deleted:\n%s", out.String()) + } + + warning := errOut.String() + for _, want := range []string{ + "workload was deleted, but its URLs were not", + "https://" + testCustom, + "https://" + testCanonical, + "datumctl compute destroy " + testWorkload, + } { + if !strings.Contains(warning, want) { + t.Errorf("message missing %q:\n%s", want, warning) + } + } +} + +// TestDestroyLeftoverURLCleanup: the retry the warning advertises works — +// the workload is already gone, and destroy removes what it left behind. +func TestDestroyLeftoverURLCleanup(t *testing.T) { + c := newFakeClient(t, publishedProxy(testCustom), publishedService()) + + var out, errOut bytes.Buffer + if err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true); err != nil { + t.Fatalf("destroyWorkload: %v", err) + } + + if !strings.Contains(out.String(), "already deleted") { + t.Errorf("output should say the workload was already gone:\n%s", out.String()) + } + if !strings.Contains(out.String(), "URLs for api deleted.") { + t.Errorf("output should report the URLs deleted:\n%s", out.String()) + } + if exists(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("leftover URL survived") + } + if exists(t, c, &networkingv1alpha.NetworkService{}) { + t.Error("leftover URL backends survived") + } +} + +// TestDestroyMissingWorkload: nothing to destroy and nothing left behind is +// the plain not-found error it always was. +func TestDestroyMissingWorkload(t *testing.T) { + c := newFakeClient(t) + + var out, errOut bytes.Buffer + err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true) + if err == nil { + t.Fatal("expected an error for a missing workload") + } + if !strings.Contains(err.Error(), `workload "api" not found in project acme-prod`) { + t.Errorf("unexpected error: %v", err) + } +} + +func TestHostnameURLs(t *testing.T) { + tests := []struct { + name string + objs []client.Object + want []string + }{ + { + name: "custom hostname first, managed last", + objs: []client.Object{publishedProxy(testCustom), publishedService()}, + want: []string{"https://" + testCustom, "https://" + testCanonical}, + }, + { + name: "managed hostname alone", + objs: []client.Object{publishedProxy(), publishedService()}, + want: []string{"https://" + testCanonical}, + }, + { + name: "unpublished workload has none", + objs: nil, + want: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(t, tc.objs...) + info, err := url.ForWorkload(context.Background(), c, testWorkload) + if err != nil { + t.Fatalf("lookup: %v", err) + } + got := hostnameURLs(info) + if strings.Join(got, ",") != strings.Join(tc.want, ",") { + t.Errorf("hostnameURLs = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/cmd/compute/destroy/leftover_test.go b/internal/cmd/compute/destroy/leftover_test.go new file mode 100644 index 00000000..cb730af9 --- /dev/null +++ b/internal/cmd/compute/destroy/leftover_test.go @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package destroy + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// clientFailingToDeleteBackends is the half of a partial delete that the +// existing tests do not cover: the URL comes down, the backends behind it do +// not. url.Unpublish deletes the proxy first, so this is the ordering a +// permission problem on one kind actually produces. +func clientFailingToDeleteBackends(t *testing.T, objs ...client.Object) client.WithWatch { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(objs...). + WithInterceptorFuncs(interceptor.Funcs{ + DeleteAllOf: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { + if _, ok := obj.(*networkingv1alpha.NetworkService); ok { + return errors.New("forbidden") + } + return cl.DeleteAllOf(ctx, obj, opts...) + }, + }). + Build() +} + +// TestDestroyFailsWhenOnlyTheBackendsAreLeft: the workload and its URL are +// gone, the backends are not. The destroy reports the workload deleted, names +// the retry — and still fails, because something it was asked to remove is +// still there. +func TestDestroyFailsWhenOnlyTheBackendsAreLeft(t *testing.T) { + c := clientFailingToDeleteBackends(t, testWorkloadObject(), publishedProxy(testCustom), publishedService()) + + var out, errOut bytes.Buffer + err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true) + if err == nil { + t.Fatal("leftover backends must fail the destroy") + } + + if !strings.Contains(out.String(), "workload/api deleted.") { + t.Errorf("stdout should still report the workload deleted:\n%s", out.String()) + } + if exists(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("the URL should have been deleted before the backends were attempted") + } + if !exists(t, c, &networkingv1alpha.NetworkService{}) { + t.Fatal("test is not exercising the leftover-backends case") + } + if !strings.Contains(errOut.String(), "datumctl compute destroy "+testWorkload) { + t.Errorf("the message must name the command that finishes the job:\n%s", errOut.String()) + } +} + +// TestDestroyRetryRemovesLeftoverBackends is the promise the message above +// makes, taken at its word: run destroy again and what was left behind is +// removed. The workload is gone and so is the proxy the URL lookup keys on, so +// the only trace left is the backends — and destroy still has to find and +// remove them, because no other command can. +func TestDestroyRetryRemovesLeftoverBackends(t *testing.T) { + // The state the first run left: no workload, no proxy, backends still + // there. + c := newFakeClient(t, publishedService()) + + var out, errOut bytes.Buffer + if err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true); err != nil { + t.Fatalf("the retry the message advertises must work, got: %v", err) + } + if exists(t, c, &networkingv1alpha.NetworkService{}) { + t.Error("the leftover backends survived the retry, with no other command able to remove them") + } + if !strings.Contains(out.String(), "already deleted") { + t.Errorf("output should say the workload was already gone:\n%s", out.String()) + } + if !strings.Contains(out.String(), "Leftover URL backends for api deleted.") { + t.Errorf("output should report exactly what it removed:\n%s", out.String()) + } +} + +// TestDestroyRetryReportsWhatIsLeftBehind: the confirmation for a cleanup-only +// run has to describe what it is about to remove. When the proxy is gone there +// is no hostname left to name, so the summary says what remains in the user's +// vocabulary rather than printing nothing at all. +func TestDestroyRetryReportsWhatIsLeftBehind(t *testing.T) { + c := newFakeClient(t, publishedService()) + + var out, errOut bytes.Buffer + if err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true); err != nil { + t.Fatalf("destroyWorkload: %v", err) + } + if !strings.Contains(out.String(), "Leftovers:") { + t.Errorf("the summary must name what is still there:\n%s", out.String()) + } + for _, machinery := range []string{"NetworkService", "HTTPProxy"} { + if strings.Contains(out.String(), machinery) { + t.Errorf("output names the machinery %q:\n%s", machinery, out.String()) + } + } +} + +// The retry does work when it is the proxy that was left behind, because that +// is what the lookup keys on. Pinned so a fix for the case above is not read +// as a regression here. +func TestDestroyRetryRemovesALeftoverProxy(t *testing.T) { + c := newFakeClient(t, publishedProxy(testCustom)) + + var out, errOut bytes.Buffer + if err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true); err != nil { + t.Fatalf("destroyWorkload: %v", err) + } + if exists(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("the leftover URL survived the retry") + } + if !strings.Contains(out.String(), "URLs for api deleted.") { + t.Errorf("output should report the URLs deleted:\n%s", out.String()) + } +} + +// TestDestroyReportsAFailedWorkloadDelete: the workload itself failing to +// delete is a failed destroy, and nothing may be unpublished after it — a URL +// removed from under a workload that still exists takes a live service down +// for no reason. +func TestDestroyReportsAFailedWorkloadDelete(t *testing.T) { + boom := errors.New("forbidden") + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(testWorkloadObject(), publishedProxy(testCustom), publishedService()). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*computev1alpha.Workload); ok { + return boom + } + return cl.Delete(ctx, obj, opts...) + }, + }). + Build() + + var out, errOut bytes.Buffer + err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true) + if err == nil { + t.Fatal("a workload that could not be deleted must fail the command") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure", err) + } + if !exists(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("the URL was taken down for a workload that is still running") + } + if !exists(t, c, &networkingv1alpha.NetworkService{}) { + t.Error("the backends were taken down for a workload that is still running") + } +} + +// TestDestroySummaryWarnsWhenURLsCannotBeRead: a destroy whose URL lookup +// fails still deletes the workload, but the user has to be told the summary is +// incomplete rather than reading a missing URLs line as "there were none". +func TestDestroySummaryWarnsWhenURLsCannotBeRead(t *testing.T) { + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(testWorkloadObject(), publishedProxy(testCustom), publishedService()). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*networkingv1alpha.HTTPProxyList); ok { + return errors.New("forbidden") + } + return cl.List(ctx, list, opts...) + }, + }). + Build() + + var out, errOut bytes.Buffer + if err := destroyWorkload(context.Background(), &out, &errOut, c, testProject, testWorkload, true); err != nil { + t.Fatalf("an unreadable URL must not fail the destroy: %v", err) + } + + if !strings.Contains(errOut.String(), "could not read URLs") { + t.Errorf("stderr must say the summary is incomplete:\n%s", errOut.String()) + } + if strings.Contains(out.String(), "URLs:") { + t.Errorf("no URL is known, so none may be claimed:\n%s", out.String()) + } + if exists(t, c, &computev1alpha.Workload{}) { + t.Error("workload survived destroy") + } + // The delete itself does not depend on the lookup: the URL still goes. + if exists(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("URL survived destroy") + } +} diff --git a/internal/cmd/compute/domains/add.go b/internal/cmd/compute/domains/add.go new file mode 100644 index 00000000..fd767a3d --- /dev/null +++ b/internal/cmd/compute/domains/add.go @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + // pollInterval matches the rollout watcher and the publish watcher, so + // every wait in this CLI ticks at the same rate. + pollInterval = 2 * time.Second + + // maxReadFailures is how many consecutive failed reads the wait rides out + // before giving up and saying why, matching the publish watcher. + maxReadFailures = 3 + + // maxWait bounds the whole wait so no path hangs, however healthy the + // polling looks. + maxWait = 15 * time.Minute + + // blockingGrace is how long to wait before repeating what the server says + // is holding a hostname up. Every hostname starts out unverified; saying so + // in the first second is noise, not diagnosis. + blockingGrace = 20 * time.Second + + // recordWindow is how long the platform gets to work out what it needs + // from the user before this command stops waiting quietly and says so. + // + // A hostname is attached the instant the update lands, which is before the + // platform has created the domain it belongs to or computed the record + // that proves the user owns it. Printing the records block in that moment + // would print it without the one record the whole flow turns on, and there + // is no second chance: the block is printed once. So nothing is announced + // until the records are known — and if they never are, that is said out + // loud rather than waited out in silence. + recordWindow = 60 * time.Second + + // stepWidth aligns the checkmarks in a fixed column. + stepWidth = 24 +) + +// stepLabels turns the per-hostname condition types the platform publishes +// into the words a developer thinks in. This maps condition *types*, which are +// API constants; the reasons behind them are never interpreted, only echoed. +var stepLabels = map[string]string{ + networkingv1alpha.HostnameConditionVerified: "Domain verified", + networkingv1alpha.HostnameConditionAvailable: "Hostname available", + networkingv1alpha.HostnameConditionDNSRecordProgrammed: "DNS record programmed", + networkingv1alpha.HostnameConditionCertificateReady: "Certificate issued", +} + +// stepOrder is the order the steps are reported in when several land at once. +// Conditions outside this list are shown after it, under their own names, so a +// check the platform adds tomorrow reaches the user without a CLI release. +var stepOrder = []string{ + networkingv1alpha.HostnameConditionVerified, + networkingv1alpha.HostnameConditionAvailable, + networkingv1alpha.HostnameConditionDNSRecordProgrammed, + networkingv1alpha.HostnameConditionCertificateReady, +} + +func addCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "add ", + Short: "Attach a custom hostname to a workload", + Long: "Attach a custom hostname to a workload's URL.\n\n" + + "Datum prints the DNS records to create, then waits for the domain to verify,\n" + + "for DNS to resolve, and for a certificate to be issued. Ctrl-C detaches — the\n" + + "work continues, and 'datumctl compute domains' shows where it got to.\n\n" + + "A workload may serve several custom hostnames. Attaching one never replaces\n" + + "another: every hostname on a workload routes to that same workload, and the\n" + + "workload keeps its Datum-managed hostname besides. To stop serving a hostname,\n" + + "detach it with 'datumctl compute domains remove'.\n\n" + + "Serving different content per hostname — path routing, host-based routing,\n" + + "several backends behind one name — is manifest territory, not this command's.", + Example: " # Attach a hostname\n" + + " datumctl compute domains add api api.example.com\n\n" + + " # A second hostname, alongside the first\n" + + " datumctl compute domains add api www.example.com", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := util.NewClient(util.ProjectFromCmd(cmd)) + if err != nil { + return err + } + // Ctrl-C detaches from the wait. It never cancels the attach: the + // hostname is already on the proxy by the time we are waiting. + ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) + defer cancel() + return runAdd(ctx, cmd.OutOrStdout(), c, util.ProjectFromCmd(cmd), args[0], args[1]) + }, + ValidArgsFunction: util.CompleteWorkloadNames, + } + + return cmd +} + +// runAdd appends a hostname to the workload's URL and waits for it to answer. +// +// Attaching the same hostname twice is not an error: the wait is the useful +// part of this command, and a user re-running it after a Ctrl-C wants to pick +// the wait back up, not to be told off. +func runAdd(ctx context.Context, out io.Writer, c client.Client, project, workloadName, rawHostname string) error { + if ctx == nil { + ctx = context.Background() + } + + hostname, err := validateHostname(rawHostname) + if err != nil { + return err + } + + info, err := urlFor(ctx, c, project, workloadName) + if err != nil { + return err + } + + if sameHostname(info.CanonicalHostname, hostname) { + return fmt.Errorf("%s is already the Datum-managed hostname for workload %q", hostname, workloadName) + } + + fmt.Fprintln(out) + + if attached(info, hostname) { + fmt.Fprintf(out, "%s is already attached to workload %q.\n", hostname, workloadName) + } else { + proxy := info.Proxy.DeepCopy() + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, gatewayv1.Hostname(hostname)) + if err := c.Update(ctx, proxy); err != nil { + return fmt.Errorf("attaching %s to workload %q: %w", hostname, workloadName, err) + } + + // Attaching is the one part of this command that is already done by + // the time the waiting starts, so it is reported before the wait + // rather than after it. A workload that now answers to more than one + // custom hostname is told so plainly: nothing was replaced. + fmt.Fprintf(out, "%s attached to workload %q.\n", hostname, workloadName) + if total := len(info.CustomHostnames) + 1; total > 1 { + fmt.Fprintf(out, "Workload %q now serves %d custom hostnames; all of them route to it.\n", + workloadName, total) + } + } + + fmt.Fprintln(out) + + return waitForHostname(ctx, out, c, workloadName, hostname) +} + +// attached reports whether a hostname is already on the URL. +func attached(info *url.Info, hostname string) bool { + for _, h := range info.CustomHostnames { + if sameHostname(h, hostname) { + return true + } + } + return false +} + +// waitForHostname polls until the hostname is verified, resolving and covered +// by a certificate, printing each step as the platform reports it. +func waitForHostname(ctx context.Context, out io.Writer, c client.Client, workloadName, hostname string) error { + w := &hostnameWatch{ + out: out, + hostname: hostname, + started: time.Now(), + seen: map[string]bool{}, + } + + // A read that keeps failing is not going to start working, and a wait that + // hides it hangs forever; a single blip is ridden out. Same budget and same + // reasoning as the publish watcher, which had this exact defect. + failures := 0 + step := func() (bool, error) { + done, err := w.check(ctx, c, workloadName) + var re readError + if errors.As(err, &re) { + failures++ + if failures >= maxReadFailures { + return false, fmt.Errorf("checking %s on workload %q: %w", hostname, workloadName, re.err) + } + return false, nil + } + failures = 0 + return done, err + } + + done, err := step() + if done || err != nil { + return err + } + + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + // Nothing bounds how long a developer waits on DNS they control, so the + // deadline is the backstop that keeps the command from hanging rather than + // a judgement about how long verification should take. + deadline := time.NewTimer(maxWait) + defer deadline.Stop() + + for { + select { + case <-ctx.Done(): + fmt.Fprintf(out, "\nDetached. Verification continues in the background.\n") + fmt.Fprintln(out, " Check it with: datumctl compute domains") + return nil + + case <-deadline.C: + return fmt.Errorf("%s on workload %q was not verified within %s — check it with: datumctl compute domains", + hostname, workloadName, maxWait) + + case <-ticker.C: + done, err := step() + if done || err != nil { + return err + } + } + } +} + +// readError marks a failure to read the current state, as opposed to a failure +// that means waiting longer is pointless. Reads are retried; anything else ends +// the wait immediately. +type readError struct{ err error } + +func (e readError) Error() string { return e.err.Error() } +func (e readError) Unwrap() error { return e.err } + +// hostnameWatch prints the progress of one hostname exactly once per fact. +type hostnameWatch struct { + out io.Writer + hostname string + started time.Time + seen map[string]bool +} + +// check reads the current state and reports whether the hostname is live, and +// whether waiting any longer would be pointless. +// +// A read failure is reported as a readError rather than swallowed. One is +// nothing — the hostname was just written and the next tick is a better answer +// — but a control plane this command cannot read is not going to start +// answering, and a wait that hides that hangs forever. waitForHostname decides +// how many to ride out. The other error this returns is the one from announce: +// the platform never told the user what to create, so there is nothing left for +// a wait to do. +func (w *hostnameWatch) check(ctx context.Context, c client.Client, workloadName string) (bool, error) { + info, err := url.ForWorkload(ctx, c, workloadName) + if err != nil { + return false, readError{err: err} + } + if info == nil { + return false, nil + } + h := hostnameFrom(info, w.hostname) + if h == nil { + return false, nil + } + + if hostnameLive(h) { + w.steps(h) + fmt.Fprintf(w.out, "\n %s\n", h.URL) + return true, nil + } + + domain := domainFor(ctx, c, w.hostname) + announced, err := w.announce(info, h, domain) + if err != nil { + return false, err + } + if !announced { + // Checkmarks and blocking notes belong under the heading that has not + // been printed yet. Nothing is said about a hostname before the user + // has been told what the hostname is waiting for. + return false, nil + } + + w.steps(h) + + if time.Since(w.started) > blockingGrace { + w.blocking(h, domain) + } + return false, nil +} + +// announce prints the records the user has to create and the note that says +// Ctrl-C is safe, and reports whether it has done so yet. +// +// It waits for the platform to have worked out what it needs. A hostname is +// attached the moment the update lands, which is before the domain behind it +// exists and before the record that proves ownership has been computed; a +// records block printed then is a block with the TXT record missing, and the +// user is left waiting on DNS they were never shown. So until the records are +// known this prints nothing and asks to be called again — with one exception: +// once recordWindow has passed, whatever the platform did publish is printed +// along with a note about what it has not, and if that is nothing at all the +// wait ends with an error rather than continuing in silence. +// +// Nothing here is printed twice, and a record that only turns up later is +// still printed when it does. +func (w *hostnameWatch) announce(info *url.Info, h *url.Hostname, domain *networkingv1alpha.Domain) (bool, error) { + records := recordsFor(info, h, domain) + known := recordsKnown(info, h, domain) + + if !known && time.Since(w.started) < recordWindow { + return false, nil + } + if !known && len(records) == 0 { + return false, fmt.Errorf( + "%s is attached to workload %q, but Datum has published nothing to create for it after %s — check it with: datumctl compute domains", + w.hostname, info.WorkloadName, recordWindow) + } + + name := w.hostname + if domain != nil && domain.Spec.DomainName != "" { + name = domain.Spec.DomainName + } + w.once("verifying", func() { fmt.Fprintf(w.out, "Verifying %s...\n", name) }) + + w.records(records) + + if !known { + w.once("incomplete", func() { + fmt.Fprintf(w.out, "\n Datum has not published a verification record for %s yet.\n", name) + fmt.Fprintln(w.out, " It will appear here if one is needed.") + }) + } + + w.once("waiting", func() { + fmt.Fprintln(w.out, "\nWaiting for DNS. Ctrl-C to detach — run 'datumctl compute domains' to check.") + fmt.Fprintln(w.out) + }) + return true, nil +} + +// records prints the records the user has not been shown yet. Under normal +// conditions that is every record at once, in one block, because announce +// waits for them all; a record the platform only worked out afterwards gets a +// block of its own rather than going unmentioned. +func (w *hostnameWatch) records(records []record) { + var unseen []record + for _, r := range records { + key := "record:" + r.Type + " " + r.Name + " " + r.Value + if w.seen[key] { + continue + } + w.seen[key] = true + unseen = append(unseen, r) + } + if len(unseen) == 0 { + return + } + + fmt.Fprintln(w.out, "\n Add these DNS records:") + fmt.Fprintln(w.out) + tw := util.NewTabWriter(w.out) + fmt.Fprintln(tw, " TYPE\tNAME\tVALUE") + for _, r := range unseen { + fmt.Fprintf(tw, " %s\t%s\t%s\n", r.Type, r.Name, r.Value) + } + _ = tw.Flush() +} + +// once runs f the first time it is asked to, and never again. +func (w *hostnameWatch) once(key string, f func()) { + if w.seen[key] { + return + } + w.seen[key] = true + f() +} + +// recordsKnown reports whether the platform has finished working out what it +// needs from the user for this hostname: whether ownership has to be proved +// and, if so, with which record, and what the hostname should be pointed at. +// +// It is deliberately about the platform having answered rather than about +// there being records to print: a domain already verified in this project +// needs no TXT record, and that is an answer. +func recordsKnown(info *url.Info, h *url.Hostname, domain *networkingv1alpha.Domain) bool { + return verificationKnown(domain) && targetKnown(info, h) +} + +// verificationKnown reports whether the platform has settled how ownership of +// the domain is proved. No Domain at all means it has not started: the +// platform creates one the first time it tries to program a hostname. +func verificationKnown(domain *networkingv1alpha.Domain) bool { + if domain == nil { + return false + } + if conditionTrue(domain.Status.Conditions, networkingv1alpha.DomainConditionVerified) { + return true + } + v := domain.Status.Verification + return v != nil && v.DNSRecord.Type != "" && v.DNSRecord.Name != "" && v.DNSRecord.Content != "" +} + +// targetKnown reports whether the hostname can be pointed somewhere: either +// the platform has assigned the hostname to point at, or it has programmed the +// record itself and there is nothing for the user to point. +func targetKnown(info *url.Info, h *url.Hostname) bool { + return info.CanonicalHostname != "" || + conditionTrue(h.Conditions, networkingv1alpha.HostnameConditionDNSRecordProgrammed) +} + +// steps prints a checkmark for each check the platform reports as passed. +func (w *hostnameWatch) steps(h *url.Hostname) { + for _, condType := range stepOrder { + if c := util.FindCondition(h.Conditions, condType); c != nil && c.Status == metav1.ConditionTrue { + w.step(condType) + } + } + for _, c := range h.Conditions { + if c.Status == metav1.ConditionTrue { + w.step(c.Type) + } + } +} + +// step prints one checkmark row, under the condition's own type when this CLI +// has no friendlier name for it. +func (w *hostnameWatch) step(condType string) { + if w.seen["step:"+condType] { + return + } + w.seen["step:"+condType] = true + + label, ok := stepLabels[condType] + if !ok { + label = condType + } + fmt.Fprintf(w.out, " %-*s ✓\n", stepWidth, label) +} + +// blocking echoes, verbatim and once each, whatever the server says is holding +// the hostname up — including the domain's own verification message, which is +// where "we looked for the TXT record and did not find it" shows up. +func (w *hostnameWatch) blocking(h *url.Hostname, domain *networkingv1alpha.Domain) { + for _, c := range h.Conditions { + if c.Status != metav1.ConditionTrue && c.Reason != "" { + w.note(c.Reason, c.Message) + } + } + if domain == nil { + return + } + if c := util.FindCondition(domain.Status.Conditions, networkingv1alpha.DomainConditionVerified); c != nil && + c.Status != metav1.ConditionTrue && c.Reason != "" { + w.note(c.Reason, c.Message) + } +} + +func (w *hostnameWatch) note(reason, message string) { + text := url.HumanBlock(reason, message) + if w.seen["note:"+text] { + return + } + w.seen["note:"+text] = true + fmt.Fprintf(w.out, " %-*s %s\n", stepWidth, "", text) +} + +// hostnameLive reports whether a hostname is ready to hand to the user: it is +// serving, and its certificate has been issued (or the platform has not +// reported a certificate check for it, in which case there is nothing to wait +// for). +func hostnameLive(h *url.Hostname) bool { + if h == nil || !h.Active { + return false + } + if c := util.FindCondition(h.Conditions, networkingv1alpha.HostnameConditionCertificateReady); c != nil { + return c.Status == metav1.ConditionTrue + } + return true +} + +// hostnameFrom finds one hostname's state in a workload's URL info. +func hostnameFrom(info *url.Info, hostname string) *url.Hostname { + for i := range info.Hostnames { + if sameHostname(info.Hostnames[i].Hostname, hostname) { + return &info.Hostnames[i] + } + } + return nil +} + +// record is one DNS record the user has to create, exactly as the platform +// reports it. Nothing in this file synthesizes a record value: a record the +// server has not told us about is a record we do not print. +type record struct { + Type string + Name string + Value string +} + +// recordsFor returns the records still outstanding for a hostname: the TXT the +// platform wants for proof of ownership, and the CNAME that points the hostname +// at the platform's own. +// +// Each is dropped once the corresponding condition goes True — a domain that is +// already verified needs no TXT, and a hostname whose record the platform +// programmed itself needs no CNAME. +func recordsFor(info *url.Info, h *url.Hostname, domain *networkingv1alpha.Domain) []record { + var records []record + + if domain != nil && !conditionTrue(domain.Status.Conditions, networkingv1alpha.DomainConditionVerified) { + if v := domain.Status.Verification; v != nil { + r := v.DNSRecord + if r.Type != "" && r.Name != "" && r.Content != "" { + records = append(records, record{Type: r.Type, Name: r.Name, Value: r.Content}) + } + } + } + + if info.CanonicalHostname != "" && !conditionTrue(h.Conditions, networkingv1alpha.HostnameConditionDNSRecordProgrammed) { + records = append(records, record{Type: "CNAME", Name: h.Hostname, Value: info.CanonicalHostname}) + } + + return records +} + +func conditionTrue(conditions []metav1.Condition, condType string) bool { + c := util.FindCondition(conditions, condType) + return c != nil && c.Status == metav1.ConditionTrue +} + +// domainFor returns the Domain resource covering a hostname. +// +// The platform creates one automatically the first time it tries to program a +// hostname, so there may not be one yet, and it is matched by suffix: a Domain +// for example.com covers api.example.com. Nothing here creates one — doing so +// would duplicate an object the platform owns. A control plane that does not +// serve Domains at all reads the same as one with no Domain yet: there is +// simply nothing extra to show. +func domainFor(ctx context.Context, c client.Client, hostname string) *networkingv1alpha.Domain { + var list networkingv1alpha.DomainList + if err := c.List(ctx, &list, client.InNamespace(util.ResourceNamespace)); err != nil { + return nil + } + + var best *networkingv1alpha.Domain + for i := range list.Items { + name := normalizeHostname(list.Items[i].Spec.DomainName) + if name == "" || !covers(name, hostname) { + continue + } + // The most specific Domain wins: a project may hold both example.com + // and api.example.com, and the latter is the one being verified. + if best == nil || len(name) > len(normalizeHostname(best.Spec.DomainName)) { + best = &list.Items[i] + } + } + return best +} + +// covers reports whether a domain name covers a hostname, by the platform's own +// rule: an exact match, or a suffix match on a label boundary. +func covers(domainName, hostname string) bool { + h := normalizeHostname(hostname) + return h == domainName || strings.HasSuffix(h, "."+domainName) +} diff --git a/internal/cmd/compute/domains/add_test.go b/internal/cmd/compute/domains/add_test.go new file mode 100644 index 00000000..8c2999e8 --- /dev/null +++ b/internal/cmd/compute/domains/add_test.go @@ -0,0 +1,616 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// unverifiedDomain returns a Domain in the shape the platform reports before +// the user has published the TXT record it asked for. +func unverifiedDomain(domainName, recordName, content string) *networkingv1alpha.Domain { + return &networkingv1alpha.Domain{ + ObjectMeta: metav1.ObjectMeta{ + Name: strings.ReplaceAll(domainName, ".", "-"), + Namespace: util.ResourceNamespace, + }, + Spec: networkingv1alpha.DomainSpec{DomainName: domainName}, + Status: networkingv1alpha.DomainStatus{ + Verification: &networkingv1alpha.DomainVerificationStatus{ + DNSRecord: networkingv1alpha.DNSVerificationRecord{ + Name: recordName, + Type: "TXT", + Content: content, + }, + }, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.DomainConditionVerified, metav1.ConditionFalse, "RecordNotFound", "verification record not found"), + }, + }, + } +} + +// cancelled returns a context that is already done, so a wait runs exactly one +// poll and then takes the detach path. +func cancelled() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} + +func TestAddAttachesHostname(t *testing.T) { + // The platform has already checked this hostname out — the user is + // re-attaching one it knows — so the wait ends on its first poll and the + // command reaches its URL. A hostname the platform has said nothing about + // is pending, and that wait is TestAddPrintsWhatTheServerReports. + proxy := verifiedProxy(testWorkload, testCanonical, testCustom) + proxy.Spec.Hostnames = nil + + c := newFakeClient(t, + proxy, + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), + ) + + var out bytes.Buffer + if err := runAdd(context.Background(), &out, c, testProject, testWorkload, "HTTPS://API.Example.com./"); err != nil { + t.Fatalf("runAdd: %v", err) + } + + // Stored normalized, so the platform sees the name it can validate. + if got := hostnamesOf(t, c); len(got) != 1 || got[0] != testCustom { + t.Errorf("hostnames = %v, want [%s]", got, testCustom) + } + + // The URL is the deliverable, and it is the last thing printed. + if !strings.Contains(out.String(), "https://"+testCustom) { + t.Errorf("missing the URL:\n%s", out.String()) + } +} + +func TestAddIsIdempotent(t *testing.T) { + c := newFakeClient(t, + verifiedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), + ) + + var out bytes.Buffer + if err := runAdd(context.Background(), &out, c, testProject, testWorkload, testCustom); err != nil { + t.Fatalf("runAdd: %v", err) + } + + if got := hostnamesOf(t, c); len(got) != 1 { + t.Errorf("hostnames = %v, want no duplicate", got) + } + if !strings.Contains(out.String(), "already attached") { + t.Errorf("re-adding should say so plainly:\n%s", out.String()) + } + if !strings.Contains(out.String(), "https://"+testCustom) { + t.Errorf("re-adding should still end at the URL:\n%s", out.String()) + } +} + +func TestAddErrors(t *testing.T) { + tests := []struct { + name string + objects []client.Object + workload string + hostname string + want string + }{ + { + name: "the Datum-managed hostname is not a custom domain", + objects: []client.Object{programmedProxy(testWorkload, testCanonical)}, + workload: testWorkload, + hostname: "https://" + strings.ToUpper(testCanonical) + "/", + want: "already the Datum-managed hostname", + }, + { + name: "wildcard", + objects: []client.Object{programmedProxy(testWorkload, testCanonical)}, + workload: testWorkload, + hostname: "*.example.com", + want: "wildcard hostnames are not supported", + }, + { + name: "not a hostname", + objects: []client.Object{programmedProxy(testWorkload, testCanonical)}, + workload: testWorkload, + hostname: "localhost", + want: "not a fully qualified hostname", + }, + { + name: "workload has no URL to attach to", + objects: []client.Object{workload(testWorkload)}, + workload: testWorkload, + hostname: testCustom, + want: wantPublishHint, + }, + { + name: "workload does not exist", + objects: nil, + workload: "ghost", + hostname: testCustom, + want: "not found in project " + testProject, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(t, tc.objects...) + var out bytes.Buffer + err := runAdd(context.Background(), &out, c, testProject, tc.workload, tc.hostname) + if err == nil { + t.Fatalf("expected an error, got output:\n%s", out.String()) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to mention %q", err, tc.want) + } + }) + } +} + +// A hostname that is not live yet prints the records the server asked for, the +// checkmarks for the checks that have passed, and nothing it made up. +func TestAddPrintsWhatTheServerReports(t *testing.T) { + c := newFakeClient(t, + pendingProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), + unverifiedDomain(testApex, "_datum."+testApex, "datum-verify=8f3a91c2b7"), + ) + + var out bytes.Buffer + if err := waitForHostname(cancelled(), &out, c, testWorkload, testCustom); err != nil { + t.Fatalf("waitForHostname: %v", err) + } + got := out.String() + + if !strings.Contains(got, "Verifying "+testApex+"...") { + t.Errorf("should name the domain being verified:\n%s", got) + } + if !strings.Contains(got, "Add these DNS records:") { + t.Errorf("missing the records block:\n%s", got) + } + if f := row(t, got, "TYPE"); strings.Join(f, " ") != "TYPE NAME VALUE" { + t.Errorf("records header = %v", f) + } + if f := row(t, got, "TXT"); strings.Join(f[1:], " ") != "_datum."+testApex+" datum-verify=8f3a91c2b7" { + t.Errorf("TXT row = %v, want the server's own record", f) + } + if f := row(t, got, "CNAME"); strings.Join(f[1:], " ") != testCustom+" "+testCanonical { + t.Errorf("CNAME row = %v, want it to point at the platform hostname", f) + } + + if !strings.Contains(got, "Ctrl-C to detach") { + t.Errorf("should say Ctrl-C is safe:\n%s", got) + } + + // Only the check that passed gets a checkmark. + if !strings.Contains(got, "Domain verified") { + t.Errorf("missing the verified checkmark:\n%s", got) + } + if strings.Contains(got, "Certificate issued") { + t.Errorf("a pending certificate must not be reported as issued:\n%s", got) + } + // And the URL is not offered before it works. + if strings.Contains(got, "\n https://"+testCustom) { + t.Errorf("a hostname that is not live should not be printed as a URL:\n%s", got) + } +} + +func TestAddDetachOnInterruptIsNotAnError(t *testing.T) { + c := newFakeClient(t, + pendingProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), + ) + + var out bytes.Buffer + if err := waitForHostname(cancelled(), &out, c, testWorkload, testCustom); err != nil { + t.Fatalf("detaching returned an error: %v", err) + } + got := out.String() + + if !strings.Contains(got, "Detached.") { + t.Errorf("missing the detach note:\n%s", got) + } + if !strings.Contains(got, "datumctl compute domains") { + t.Errorf("detaching should point at the command that shows the state:\n%s", got) + } +} + +func TestRecordsFor(t *testing.T) { + verified := unverifiedDomain(testApex, "_datum."+testApex, "token") + verified.Status.Conditions = []metav1.Condition{ + cond(networkingv1alpha.DomainConditionVerified, metav1.ConditionTrue, "Verified", ""), + } + + programmedHostname := &url.Hostname{ + Hostname: testCustom, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionDNSRecordProgrammed, metav1.ConditionTrue, "RecordCreated", ""), + }, + } + + tests := []struct { + name string + info *url.Info + hostname *url.Hostname + domain *networkingv1alpha.Domain + want []string + }{ + { + name: "both records outstanding", + info: &url.Info{CanonicalHostname: testCanonical}, + hostname: &url.Hostname{Hostname: testCustom}, + domain: unverifiedDomain(testApex, "_datum."+testApex, "token"), + want: []string{"TXT _datum." + testApex + " token", "CNAME " + testCustom + " " + testCanonical}, + }, + { + name: "an already verified domain needs no TXT", + info: &url.Info{CanonicalHostname: testCanonical}, + hostname: &url.Hostname{Hostname: testCustom}, + domain: verified, + want: []string{"CNAME " + testCustom + " " + testCanonical}, + }, + { + name: "a record the platform programmed itself needs no CNAME", + info: &url.Info{CanonicalHostname: testCanonical}, + hostname: programmedHostname, + domain: verified, + want: nil, + }, + { + name: "no Domain resource yet", + info: &url.Info{CanonicalHostname: testCanonical}, + hostname: &url.Hostname{Hostname: testCustom}, + domain: nil, + want: []string{"CNAME " + testCustom + " " + testCanonical}, + }, + { + name: "no platform hostname assigned yet, so nothing to point at", + info: &url.Info{}, + hostname: &url.Hostname{Hostname: testCustom}, + domain: nil, + want: nil, + }, + { + name: "a Domain with no verification record reported", + info: &url.Info{}, + hostname: &url.Hostname{Hostname: testCustom}, + domain: &networkingv1alpha.Domain{ + Spec: networkingv1alpha.DomainSpec{DomainName: testApex}, + }, + want: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var got []string + for _, r := range recordsFor(tc.info, tc.hostname, tc.domain) { + got = append(got, strings.Join([]string{r.Type, r.Name, r.Value}, " ")) + } + if strings.Join(got, "|") != strings.Join(tc.want, "|") { + t.Errorf("records = %v, want %v", got, tc.want) + } + }) + } +} + +func TestDomainForPicksTheMostSpecific(t *testing.T) { + c := newFakeClient(t, + unverifiedDomain(testApex, "_datum."+testApex, "apex"), + unverifiedDomain(testCustom, "_datum."+testCustom, "sub"), + unverifiedDomain("other.com", "_datum.other.com", "other"), + ) + + got := domainFor(context.Background(), c, "https://API.Example.com./") + if got == nil || got.Spec.DomainName != testCustom { + t.Fatalf("domainFor = %v, want %s", got, testCustom) + } + + if got := domainFor(context.Background(), c, "www."+testApex); got == nil || got.Spec.DomainName != testApex { + t.Errorf("domainFor(www.%s) = %v, want the apex Domain", testApex, got) + } + if got := domainFor(context.Background(), c, "nothing.test"); got != nil { + t.Errorf("domainFor(nothing.test) = %v, want nil", got) + } +} + +func TestCovers(t *testing.T) { + tests := []struct { + domain, hostname string + want bool + }{ + {testApex, testApex, true}, + {testApex, testCustom, true}, + {testApex, "foo." + testCustom, true}, + {testApex, "test-example.com", false}, + {testApex, "notexample.com", false}, + {testCustom, testApex, false}, + } + + for _, tc := range tests { + t.Run(tc.domain+"/"+tc.hostname, func(t *testing.T) { + if got := covers(tc.domain, tc.hostname); got != tc.want { + t.Errorf("covers(%q, %q) = %v, want %v", tc.domain, tc.hostname, got, tc.want) + } + }) + } +} + +func TestHostnameLive(t *testing.T) { + tests := []struct { + name string + h *url.Hostname + want bool + }{ + {"nil", nil, false}, + {"not serving", &url.Hostname{Active: false}, false}, + { + name: "serving, certificate issued", + h: &url.Hostname{Active: true, Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionTrue, "CertificateIssued", ""), + }}, + want: true, + }, + { + name: "serving, certificate still coming", + h: &url.Hostname{Active: true, Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionFalse, "Pending", ""), + }}, + want: false, + }, + { + name: "serving, and the platform said nothing about a certificate", + h: &url.Hostname{Active: true}, + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := hostnameLive(tc.h); got != tc.want { + t.Errorf("hostnameLive = %v, want %v", got, tc.want) + } + }) + } +} + +// ----------------------------------------------------------------------- +// what gets announced, and when +// ----------------------------------------------------------------------- + +// TestAddPrintsNothingBeforeThePlatformHasAnythingToSay: the first poll runs +// moments after the attach, before the platform has created the domain behind +// the hostname or computed the record that proves the user owns it. Announcing +// then prints a records block with the verification record missing from it — +// and the block is printed once, so that record would never reach the user. +func TestAddPrintsNothingBeforeThePlatformHasAnythingToSay(t *testing.T) { + // No Domain object: the platform has not got to this hostname yet. + c := newFakeClient(t, + pendingProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), + ) + + var out bytes.Buffer + if err := waitForHostname(cancelled(), &out, c, testWorkload, testCustom); err != nil { + t.Fatalf("waitForHostname: %v", err) + } + got := out.String() + + for _, unwanted := range []string{"Verifying", "Add these DNS records", "Waiting for DNS", "Domain verified"} { + if strings.Contains(got, unwanted) { + t.Errorf("nothing has been published yet, so nothing should be announced — found %q in:\n%s", unwanted, got) + } + } + if !strings.Contains(got, "Detached.") { + t.Errorf("missing the detach note:\n%s", got) + } +} + +// TestAddPrintsTheVerificationRecordWhenItAppears: the TXT record is the whole +// custom-domain flow. It shows up a poll or two after the attach, and it has to +// be printed when it does — being told to wait for DNS records that were never +// shown is the failure this guards. +func TestAddPrintsTheVerificationRecordWhenItAppears(t *testing.T) { + info := &url.Info{WorkloadName: testWorkload, CanonicalHostname: testCanonical} + h := &url.Hostname{Hostname: testCustom, URL: "https://" + testCustom} + + var out bytes.Buffer + w := &hostnameWatch{out: &out, hostname: testCustom, started: time.Now(), seen: map[string]bool{}} + + // First poll: the platform has not created the Domain yet. + announced, err := w.announce(info, h, nil) + if err != nil { + t.Fatalf("announce: %v", err) + } + if announced { + t.Errorf("announced before the platform had published anything:\n%s", out.String()) + } + if out.Len() != 0 { + t.Errorf("nothing to announce yet, but printed:\n%s", out.String()) + } + + // The platform catches up, and asks for the record it needs. + domain := unverifiedDomain(testApex, "_datum."+testApex, "datum-verify=8f3a91c2b7") + announced, err = w.announce(info, h, domain) + if err != nil { + t.Fatalf("announce: %v", err) + } + if !announced { + t.Fatalf("the records are published — they must be announced:\n%s", out.String()) + } + got := out.String() + + if !strings.Contains(got, "Verifying "+testApex+"...") { + t.Errorf("should name the domain being verified:\n%s", got) + } + if f := row(t, got, "TXT"); strings.Join(f[1:], " ") != "_datum."+testApex+" datum-verify=8f3a91c2b7" { + t.Errorf("TXT row = %v, want the record the platform asked for", f) + } + if f := row(t, got, "CNAME"); strings.Join(f[1:], " ") != testCustom+" "+testCanonical { + t.Errorf("CNAME row = %v", f) + } + if !strings.Contains(got, "Ctrl-C to detach") { + t.Errorf("should say Ctrl-C is safe:\n%s", got) + } + + // And nothing is said twice on the polls that follow. + before := out.Len() + if _, err := w.announce(info, h, domain); err != nil { + t.Fatalf("announce: %v", err) + } + if out.Len() != before { + t.Errorf("announced twice:\n%s", out.String()[before:]) + } +} + +// TestAddSaysWhenNoVerificationRecordArrives: waiting quietly is only defensible +// while the platform is still working. Past that, whatever it did publish is +// printed along with the fact that the rest has not been. +func TestAddSaysWhenNoVerificationRecordArrives(t *testing.T) { + info := &url.Info{WorkloadName: testWorkload, CanonicalHostname: testCanonical} + h := &url.Hostname{Hostname: testCustom, URL: "https://" + testCustom} + + var out bytes.Buffer + w := &hostnameWatch{ + out: &out, + hostname: testCustom, + started: time.Now().Add(-2 * recordWindow), + seen: map[string]bool{}, + } + + announced, err := w.announce(info, h, nil) + if err != nil { + t.Fatalf("announce: %v", err) + } + if !announced { + t.Fatalf("there is a record to print, so it must be printed:\n%s", out.String()) + } + got := out.String() + + if f := row(t, got, "CNAME"); strings.Join(f[1:], " ") != testCustom+" "+testCanonical { + t.Errorf("CNAME row = %v, want what the platform did publish", f) + } + if !strings.Contains(got, "has not published a verification record") { + t.Errorf("a missing verification record must be said out loud:\n%s", got) + } + + // And when it finally turns up, it is still printed. + domain := unverifiedDomain(testApex, "_datum."+testApex, "late") + if _, err := w.announce(info, h, domain); err != nil { + t.Fatalf("announce: %v", err) + } + if f := row(t, out.String(), "TXT"); strings.Join(f[1:], " ") != "_datum."+testApex+" late" { + t.Errorf("TXT row = %v, want the record printed once it exists", f) + } +} + +// TestAddStopsWhenThePlatformPublishesNothing: a wait with nothing to wait for +// and nothing to show is a hang. It ends, and says why. +func TestAddStopsWhenThePlatformPublishesNothing(t *testing.T) { + // No managed hostname to point at, and no Domain: there is no record this + // command could print, so there is nothing for the user to do. + info := &url.Info{WorkloadName: testWorkload} + h := &url.Hostname{Hostname: testCustom, URL: "https://" + testCustom} + + var out bytes.Buffer + w := &hostnameWatch{ + out: &out, + hostname: testCustom, + started: time.Now().Add(-2 * recordWindow), + seen: map[string]bool{}, + } + + announced, err := w.announce(info, h, nil) + if err == nil { + t.Fatalf("a wait with nothing to wait for must end, got output:\n%s", out.String()) + } + if announced { + t.Errorf("nothing was published, so nothing was announced") + } + for _, want := range []string{testCustom, testWorkload, "datumctl compute domains"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to mention %q", err, want) + } + } +} + +// TestAddWaitFailsWhenNothingIsEverPublished: and the failure reaches the +// command, rather than being swallowed by the poll loop. +func TestAddWaitFailsWhenNothingIsEverPublished(t *testing.T) { + proxy := programmedProxy(testWorkload, "", testCustom) + c := newFakeClient(t, proxy, serviceWith(testWorkload, 8080, location("DFW", 1, 1, true))) + + w := &hostnameWatch{ + out: io.Discard, + hostname: testCustom, + started: time.Now().Add(-2 * recordWindow), + seen: map[string]bool{}, + } + done, err := w.check(context.Background(), c, testWorkload) + if err == nil { + t.Fatal("the poll loop must surface a wait that can never finish") + } + if done { + t.Error("the hostname is not live") + } +} + +// ----------------------------------------------------------------------- +// how many custom hostnames a workload may have +// ----------------------------------------------------------------------- + +// TestAddAllowsSeveralCustomHostnames: principle 6 is one workload, one URL — +// one routing target, not a cap on names. Every comparable CLI appends, the +// Datum-managed hostname already sits alongside whatever the user attached, and +// replacing a production hostname because someone added a staging alias would +// be a surprise no platform in the table commits. So `add` appends, and says +// what the workload serves now. +func TestAddAllowsSeveralCustomHostnames(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + if err := runAdd(cancelled(), &out, c, testProject, testWorkload, "www."+testApex); err != nil { + t.Fatalf("runAdd: %v", err) + } + got := out.String() + + if want := testCustom + ",www." + testApex; strings.Join(hostnamesOf(t, c), ",") != want { + t.Fatalf("hostnames = %v, want %q — the first one is not replaced", hostnamesOf(t, c), want) + } + if !strings.Contains(got, "www."+testApex+" attached to workload") { + t.Errorf("attaching must say it attached:\n%s", got) + } + if !strings.Contains(got, "2 custom hostnames") { + t.Errorf("a workload serving more than one hostname must say so:\n%s", got) + } +} + +// TestAddHelpAgreesThatSeveralHostnamesAreAllowed: the behavior above is only +// coherent if the help says it. A user who expects `add` to replace has to be +// told otherwise before they run it, not after. +func TestAddHelpAgreesThatSeveralHostnamesAreAllowed(t *testing.T) { + long := addCommand().Long + + for _, want := range []string{"several custom hostnames", "never replaces", "domains remove"} { + if !strings.Contains(long, want) { + t.Errorf("help must mention %q:\n%s", want, long) + } + } +} diff --git a/internal/cmd/compute/domains/domains.go b/internal/cmd/compute/domains/domains.go new file mode 100644 index 00000000..4bb79094 --- /dev/null +++ b/internal/cmd/compute/domains/domains.go @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package domains implements `datumctl compute domains`: the project-wide list +// of every hostname a workload answers on, the per-URL detail view a developer +// reaches for when a URL is not working, and the two commands that attach and +// detach a custom hostname. +// +// Everything here reads and writes through the url package, which owns how a +// workload becomes a URL. Nothing here names the machinery: the vocabulary is +// "domain", "certificate", "backends". +package domains + +import ( + "context" + "fmt" + "io" + "sort" + + "github.com/spf13/cobra" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" +) + +// managedMarker flags the hostname the platform assigned. It is the one row in +// the list a user cannot act on, so it says so on the row rather than only in +// the error they would get if they tried. +const managedMarker = "(Datum-managed)" + +// dash is what an unreported value renders as, so a column is never blank. +const dash = "—" + +// Command returns the "domains" command group. +func Command() *cobra.Command { + var output string + var noHeaders bool + + cmd := &cobra.Command{ + Use: "domains [workload]", + Short: "List the domains in a project, or show one workload's URL in detail", + Long: "List every domain in the project, or pass a workload to see its URL in detail.\n\n" + + "The detail view is the one to reach for when a URL is not answering: it shows\n" + + "every hostname the workload serves, its certificate, and how many backends are\n" + + "healthy in each city.\n\n" + + "Every workload that declares an HTTP port gets a permanent Datum-managed domain.\n" + + "It always appears in this list and cannot be removed.", + Example: " # Every domain in the project\n" + + " datumctl compute domains\n\n" + + " # One workload's URL, in detail\n" + + " datumctl compute domains api\n\n" + + " # Attach a custom hostname\n" + + " datumctl compute domains add api api.example.com\n\n" + + " # Machine-readable output\n" + + " datumctl compute domains -o json", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := util.NewClient(util.ProjectFromCmd(cmd)) + if err != nil { + return err + } + format := util.OutputFormat(output) + if len(args) == 1 { + return runDetail(cmd.Context(), cmd.OutOrStdout(), c, util.ProjectFromCmd(cmd), args[0], format) + } + return runList(cmd.Context(), cmd.OutOrStdout(), c, util.ProjectFromCmd(cmd), format, noHeaders) + }, + ValidArgsFunction: util.CompleteWorkloadNames, + } + + cmd.Flags().StringVarP(&output, "output", "o", "table", "Output format: table, json, yaml") + cmd.Flags().BoolVar(&noHeaders, "no-headers", false, "Omit the table header row (table only)") + _ = cmd.RegisterFlagCompletionFunc("output", util.CompleteOutputFormats("table", "json", "yaml")) + + cmd.AddCommand(addCommand(), removeCommand()) + + return cmd +} + +// ----------------------------------------------------------------------- +// domains — the project-wide list +// ----------------------------------------------------------------------- + +// listRow is one hostname in the project-wide list. It is also the JSON and +// YAML shape, so `-o json | jq -r '.[].domain'` is a list of hostnames and +// nothing more needs to be parsed out of a table. +type listRow struct { + // Workload is the workload serving this domain. + Workload string `json:"workload"` + // Domain is the bare hostname; URL is the same thing as an https:// URL. + Domain string `json:"domain"` + URL string `json:"url"` + // Managed is true for the permanent hostname the platform assigned, which + // cannot be removed. + Managed bool `json:"managed"` + // Status is "active", or whatever the server says is blocking it. + Status string `json:"status"` + // Certificate is "valid", or the server's reason, or empty when the + // platform has not reported on one. + Certificate string `json:"certificate,omitempty"` + // Detail is the server's message for the first blocking condition. + Detail string `json:"detail,omitempty"` +} + +// runList renders every domain in the project. It costs two List calls in +// total, however many workloads there are, because url.ForAll does. +func runList(ctx context.Context, out io.Writer, c client.Client, project string, format util.OutputFormat, noHeaders bool) error { + if ctx == nil { + ctx = context.Background() + } + + infos, err := url.ForAll(ctx, c) + if err != nil { + return err + } + rows := listRows(infos) + + switch format { + case util.OutputJSON: + return util.PrintJSON(out, rows) + case util.OutputYAML: + return util.PrintYAML(out, rows) + } + + renderList(out, rows, project, noHeaders) + return nil +} + +// listRows flattens the per-workload URL info into one row per hostname, +// ordered by workload so the list is stable between runs. Within a workload +// the custom hostnames come first and the managed one last, which is the order +// url.Info already carries: the name the user chose is the one they are +// looking for. +func listRows(infos map[string]*url.Info) []listRow { + names := make([]string, 0, len(infos)) + for name := range infos { + names = append(names, name) + } + sort.Strings(names) + + var rows []listRow + for _, name := range names { + for _, h := range infos[name].Hostnames { + rows = append(rows, listRow{ + Workload: name, + Domain: h.Hostname, + URL: h.URL, + Managed: h.Managed, + Status: h.Status, + Certificate: h.Certificate, + Detail: h.Detail, + }) + } + } + return rows +} + +// renderList writes the table. A project with no domains gets a sentence that +// says how to get one rather than an empty table. +func renderList(out io.Writer, rows []listRow, project string, noHeaders bool) { + fmt.Fprintln(out) + + if len(rows) == 0 { + fmt.Fprintf(out, "No domains in project %s.\n\n", project) + fmt.Fprintln(out, " A workload gets a permanent URL when it declares an HTTP port:") + fmt.Fprintln(out, " datumctl compute deploy --image= --http-port 8080") + return + } + + tw := util.NewTabWriter(out) + if !noHeaders { + fmt.Fprintln(tw, " WORKLOAD\tDOMAIN\tSTATUS\tCERTIFICATE") + } + for _, r := range rows { + certificate := r.Certificate + if certificate == "" { + certificate = dash + } + // The marker rides on the end of the last column rather than forming a + // column of its own: it is an annotation on the row, and a column of + // mostly-blank cells would pad every other row out to its width. + if r.Managed { + certificate += " " + managedMarker + } + fmt.Fprintf(tw, " %s\t%s\t%s\t%s\n", r.Workload, r.Domain, r.Status, certificate) + } + _ = tw.Flush() +} + +// ----------------------------------------------------------------------- +// domains — the per-URL detail view +// ----------------------------------------------------------------------- + +// runDetail renders one workload's URL: its hostnames, its certificate, and +// the per-city backend table that makes a multi-city deployment legible. +func runDetail(ctx context.Context, out io.Writer, c client.Client, project, workloadName string, format util.OutputFormat) error { + if ctx == nil { + ctx = context.Background() + } + + info, err := url.ForWorkload(ctx, c, workloadName) + if err != nil { + return err + } + if info == nil { + return notPublished(ctx, c, project, workloadName) + } + + switch format { + case util.OutputJSON: + // JSON is the scripting view: the summary a caller wants to read a URL + // or a health count out of. + return util.PrintJSON(out, info) + case util.OutputYAML: + // YAML is the escape hatch the spec promises — the real platform + // objects, for when the summary is not enough to debug with. + return util.PrintYAML(out, info.Objects()) + } + + fmt.Fprintln(out) + url.RenderDetail(out, info) + return nil +} + +// notPublished explains that a workload has no URL. A workload that does not +// exist and a workload that exists but never declared an HTTP port are +// different problems with different fixes, so they get different errors. +func notPublished(ctx context.Context, c client.Client, project, workloadName string) error { + var workload computev1alpha.Workload + err := c.Get(ctx, types.NamespacedName{Namespace: util.ResourceNamespace, Name: workloadName}, &workload) + if k8serrors.IsNotFound(err) { + return fmt.Errorf("workload %q not found in project %s — run 'datumctl compute workloads' to see what is deployed", workloadName, project) + } + return fmt.Errorf( + "workload %q has no URL — it does not declare an HTTP port.\nTo publish it: datumctl compute deploy %s --http-port ", + workloadName, workloadName, + ) +} + +// urlFor loads a workload's URL for a command that is about to change it, +// turning "no URL" into the error that says how to get one. Both add and +// remove need exactly this. +func urlFor(ctx context.Context, c client.Client, project, workloadName string) (*url.Info, error) { + info, err := url.ForWorkload(ctx, c, workloadName) + if err != nil { + return nil, err + } + if info == nil || info.Proxy == nil { + return nil, notPublished(ctx, c, project, workloadName) + } + return info, nil +} diff --git a/internal/cmd/compute/domains/domains_test.go b/internal/cmd/compute/domains/domains_test.go new file mode 100644 index 00000000..e1a6fbd5 --- /dev/null +++ b/internal/cmd/compute/domains/domains_test.go @@ -0,0 +1,449 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "bytes" + "context" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// Fixtures shared by the tests in this package. +const ( + testProject = "acme-prod" + testWorkload = "api" + testPortName = "http" + + testCanonical = "a1b2c3d4.datumproxy.net" + testApex = "example.com" + testCustom = "api." + testApex + + // wantPublishHint is what an unpublished workload's error has to point at: + // the flag that gives it a URL in the first place. + wantPublishHint = "--http-port" + + // wantNoHostname is the error for a command called without a hostname. + wantNoHostname = "no hostname given" +) + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := computev1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering compute scheme: %v", err) + } + if err := networkingv1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering networking scheme: %v", err) + } + return s +} + +func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithStatusSubresource(&networkingv1alpha.HTTPProxy{}, &networkingv1alpha.NetworkService{}, &networkingv1alpha.Domain{}). + WithObjects(objs...). + Build() +} + +func workload(name string) *computev1alpha.Workload { + return &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: util.ResourceNamespace, + UID: types.UID("uid-" + name), + }, + } +} + +func cond(condType string, status metav1.ConditionStatus, reason, message string) metav1.Condition { + return metav1.Condition{Type: condType, Status: status, Reason: reason, Message: message} +} + +// programmedProxy returns a proxy in the shape the platform reports once it is +// serving: accepted, programmed, and holding a certificate. +func programmedProxy(name, canonical string, hostnames ...string) *networkingv1alpha.HTTPProxy { + p := url.BuildHTTPProxy(workload(name), testPortName, hostnames) + p.Status.CanonicalHostname = canonical + p.Status.Conditions = []metav1.Condition{ + cond(networkingv1alpha.HTTPProxyConditionAccepted, metav1.ConditionTrue, "Accepted", ""), + cond(networkingv1alpha.HTTPProxyConditionProgrammed, metav1.ConditionTrue, "Programmed", ""), + cond(networkingv1alpha.HTTPProxyConditionCertificatesReady, metav1.ConditionTrue, "AllCertificatesReady", ""), + } + return p +} + +// verifiedProxy returns a proxy whose custom hostname the platform has already +// checked and is serving: verified, its DNS record programmed, its certificate +// issued. A hostname is only ever active on the strength of its own entry +// here — the proxy-level conditions are a roll-up over every hostname and +// cannot vouch for any one of them. +func verifiedProxy(name, canonical, hostname string) *networkingv1alpha.HTTPProxy { + p := programmedProxy(name, canonical, hostname) + p.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: hostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionTrue, "Verified", ""), + cond(networkingv1alpha.HostnameConditionAvailable, metav1.ConditionTrue, "Claimed", ""), + cond(networkingv1alpha.HostnameConditionDNSRecordProgrammed, metav1.ConditionTrue, "RecordCreated", ""), + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionTrue, "CertificateIssued", ""), + }, + }} + return p +} + +// pendingProxy returns a proxy whose custom hostname is still waiting on DNS. +func pendingProxy(name, canonical, hostname string) *networkingv1alpha.HTTPProxy { + p := programmedProxy(name, canonical, hostname) + p.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: hostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionTrue, "Verified", ""), + cond(networkingv1alpha.HostnameConditionDNSRecordProgrammed, metav1.ConditionFalse, "Pending", "waiting for DNS"), + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionFalse, "ChallengeInProgress", "ACME challenge in progress"), + }, + }} + return p +} + +func serviceWith(name string, port int32, locations ...networkingv1alpha.NetworkServiceLocationStatus) *networkingv1alpha.NetworkService { + s := url.BuildNetworkService(workload(name), testPortName, port) + var members, healthy int32 + for _, l := range locations { + members += l.Members + healthy += l.Healthy + } + s.Status.Summary = networkingv1alpha.NetworkServiceSummary{ + Locations: int32(len(locations)), + Members: members, + Healthy: healthy, + } + s.Status.Locations = locations + s.Status.Conditions = []metav1.Condition{ + cond(networkingv1alpha.NetworkServiceMembersResolved, metav1.ConditionTrue, "MembersResolved", ""), + cond(networkingv1alpha.NetworkServiceReady, metav1.ConditionTrue, "Ready", ""), + } + return s +} + +func location(city string, members, healthy int32, serving bool) networkingv1alpha.NetworkServiceLocationStatus { + return networkingv1alpha.NetworkServiceLocationStatus{Name: city, Members: members, Healthy: healthy, Serving: serving} +} + +// row returns the whitespace-separated fields of the first line whose fields +// begin with the given ones, so assertions never depend on column widths. +func row(t *testing.T, output string, first ...string) []string { + t.Helper() + for _, line := range strings.Split(output, "\n") { + fields := strings.Fields(line) + if len(fields) < len(first) { + continue + } + match := true + for i, want := range first { + if fields[i] != want { + match = false + break + } + } + if match { + return fields + } + } + t.Fatalf("no line starting with %v in:\n%s", first, output) + return nil +} + +// ----------------------------------------------------------------------- +// the list view +// ----------------------------------------------------------------------- + +func TestListRendersEveryHostname(t *testing.T) { + c := newFakeClient(t, + verifiedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), + pendingProxy("web", "e5f6a7b8.datumproxy.net", "www.acme.io"), + serviceWith("web", 3000, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + if err := runList(context.Background(), &out, c, testProject, util.OutputTable, false); err != nil { + t.Fatalf("runList: %v", err) + } + got := out.String() + + if f := row(t, got, "WORKLOAD"); strings.Join(f, " ") != "WORKLOAD DOMAIN STATUS CERTIFICATE" { + t.Errorf("header = %v", f) + } + + // The custom hostname comes before the managed one for the same workload: + // the name the user chose is the one they are looking for. + custom := row(t, got, testWorkload, testCustom) + if strings.Join(custom[2:], " ") != "active valid" { + t.Errorf("custom row = %v, want active valid", custom) + } + if strings.Contains(strings.Join(custom, " "), managedMarker) { + t.Errorf("custom hostname marked as Datum-managed: %v", custom) + } + + managedRow := row(t, got, testWorkload, testCanonical) + if strings.Join(managedRow[2:], " ") != "active valid "+managedMarker { + t.Errorf("managed row = %v, want the %s marker", managedRow, managedMarker) + } + + // The server's own words for a hostname that is not serving yet: its + // messages, never the camelCase reasons behind them. + pending := strings.Join(row(t, got, "web", "www.acme.io")[2:], " ") + if pending != "waiting for DNS ACME challenge in progress" { + t.Errorf("pending row = %q, want the server's messages verbatim", pending) + } + if strings.Contains(got, "ChallengeInProgress") { + t.Errorf("a raw condition reason reached the table:\n%s", got) + } + + // Workloads are ordered, so two runs read the same. + if strings.Index(got, "\n api ") > strings.Index(got, "\n web ") { + t.Errorf("workloads out of order:\n%s", got) + } +} + +func TestListManagedHostnameAlwaysAppears(t *testing.T) { + // A workload with no custom hostname still has a domain: the managed one. + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + if err := runList(context.Background(), &out, c, testProject, util.OutputTable, false); err != nil { + t.Fatalf("runList: %v", err) + } + + f := row(t, out.String(), testWorkload, testCanonical) + if f[len(f)-1] != managedMarker { + t.Errorf("row = %v, want it marked %s", f, managedMarker) + } +} + +func TestListEmpty(t *testing.T) { + c := newFakeClient(t) + + var out bytes.Buffer + if err := runList(context.Background(), &out, c, testProject, util.OutputTable, false); err != nil { + t.Fatalf("runList: %v", err) + } + got := out.String() + + if !strings.Contains(got, "No domains in project "+testProject) { + t.Errorf("missing the empty-state sentence:\n%s", got) + } + if !strings.Contains(got, "--http-port") { + t.Errorf("empty state should say how to get a domain:\n%s", got) + } + if strings.Contains(got, "WORKLOAD") { + t.Errorf("empty state should not print a header row:\n%s", got) + } +} + +func TestListNoHeaders(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + if err := runList(context.Background(), &out, c, testProject, util.OutputTable, true); err != nil { + t.Fatalf("runList: %v", err) + } + if strings.Contains(out.String(), "WORKLOAD") { + t.Errorf("--no-headers still printed a header:\n%s", out.String()) + } +} + +func TestListJSONIsScriptable(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), + ) + + var out bytes.Buffer + if err := runList(context.Background(), &out, c, testProject, util.OutputJSON, false); err != nil { + t.Fatalf("runList: %v", err) + } + got := out.String() + + for _, want := range []string{`"domain": "` + testCustom + `"`, `"url": "https://` + testCustom + `"`, `"managed": true`} { + if !strings.Contains(got, want) { + t.Errorf("JSON missing %s:\n%s", want, got) + } + } +} + +func TestListRowsOrderCustomBeforeManaged(t *testing.T) { + rows := listRows(map[string]*url.Info{ + "web": {Hostnames: []url.Hostname{{Hostname: "www.acme.io"}}}, + "api": {Hostnames: []url.Hostname{{Hostname: testCustom}, {Hostname: testCanonical, Managed: true}}}, + }) + + got := make([]string, 0, len(rows)) + for _, r := range rows { + got = append(got, r.Workload+"/"+r.Domain) + } + want := []string{"api/" + testCustom, "api/" + testCanonical, "web/www.acme.io"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("rows = %v, want %v", got, want) + } +} + +// ----------------------------------------------------------------------- +// the detail view +// ----------------------------------------------------------------------- + +func TestDetailDegradedAcrossCities(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true), location("IAD", 2, 0, false)), + ) + + var out bytes.Buffer + if err := runDetail(context.Background(), &out, c, testProject, testWorkload, util.OutputTable); err != nil { + t.Fatalf("runDetail: %v", err) + } + got := out.String() + + if !strings.Contains(got, "https://"+testCustom) || !strings.Contains(got, "https://"+testCanonical) { + t.Errorf("detail view should list both hostnames:\n%s", got) + } + if f := row(t, got, "Backend"); strings.Join(f[1:], " ") != "port 8080/tcp" { + t.Errorf("backend row = %v", f) + } + if !strings.Contains(got, "Degraded — 2 of 4 backends healthy") { + t.Errorf("missing the health summary:\n%s", got) + } + + if f := row(t, got, "CITY"); strings.Join(f, " ") != "CITY BACKENDS HEALTHY SERVING" { + t.Errorf("per-city header = %v", f) + } + if f := row(t, got, "DFW"); strings.Join(f[1:], " ") != "2 2 yes" { + t.Errorf("DFW row = %v", f) + } + if f := row(t, got, "IAD"); strings.Join(f[1:], " ") != "2 0 no" { + t.Errorf("IAD row = %v", f) + } + + if !strings.Contains(got, "IAD: no healthy backends") { + t.Errorf("missing the degraded narrative:\n%s", got) + } + if !strings.Contains(got, "Next steps:") || + !strings.Contains(got, "datumctl compute instances --workload=api --city=IAD") { + t.Errorf("missing the next-steps block:\n%s", got) + } +} + +func TestDetailHealthyHasNoDiagnosis(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical), + serviceWith(testWorkload, 8080, location("DFW", 2, 2, true), location("IAD", 2, 2, true)), + ) + + var out bytes.Buffer + if err := runDetail(context.Background(), &out, c, testProject, testWorkload, util.OutputTable); err != nil { + t.Fatalf("runDetail: %v", err) + } + got := out.String() + + if !strings.Contains(got, "Healthy — 4 of 4 backends healthy") { + t.Errorf("missing the health summary:\n%s", got) + } + if strings.Contains(got, "Next steps:") { + t.Errorf("a healthy URL should not suggest a fix:\n%s", got) + } +} + +func TestDetailJSON(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + if err := runDetail(context.Background(), &out, c, testProject, testWorkload, util.OutputJSON); err != nil { + t.Fatalf("runDetail: %v", err) + } + if !strings.Contains(out.String(), `"url": "https://`+testCanonical+`"`) { + t.Errorf("JSON detail missing the URL:\n%s", out.String()) + } +} + +func TestDetailErrors(t *testing.T) { + tests := []struct { + name string + objects []client.Object + workload string + want string + }{ + { + name: "workload exists but was never published", + objects: []client.Object{workload(testWorkload)}, + workload: testWorkload, + want: wantPublishHint, + }, + { + name: "workload does not exist", + objects: nil, + workload: "ghost", + want: "not found in project " + testProject, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(t, tc.objects...) + var out bytes.Buffer + err := runDetail(context.Background(), &out, c, testProject, tc.workload, util.OutputTable) + if err == nil { + t.Fatalf("expected an error, got output:\n%s", out.String()) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to mention %q", err, tc.want) + } + }) + } +} + +// hostnamesOf reads back what a proxy declares, for mutation assertions. +func hostnamesOf(t *testing.T, c client.Client) []string { + t.Helper() + var proxy networkingv1alpha.HTTPProxy + key := types.NamespacedName{Namespace: util.ResourceNamespace, Name: testWorkload} + if err := c.Get(context.Background(), key, &proxy); err != nil { + t.Fatalf("reading the URL for workload %q: %v", testWorkload, err) + } + out := make([]string, 0, len(proxy.Spec.Hostnames)) + for _, h := range proxy.Spec.Hostnames { + out = append(out, string(h)) + } + return out +} + +func hostnameList(hostnames ...string) []gatewayv1.Hostname { + out := make([]gatewayv1.Hostname, 0, len(hostnames)) + for _, h := range hostnames { + out = append(out, gatewayv1.Hostname(h)) + } + return out +} diff --git a/internal/cmd/compute/domains/hostname.go b/internal/cmd/compute/domains/hostname.go new file mode 100644 index 00000000..4795c220 --- /dev/null +++ b/internal/cmd/compute/domains/hostname.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "fmt" + "net" + "regexp" + "strings" +) + +// maxHostnameLength is the length the API allows for a domain name. +const maxHostnameLength = 253 + +// hostnamePattern is the shape the platform stores a hostname in: lowercase +// RFC 1123 labels separated by dots. It matches the Domain API's own pattern, +// so a hostname this accepts is one the server will accept too. +var hostnamePattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`) + +// normalizeHostname reduces what a user might type or paste to the bare +// hostname the platform stores. +// +// People copy a domain out of a browser bar, out of a dig output, or out of +// this CLI's own output, so "https://API.Example.com/health", "api.example.com." +// and "api.example.com:443" all reach us meaning the same name. Comparisons — +// above all the one that refuses to remove the Datum-managed hostname — are +// made on this form, never on the raw string. +func normalizeHostname(raw string) string { + h := strings.TrimSpace(raw) + + // A pasted URL: drop the scheme, then everything after the authority. + if i := strings.Index(h, "://"); i >= 0 { + h = h[i+len("://"):] + } + if i := strings.IndexAny(h, "/?#"); i >= 0 { + h = h[:i] + } + if i := strings.LastIndex(h, "@"); i >= 0 { + h = h[i+1:] + } + // A port, but only when there is one colon to be sure it is a port: an + // IPv6 literal has several, and must survive intact to be rejected as an + // address rather than mangled into something that looks like a bad name. + if strings.HasPrefix(h, "[") { + if i := strings.Index(h, "]"); i >= 0 { + h = h[1:i] + } + } else if i := strings.LastIndex(h, ":"); i >= 0 && isDigits(h[i+1:]) && !strings.Contains(h[:i], ":") { + h = h[:i] + } + + // A trailing dot is the same name, spelled absolutely. + h = strings.TrimRight(h, ".") + + return strings.ToLower(h) +} + +// sameHostname reports whether two hostnames name the same host, whatever form +// each was written in. An empty hostname matches nothing, so an unassigned +// managed hostname never swallows a user's argument. +func sameHostname(a, b string) bool { + na, nb := normalizeHostname(a), normalizeHostname(b) + return na != "" && na == nb +} + +// validateHostname normalizes a hostname and reports why it cannot be used, in +// terms of what the user typed rather than the field the API would complain +// about. +func validateHostname(raw string) (string, error) { + h := normalizeHostname(raw) + + switch { + case h == "": + return "", fmt.Errorf("no hostname given — pass the hostname to attach, for example api.example.com") + case strings.Contains(h, "*"): + return "", fmt.Errorf("wildcard hostnames are not supported — attach each hostname you want to serve") + case len(h) > maxHostnameLength: + return "", fmt.Errorf("hostname %q is longer than %d characters", h, maxHostnameLength) + case net.ParseIP(h) != nil: + return "", fmt.Errorf("%q is an IP address — a domain must be a hostname, for example api.example.com", h) + case !strings.Contains(h, "."): + return "", fmt.Errorf("%q is not a fully qualified hostname — use something like api.example.com", h) + case !hostnamePattern.MatchString(h): + return "", fmt.Errorf("%q is not a valid hostname — use letters, digits, dashes and dots", h) + } + + return h, nil +} + +func isDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/internal/cmd/compute/domains/hostname_test.go b/internal/cmd/compute/domains/hostname_test.go new file mode 100644 index 00000000..c714718b --- /dev/null +++ b/internal/cmd/compute/domains/hostname_test.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "strings" + "testing" +) + +func TestNormalizeHostname(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"already normal", testCustom, testCustom}, + {"upper case", "API.Example.COM", testCustom}, + {"one trailing dot", testCustom + ".", testCustom}, + {"several trailing dots", testCustom + "..", testCustom}, + {"surrounding space", " " + testCustom + " \n", testCustom}, + {"https prefix", "https://" + testCustom, testCustom}, + {"http prefix", "http://" + testCustom, testCustom}, + {"url with path", "https://" + testCustom + "/healthz", testCustom}, + {"url with query", "https://" + testCustom + "?x=1", testCustom}, + {"url with fragment", "https://" + testCustom + "#top", testCustom}, + {"url with port", "https://" + testCustom + ":443/", testCustom}, + {"bare host and port", testCustom + ":8443", testCustom}, + {"credentials", "https://user:pw@" + testCustom + "/", testCustom}, + {"everything at once", " HTTPS://API.Example.com.:443/health?x=1 ", testCustom}, + {"nothing at all", "", ""}, + {"only a dot", ".", ""}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := normalizeHostname(tc.in); got != tc.want { + t.Errorf("normalizeHostname(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +func TestSameHostname(t *testing.T) { + tests := []struct { + name string + a, b string + want bool + }{ + {"identical", testCanonical, testCanonical, true}, + {"case differs", "A1B2C3D4.DatumProxy.NET", testCanonical, true}, + {"one trailing dot", testCanonical + ".", testCanonical, true}, + {"scheme pasted", "https://" + testCanonical, testCanonical, true}, + {"scheme and slash pasted", "https://" + testCanonical + "/", testCanonical, true}, + {"different hosts", testCustom, testCanonical, false}, + {"one empty", "", testCanonical, false}, + {"both empty", "", "", false}, + {"suffix is not the same host", "evil-" + testCanonical, testCanonical, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sameHostname(tc.a, tc.b); got != tc.want { + t.Errorf("sameHostname(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + }) + } +} + +func TestValidateHostname(t *testing.T) { + tests := []struct { + name string + in string + want string + wantErr string + }{ + {name: "plain", in: testCustom, want: testCustom}, + {name: "normalized on the way through", in: "HTTPS://API.Example.com./", want: testCustom}, + {name: "only whitespace", in: " ", wantErr: wantNoHostname}, + {name: "wildcard", in: "*.example.com", wantErr: "wildcard hostnames are not supported"}, + {name: "ipv4", in: "192.0.2.10", wantErr: "is an IP address"}, + {name: "ipv6", in: "2001:db8::1", wantErr: "is an IP address"}, + {name: "single label", in: "localhost", wantErr: "not a fully qualified hostname"}, + {name: "underscore", in: "api_v2.example.com", wantErr: "not a valid hostname"}, + {name: "leading dash label", in: "-api.example.com", wantErr: "not a valid hostname"}, + {name: "too long", in: strings.Repeat("a", 250) + ".example.com", wantErr: "longer than 253"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := validateHostname(tc.in) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("validateHostname(%q) = %q, want an error", tc.in, got) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("error = %q, want it to mention %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("validateHostname(%q): %v", tc.in, err) + } + if got != tc.want { + t.Errorf("validateHostname(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} diff --git a/internal/cmd/compute/domains/remove.go b/internal/cmd/compute/domains/remove.go new file mode 100644 index 00000000..5f9349ac --- /dev/null +++ b/internal/cmd/compute/domains/remove.go @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" + "golang.org/x/term" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" +) + +func removeCommand() *cobra.Command { + var yes bool + + cmd := &cobra.Command{ + Use: "remove ", + Short: "Detach a custom hostname from a workload", + Long: "Detach a custom hostname from a workload's URL.\n\n" + + "The workload keeps serving on its Datum-managed hostname, which is permanent\n" + + "and cannot be removed.\n\n" + + "The domain itself is left alone: it stays verified in this project and can be\n" + + "attached to this or another workload without verifying it again.", + Example: " # Detach a hostname\n" + + " datumctl compute domains remove api api.example.com\n\n" + + " # Without the confirmation prompt\n" + + " datumctl compute domains remove api api.example.com --yes", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := util.NewClient(util.ProjectFromCmd(cmd)) + if err != nil { + return err + } + return runRemove(cmd.Context(), cmd.OutOrStdout(), c, util.ProjectFromCmd(cmd), args[0], args[1], yes) + }, + ValidArgsFunction: completeRemoveArgs, + } + + cmd.Flags().BoolVarP(&yes, "yes", "y", false, "Skip the confirmation prompt") + + return cmd +} + +// runRemove detaches a hostname from a workload's URL. +// +// The Datum-managed hostname is refused: it is the workload's permanent +// address and the fallback a user checks when a custom domain is misbehaving. +// The check is made on the normalized form of both names, so pasting +// "https://A1B2C3D4.DatumProxy.net/" back out of a browser is refused just as +// plainly as typing the hostname exactly. +func runRemove(ctx context.Context, out io.Writer, c client.Client, project, workloadName, rawHostname string, yes bool) error { + if ctx == nil { + ctx = context.Background() + } + + hostname := normalizeHostname(rawHostname) + if hostname == "" { + return fmt.Errorf("no hostname given — pass the hostname to detach, for example api.example.com") + } + + info, err := urlFor(ctx, c, project, workloadName) + if err != nil { + return err + } + + if managed(info, hostname) { + return fmt.Errorf( + "%s is the Datum-managed domain for workload %q and cannot be removed — it is permanent, and stops serving only when the workload stops declaring an HTTP port", + hostname, workloadName, + ) + } + + index := indexOf(info.Proxy.Spec.Hostnames, hostname) + if index < 0 { + return notAttached(info, workloadName, hostname) + } + + // The stored spelling, not the user's: the confirmation and the summary + // should show what is actually on the workload. + attachedAs := string(info.Proxy.Spec.Hostnames[index]) + + ok, err := confirm(out, fmt.Sprintf( + "Remove %s from workload %q? It will stop serving on that hostname. (y/N): ", attachedAs, workloadName), yes) + if err != nil { + return err + } + if !ok { + fmt.Fprintln(out, "Aborted.") + return nil + } + + proxy := info.Proxy.DeepCopy() + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames[:index], proxy.Spec.Hostnames[index+1:]...) + if err := c.Update(ctx, proxy); err != nil { + return fmt.Errorf("detaching %s from workload %q: %w", attachedAs, workloadName, err) + } + + fmt.Fprintf(out, " %s removed from workload %q.\n", attachedAs, workloadName) + if info.CanonicalHostname != "" { + fmt.Fprintf(out, " Still serving on https://%s\n", info.CanonicalHostname) + } + fmt.Fprintln(out, "\n The domain stays verified in this project. Attach it again with:") + fmt.Fprintf(out, " datumctl compute domains add %s %s\n", workloadName, attachedAs) + + return nil +} + +// managed reports whether a hostname is the one the platform assigned. Both the +// status field and the parsed hostname list are checked, so the refusal does +// not depend on which of the two the server filled in. +func managed(info *url.Info, hostname string) bool { + if sameHostname(info.CanonicalHostname, hostname) { + return true + } + for _, h := range info.Hostnames { + if h.Managed && sameHostname(h.Hostname, hostname) { + return true + } + } + return false +} + +// indexOf finds a hostname in the proxy's declared list, comparing normalized +// forms so a trailing dot or a different case still matches what is stored. +func indexOf(hostnames []gatewayv1.Hostname, hostname string) int { + for i, h := range hostnames { + if sameHostname(string(h), hostname) { + return i + } + } + return -1 +} + +// notAttached explains that there is nothing to remove, and lists what could +// be removed instead. +func notAttached(info *url.Info, workloadName, hostname string) error { + if len(info.CustomHostnames) == 0 { + return fmt.Errorf("workload %q has no custom domains — it serves on its Datum-managed domain only", workloadName) + } + return fmt.Errorf("%s is not attached to workload %q — it serves on: %s", + hostname, workloadName, strings.Join(info.CustomHostnames, ", ")) +} + +// confirm asks before doing something that stops traffic. Without a terminal +// to ask on it refuses rather than assuming: a script that meant to remove a +// domain can say so with --yes. +func confirm(out io.Writer, prompt string, yes bool) (bool, error) { + if yes { + return true, nil + } + if !term.IsTerminal(int(os.Stdin.Fd())) { + return false, fmt.Errorf("refusing to remove a domain without confirmation — re-run with --yes") + } + + fmt.Fprint(out, prompt) + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + if err != nil { + return false, fmt.Errorf("reading confirmation: %w", err) + } + line = strings.TrimSpace(line) + return line == "y" || line == "Y", nil +} + +// completeRemoveArgs completes the workload first and then the hostnames that +// workload actually serves, minus the Datum-managed one, which cannot be +// removed and so should never be offered. +func completeRemoveArgs(cmd *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { + if len(args) == 0 { + return util.CompleteWorkloadNames(cmd, args, "") + } + if len(args) > 1 { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + c, err := util.NewClient(util.ProjectFromCmd(cmd)) + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + info, err := url.ForWorkload(context.Background(), c, args[0]) + if err != nil || info == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + return info.CustomHostnames, cobra.ShellCompDirectiveNoFileComp +} diff --git a/internal/cmd/compute/domains/remove_test.go b/internal/cmd/compute/domains/remove_test.go new file mode 100644 index 00000000..65d6c3a9 --- /dev/null +++ b/internal/cmd/compute/domains/remove_test.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "bytes" + "context" + "strings" + "testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/compute/internal/cmd/compute/url" +) + +// TestRemoveRefusesTheManagedDomain is the rule the spec states outright: the +// Datum-managed URL always appears in the list and cannot be removed. It has to +// hold however the user spells it, because the spelling they have to hand is +// whatever they copied out of a browser or out of this CLI's own output. +func TestRemoveRefusesTheManagedDomain(t *testing.T) { + variants := []struct { + name string + given string + }{ + {"exactly", testCanonical}, + {"upper case", "A1B2C3D4.DATUMPROXY.NET"}, + {"mixed case", "A1b2C3d4.DatumProxy.Net"}, + {"with a trailing dot", testCanonical + "."}, + {"https prefix", "https://" + testCanonical}, + {"https prefix and trailing slash", "https://" + testCanonical + "/"}, + {"https prefix and a path", "https://" + testCanonical + "/healthz"}, + {"https prefix, upper case and trailing dot", "HTTPS://A1B2C3D4.DatumProxy.NET./"}, + {"surrounding whitespace", " " + testCanonical + " "}, + {"with a port", testCanonical + ":443"}, + } + + for _, tc := range variants { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + err := runRemove(context.Background(), &out, c, testProject, testWorkload, tc.given, true) + if err == nil { + t.Fatalf("removing the managed domain as %q was allowed", tc.given) + } + if !strings.Contains(err.Error(), "cannot be removed") { + t.Errorf("error = %q, want it to say the domain cannot be removed", err) + } + + // And it is still there. + if got := hostnamesOf(t, c); len(got) != 1 || got[0] != testCustom { + t.Errorf("hostnames = %v, want the custom one untouched", got) + } + }) + } +} + +// The managed hostname is refused even when the platform reported it only in +// the per-hostname list and not in the canonical field. +func TestRemoveRefusesManagedHostnameFromInfo(t *testing.T) { + info := &url.Info{ + Hostnames: []url.Hostname{{Hostname: testCanonical, Managed: true}}, + } + if !managed(info, "https://"+testCanonical+"/") { + t.Error("a hostname marked managed should be refused") + } + if managed(info, testCustom) { + t.Error("a custom hostname should not be treated as managed") + } +} + +func TestRemoveDetachesHostname(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical, testCustom, "www.example.com"), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + if err := runRemove(context.Background(), &out, c, testProject, testWorkload, "HTTPS://API.Example.com./", true); err != nil { + t.Fatalf("runRemove: %v", err) + } + + if got := hostnamesOf(t, c); len(got) != 1 || got[0] != "www.example.com" { + t.Errorf("hostnames = %v, want only www.example.com left", got) + } + + got := out.String() + if !strings.Contains(got, testCustom+" removed from workload") { + t.Errorf("missing the confirmation line:\n%s", got) + } + if !strings.Contains(got, "Still serving on https://"+testCanonical) { + t.Errorf("should say what the workload still serves on:\n%s", got) + } + // Per the spec, a verified domain outlives the workload it was attached to. + if !strings.Contains(got, "stays verified in this project") { + t.Errorf("should say the domain itself is kept:\n%s", got) + } + if !strings.Contains(got, "datumctl compute domains add "+testWorkload+" "+testCustom) { + t.Errorf("should say how to attach it again:\n%s", got) + } +} + +func TestRemoveErrors(t *testing.T) { + tests := []struct { + name string + objects []client.Object + workload string + hostname string + yes bool + want string + }{ + { + name: "hostname is not attached", + objects: []client.Object{programmedProxy(testWorkload, testCanonical, testCustom)}, + workload: testWorkload, + hostname: "other.example.com", + yes: true, + want: "is not attached to workload", + }, + { + name: "workload has no custom domains at all", + objects: []client.Object{programmedProxy(testWorkload, testCanonical)}, + workload: testWorkload, + hostname: testCustom, + yes: true, + want: "has no custom domains", + }, + { + name: "workload has no URL", + objects: []client.Object{workload(testWorkload)}, + workload: testWorkload, + hostname: testCustom, + yes: true, + want: wantPublishHint, + }, + { + name: "hostname argument is blank", + objects: []client.Object{programmedProxy(testWorkload, testCanonical, testCustom)}, + workload: testWorkload, + hostname: " ", + yes: true, + want: wantNoHostname, + }, + { + // go test runs without a terminal, which is the same position a CI + // job is in: say so rather than removing a domain unasked. + name: "no terminal to confirm on and no --yes", + objects: []client.Object{programmedProxy(testWorkload, testCanonical, testCustom)}, + workload: testWorkload, + hostname: testCustom, + yes: false, + want: "--yes", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(t, tc.objects...) + var out bytes.Buffer + err := runRemove(context.Background(), &out, c, testProject, tc.workload, tc.hostname, tc.yes) + if err == nil { + t.Fatalf("expected an error, got output:\n%s", out.String()) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to mention %q", err, tc.want) + } + }) + } +} + +func TestRemoveLeavesTheProxyWhenRefusing(t *testing.T) { + c := newFakeClient(t, programmedProxy(testWorkload, testCanonical, testCustom)) + + var out bytes.Buffer + if err := runRemove(context.Background(), &out, c, testProject, testWorkload, "other.example.com", true); err == nil { + t.Fatal("expected an error") + } + if got := hostnamesOf(t, c); len(got) != 1 || got[0] != testCustom { + t.Errorf("hostnames = %v, want them untouched", got) + } +} + +func TestIndexOfNormalizes(t *testing.T) { + hostnames := hostnameList("www.example.com", "API.Example.com.") + + tests := []struct { + name string + given string + want int + }{ + {"exact", "www.example.com", 0}, + {"stored in another spelling", testCustom, 1}, + {"pasted as a URL", "https://api.example.com/", 1}, + {"absent", "nope.example.com", -1}, + {"nothing at all", "", -1}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := indexOf(hostnames, tc.given); got != tc.want { + t.Errorf("indexOf(%q) = %d, want %d", tc.given, got, tc.want) + } + }) + } +} diff --git a/internal/cmd/compute/domains/wait_failure_test.go b/internal/cmd/compute/domains/wait_failure_test.go new file mode 100644 index 00000000..3759abef --- /dev/null +++ b/internal/cmd/compute/domains/wait_failure_test.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// TestAddStopsWhenTheHostnameCanNeverBeRead: `domains add` attaches the +// hostname and then waits for the platform to verify it. A control plane the +// wait cannot read is not going to start answering, so swallowing the read +// error leaves the command polling in silence forever — the user is told to +// wait for DNS and never told anything again. It has to give up and say why. +// +// Regression test: before the failure budget, this hung past the guard below. +func TestAddStopsWhenTheHostnameCanNeverBeRead(t *testing.T) { + boom := errors.New("httpproxies.networking.datumapis.com is forbidden") + + // Reads fail only after the hostname is attached, so the command gets far + // enough to start waiting — which is the state that used to hang. + attached := false + c := interceptor.NewClient(newFakeClient(t, + programmedProxy(testWorkload, testCanonical), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ), interceptor.Funcs{ + Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + attached = true + return cl.Update(ctx, obj, opts...) + }, + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if attached { + return boom + } + return cl.List(ctx, list, opts...) + }, + }) + + var out bytes.Buffer + done := make(chan error, 1) + go func() { + done <- runAdd(context.Background(), &out, c, testProject, testWorkload, testCustom) + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("a hostname whose state cannot be read must fail the command, not wait forever") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure to survive wrapping", err) + } + if !strings.Contains(err.Error(), testCustom) { + t.Errorf("error = %q, want it to name the hostname", err) + } + case <-time.After(30 * time.Second): + t.Fatalf("runAdd never returned — the wait is hanging on an unreadable control plane:\n%s", out.String()) + } +} + +// TestAddRidesOutATransientReadFailure: the counterpart. One failed read inside +// the wait is nothing — the platform was just asked to check the hostname — and +// giving up there would break a command that is going fine. +// +// The hostname is already attached and verified, so runAdd reports that and goes +// straight to the wait. The first read is the one runAdd itself makes; the +// second is the wait's, and that is the one that fails. +func TestAddRidesOutATransientReadFailure(t *testing.T) { + reads := 0 + failed := false + c := interceptor.NewClient(newFakeClient(t, + verifiedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*networkingv1alpha.HTTPProxyList); ok { + reads++ + if reads == 2 { + failed = true + return errors.New("etcdserver: request timed out") + } + } + return cl.List(ctx, list, opts...) + }, + }) + + var out bytes.Buffer + done := make(chan error, 1) + go func() { + done <- runAdd(context.Background(), &out, c, testProject, testWorkload, testCustom) + }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("one transient read failure must not fail the command: %v\n%s", err, out.String()) + } + if !failed { + t.Error("the interceptor never injected its failure, so nothing was ridden out") + } + case <-time.After(30 * time.Second): + t.Fatalf("runAdd never returned:\n%s", out.String()) + } +} diff --git a/internal/cmd/compute/domains/write_test.go b/internal/cmd/compute/domains/write_test.go new file mode 100644 index 00000000..629222ab --- /dev/null +++ b/internal/cmd/compute/domains/write_test.go @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package domains + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// failingClient answers reads normally and refuses one write, which is what a +// conflicting concurrent deploy or a missing update permission looks like. +func failingClient(t *testing.T, boom error, objs ...client.Object) client.WithWatch { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithStatusSubresource(&networkingv1alpha.HTTPProxy{}, &networkingv1alpha.NetworkService{}, &networkingv1alpha.Domain{}). + WithObjects(objs...). + WithInterceptorFuncs(interceptor.Funcs{ + Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { + return boom + }, + }). + Build() +} + +// TestAddReportsAWriteFailure: a hostname that could not be attached must fail +// the command and say so. The wait that follows is the part that looks like +// progress, so falling into it after a failed write would show a user a +// verification that is never going to happen. +func TestAddReportsAWriteFailure(t *testing.T) { + boom := errors.New("Operation cannot be fulfilled on httpproxies.networking.datumapis.com \"api\": the object has been modified") + c := failingClient(t, boom, programmedProxy(testWorkload, testCanonical), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true))) + + var out bytes.Buffer + err := runAdd(context.Background(), &out, c, testProject, testWorkload, testCustom) + if err == nil { + t.Fatal("a hostname that could not be attached must fail the command") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's message to reach the user", err) + } + for _, want := range []string{testCustom, testWorkload} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to name %q", err, want) + } + } + if strings.Contains(out.String(), "Verifying") { + t.Errorf("nothing was attached, so nothing may be reported as verifying:\n%s", out.String()) + } + if got := hostnamesOf(t, c); len(got) != 0 { + t.Errorf("hostnames = %v, want none attached", got) + } +} + +// TestRemoveReportsAWriteFailure: the same on the way out. A detach that did +// not happen must not print the "removed" summary, which is what a user reads +// as confirmation that the hostname is free to attach elsewhere. +func TestRemoveReportsAWriteFailure(t *testing.T) { + boom := errors.New("forbidden") + c := failingClient(t, boom, programmedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true))) + + var out bytes.Buffer + err := runRemove(context.Background(), &out, c, testProject, testWorkload, testCustom, true) + if err == nil { + t.Fatal("a hostname that could not be detached must fail the command") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure", err) + } + if strings.Contains(out.String(), "removed from workload") { + t.Errorf("nothing was detached, so nothing may claim it was:\n%s", out.String()) + } + if got := hostnamesOf(t, c); len(got) != 1 || got[0] != testCustom { + t.Errorf("hostnames = %v, want the hostname still attached", got) + } +} + +// TestAddKeepsTheHostnamesAlreadyThere: attaching a second domain must not +// disturb the first. `domains add` writes the whole spec back, so this is the +// same class of mistake a redeploy makes, one command over. +func TestAddKeepsTheHostnamesAlreadyThere(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + // The wait is not what this test is about, and a hostname the platform has + // not reported on stays pending forever: detach after the first poll. + var out bytes.Buffer + if err := runAdd(cancelled(), &out, c, testProject, testWorkload, "www."+testApex); err != nil { + t.Fatalf("runAdd: %v", err) + } + + if want := testCustom + ",www." + testApex; strings.Join(hostnamesOf(t, c), ",") != want { + t.Errorf("hostnames = %v, want %q — declared order, nothing lost", hostnamesOf(t, c), want) + } +} + +// TestRemoveKeepsTheOrderOfWhatIsLeft: removing the middle hostname of three +// must leave the other two in the order they were declared, since that order +// is what every view renders and what `add` appends to. +func TestRemoveKeepsTheOrderOfWhatIsLeft(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical, "one."+testApex, "two."+testApex, "three."+testApex), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + if err := runRemove(context.Background(), &out, c, testProject, testWorkload, "two."+testApex, true); err != nil { + t.Fatalf("runRemove: %v", err) + } + + if want := "one." + testApex + ",three." + testApex; strings.Join(hostnamesOf(t, c), ",") != want { + t.Errorf("hostnames = %v, want %q", hostnamesOf(t, c), want) + } +} + +// TestRemoveTheLastCustomHostname: the workload falls back to serving on its +// Datum-managed domain alone, and is told so — that sentence is the difference +// between "I have taken my site down" and "I have detached my domain". +func TestRemoveTheLastCustomHostname(t *testing.T) { + c := newFakeClient(t, + programmedProxy(testWorkload, testCanonical, testCustom), + serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), + ) + + var out bytes.Buffer + if err := runRemove(context.Background(), &out, c, testProject, testWorkload, testCustom, true); err != nil { + t.Fatalf("runRemove: %v", err) + } + + if got := hostnamesOf(t, c); len(got) != 0 { + t.Errorf("hostnames = %v, want none", got) + } + if !strings.Contains(out.String(), "Still serving on https://"+testCanonical) { + t.Errorf("removing the last custom domain must say what still answers:\n%s", out.String()) + } + + // And the list view still has a row for the workload: the managed domain + // is permanent. + var list bytes.Buffer + if err := runList(context.Background(), &list, c, testProject, util.OutputTable, false); err != nil { + t.Fatalf("runList: %v", err) + } + f := row(t, list.String(), testWorkload, testCanonical) + if f[len(f)-1] != managedMarker { + t.Errorf("row = %v, want the managed domain still listed", f) + } +} + +// TestListPropagatesLookupFailures: `domains` exists to answer "is my URL +// working". An unreadable control plane must say so rather than render an +// empty list, which reads as "you have no domains". +func TestListPropagatesLookupFailures(t *testing.T) { + boom := errors.New("connection refused") + c := interceptor.NewClient(newFakeClient(t, + programmedProxy(testWorkload, testCanonical, testCustom), + ), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return boom + }, + }) + + var out bytes.Buffer + err := runList(context.Background(), &out, c, testProject, util.OutputTable, false) + if !errors.Is(err, boom) { + t.Fatalf("error = %v, want the lookup failure", err) + } + if strings.Contains(out.String(), "No domains in project") { + t.Errorf("an unreadable project must not be reported as an empty one:\n%s", out.String()) + } +} + +// TestDetailPropagatesLookupFailures: the same for the detail view, which is +// the one a user reaches for precisely when something is wrong. +func TestDetailPropagatesLookupFailures(t *testing.T) { + boom := errors.New("connection refused") + c := interceptor.NewClient(newFakeClient(t, workload(testWorkload)), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return boom + }, + }) + + var out bytes.Buffer + err := runDetail(context.Background(), &out, c, testProject, testWorkload, util.OutputTable) + if !errors.Is(err, boom) { + t.Fatalf("error = %v, want the lookup failure", err) + } + if strings.Contains(err.Error(), "--http-port") { + t.Errorf("a failed lookup must not be reported as a workload that was never published: %v", err) + } +} diff --git a/internal/cmd/compute/open/browser.go b/internal/cmd/compute/open/browser.go new file mode 100644 index 00000000..58f82a15 --- /dev/null +++ b/internal/cmd/compute/open/browser.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package open + +import ( + "context" + "fmt" + neturl "net/url" + "os/exec" + "runtime" + "strings" +) + +// xdgOpen is the URL handler on every platform that is not macOS or Windows. +const xdgOpen = "xdg-open" + +// opener launches a URL in the user's default browser. It is a function type +// so tests can substitute one that records the URL instead of opening it. +type opener func(ctx context.Context, rawURL string) error + +// openBrowser is the opener used in production. Tests pass their own to run +// rather than reassigning this, so no test can start a real browser. +var openBrowser opener = openInDefaultBrowser + +// openInDefaultBrowser hands a URL to the platform's URL handler. +// +// The URL is validated before it gets here, so nothing but an https:// URL the +// server produced is ever passed on. No shell is involved: the argument is +// passed to the handler directly and is never word-split or expanded. +func openInDefaultBrowser(ctx context.Context, rawURL string) error { + if err := validateURL(rawURL); err != nil { + return err + } + + name, args := browserCommand(runtime.GOOS, rawURL) + + cmd := exec.CommandContext(ctx, name, args...) + if err := cmd.Run(); err != nil { + return fmt.Errorf("opening browser with %s: %w", name, err) + } + return nil +} + +// browserCommand returns the URL handler for an operating system and the +// arguments to hand it. Split out from openInDefaultBrowser so it can be +// tested for every platform without launching anything. +func browserCommand(goos, rawURL string) (name string, args []string) { + switch goos { + case "darwin": + return "open", []string{rawURL} + case "windows": + return "rundll32", []string{"url.dll,FileProtocolHandler", rawURL} + default: + return xdgOpen, []string{rawURL} + } +} + +// validateURL rejects anything that is not an https:// URL of the shape the +// platform hands back. The CLI only ever opens a URL the server reported, and +// this is the check that keeps it that way. +func validateURL(rawURL string) error { + if rawURL == "" { + return fmt.Errorf("no URL to open") + } + if strings.TrimSpace(rawURL) != rawURL { + return fmt.Errorf("refusing to open %q: URL has leading or trailing whitespace", rawURL) + } + + u, err := neturl.Parse(rawURL) + if err != nil { + return fmt.Errorf("refusing to open %q: not a valid URL", rawURL) + } + if u.Scheme != "https" { + return fmt.Errorf("refusing to open %q: only https:// URLs are opened", rawURL) + } + if u.Host == "" { + return fmt.Errorf("refusing to open %q: URL has no host", rawURL) + } + if u.User != nil { + return fmt.Errorf("refusing to open %q: URL carries embedded credentials", rawURL) + } + return nil +} diff --git a/internal/cmd/compute/open/browser_test.go b/internal/cmd/compute/open/browser_test.go new file mode 100644 index 00000000..747055cf --- /dev/null +++ b/internal/cmd/compute/open/browser_test.go @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package open + +import ( + "context" + "strings" + "testing" +) + +// The substrings the refusal tests match on, named once so the same wording is +// asserted here and in open_test.go. +const ( + wantNotHTTPS = "only https:// URLs" + wantCredentials = "embedded credentials" + wantMalformed = "not a valid URL" +) + +func TestValidateURL(t *testing.T) { + tests := []struct { + name string + rawURL string + wantErr string + }{ + {name: "managed hostname", rawURL: "https://a1b2c3d4.datumproxy.net"}, + {name: "custom hostname with a path", rawURL: "https://api.example.com/healthz"}, + {name: "explicit port", rawURL: "https://api.example.com:8443"}, + + {name: "empty", rawURL: "", wantErr: "no URL to open"}, + {name: "plaintext http", rawURL: "http://api.example.com", wantErr: wantNotHTTPS}, + {name: "javascript scheme", rawURL: "javascript:alert(1)", wantErr: wantNotHTTPS}, + {name: "file scheme", rawURL: "file:///etc/passwd", wantErr: wantNotHTTPS}, + {name: "scheme-relative", rawURL: "//api.example.com", wantErr: wantNotHTTPS}, + {name: "bare hostname", rawURL: "api.example.com", wantErr: wantNotHTTPS}, + {name: "looks like a flag", rawURL: "--version", wantErr: wantNotHTTPS}, + {name: "no host", rawURL: "https://", wantErr: "no host"}, + {name: "embedded credentials", rawURL: "https://user:pw@api.example.com", wantErr: wantCredentials}, + {name: "leading whitespace", rawURL: " https://api.example.com", wantErr: "whitespace"}, + {name: "trailing newline", rawURL: "https://api.example.com\n", wantErr: "whitespace"}, + {name: "unparseable", rawURL: "https://not a host", wantErr: wantMalformed}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := validateURL(tc.rawURL) + switch { + case tc.wantErr == "" && err != nil: + t.Fatalf("validateURL(%q) = %v, want nil", tc.rawURL, err) + case tc.wantErr != "" && err == nil: + t.Fatalf("validateURL(%q) = nil, want an error containing %q", tc.rawURL, tc.wantErr) + case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): + t.Fatalf("validateURL(%q) = %q, want it to contain %q", tc.rawURL, err, tc.wantErr) + } + }) + } +} + +// The real opener refuses anything validateURL refuses, before a process is +// ever started. +func TestOpenInDefaultBrowserValidatesFirst(t *testing.T) { + if err := openInDefaultBrowser(context.Background(), "http://api.example.com"); err == nil { + t.Fatal("openInDefaultBrowser opened a plaintext URL, want a refusal") + } +} + +func TestBrowserCommand(t *testing.T) { + const rawURL = "https://a1b2c3d4.datumproxy.net" + + tests := []struct { + goos string + wantName string + wantArgs []string + }{ + {goos: "darwin", wantName: "open", wantArgs: []string{rawURL}}, + {goos: "linux", wantName: xdgOpen, wantArgs: []string{rawURL}}, + {goos: "freebsd", wantName: xdgOpen, wantArgs: []string{rawURL}}, + {goos: "windows", wantName: "rundll32", wantArgs: []string{"url.dll,FileProtocolHandler", rawURL}}, + } + + for _, tc := range tests { + t.Run(tc.goos, func(t *testing.T) { + name, args := browserCommand(tc.goos, rawURL) + if name != tc.wantName { + t.Errorf("name = %q, want %q", name, tc.wantName) + } + if strings.Join(args, " ") != strings.Join(tc.wantArgs, " ") { + t.Errorf("args = %q, want %q", args, tc.wantArgs) + } + }) + } +} diff --git a/internal/cmd/compute/open/custom_test.go b/internal/cmd/compute/open/custom_test.go new file mode 100644 index 00000000..819ed894 --- /dev/null +++ b/internal/cmd/compute/open/custom_test.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package open + +import ( + "context" + "errors" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "go.datum.net/compute/internal/cmd/compute/url" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const testCustom = "api.example.com" + +// withCustomHostname returns the objects for a workload serving on a custom +// hostname as well as its managed one. active decides whether the custom +// hostname is the one answering yet. +func withCustomHostname(t *testing.T, active bool) []client.Object { + t.Helper() + objs := publishedObjects(t, testCanonical, true) + + proxy := objs[1].(*networkingv1alpha.HTTPProxy) + proxy.Spec.Hostnames = []gatewayv1.Hostname{testCustom} + + status := metav1.ConditionFalse + reason := "DomainNotVerified" + if active { + status = metav1.ConditionTrue + reason = "Verified" + } + proxy.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: testCustom, + Conditions: []metav1.Condition{ + {Type: networkingv1alpha.HostnameConditionVerified, Status: status, Reason: reason, + Message: "waiting for the TXT record"}, + {Type: networkingv1alpha.HostnameConditionCertificateReady, Status: status, Reason: "CertificateIssued"}, + }, + }} + return objs +} + +// TestOpenPrefersTheCustomHostname: once a custom domain is serving, it is the +// address the user thinks of as their site, and it is the one a browser gets. +// The managed hostname still works and is still listed by `domains`; it is +// just not what `open` is for. +func TestOpenPrefersTheCustomHostname(t *testing.T) { + c := newFakeClient(t, withCustomHostname(t, true)...) + opener := &recordingOpener{} + + var out strings.Builder + if err := run(context.Background(), &out, c, testWorkload, false, opener.open); err != nil { + t.Fatalf("run: %v", err) + } + + want := "https://" + testCustom + if len(opener.calls) != 1 || opener.calls[0] != want { + t.Errorf("opened %v, want %q", opener.calls, want) + } + if !strings.Contains(out.String(), want) { + t.Errorf("output = %q, want it to name the custom hostname", out.String()) + } + if strings.Contains(out.String(), testCanonical) { + t.Errorf("output = %q, should not offer the managed hostname once a custom one serves", out.String()) + } +} + +// TestOpenFallsBackWhileACustomHostnameIsPending: a hostname that has not +// verified does not answer, so opening it would show the user a browser error +// for a workload that is running perfectly. The managed hostname is what works. +func TestOpenFallsBackWhileACustomHostnameIsPending(t *testing.T) { + c := newFakeClient(t, withCustomHostname(t, false)...) + opener := &recordingOpener{} + + var out strings.Builder + if err := run(context.Background(), &out, c, testWorkload, false, opener.open); err != nil { + t.Fatalf("run: %v", err) + } + + if len(opener.calls) != 1 || opener.calls[0] != testURL { + t.Errorf("opened %v, want the managed URL %q while the custom hostname is pending", opener.calls, testURL) + } +} + +// TestOpenURLMatchesWhatDomainsShows is the cross-command contract: the URL +// `open --url` prints is exactly the one the domains detail view reports, so a +// user never has two answers to "what is my URL". +func TestOpenURLMatchesWhatDomainsShows(t *testing.T) { + c := newFakeClient(t, withCustomHostname(t, true)...) + + info, err := url.ForWorkload(context.Background(), c, testWorkload) + if err != nil { + t.Fatalf("lookup: %v", err) + } + + var out strings.Builder + if err := run(context.Background(), &out, c, testWorkload, true, (&recordingOpener{}).open); err != nil { + t.Fatalf("run: %v", err) + } + if got := strings.TrimSpace(out.String()); got != info.URL { + t.Errorf("open --url printed %q, but the URL info says %q", got, info.URL) + } +} + +// A workload the CLI cannot read at all is a different problem from one that +// does not exist, and must not be reported as either "not found" or "no URL". +func TestOpenReportsAnUnreadableWorkload(t *testing.T) { + boom := errors.New("workloads.compute.datumapis.com is forbidden") + c := interceptor.NewClient(newFakeClient(t), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return boom + }, + }) + + opener := &recordingOpener{} + var out strings.Builder + err := run(context.Background(), &out, c, testWorkload, false, opener.open) + if !errors.Is(err, boom) { + t.Fatalf("error = %v, want the server's failure", err) + } + if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "--http-port") { + t.Errorf("a permission failure must not read as a missing or unpublished workload: %v", err) + } + if len(opener.calls) != 0 { + t.Errorf("opened %v, want nothing", opener.calls) + } +} diff --git a/internal/cmd/compute/open/open.go b/internal/cmd/compute/open/open.go new file mode 100644 index 00000000..c9dc9917 --- /dev/null +++ b/internal/cmd/compute/open/open.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package open implements `datumctl compute open`: the one-word path from a +// workload name to the URL in a browser. +package open + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" +) + +// Command returns the `datumctl compute open` command. +func Command() *cobra.Command { + var urlOnly bool + + cmd := &cobra.Command{ + Use: "open ", + Short: "Open a workload's URL in the default browser", + Long: "Open a workload's URL in the default browser.\n\n" + + "With --url the URL is printed and nothing is opened, so it can be piped\n" + + "into another command.", + Example: " # Open the URL for the \"api\" workload\n" + + " datumctl compute open api\n\n" + + " # Print the URL instead of opening it\n" + + " datumctl compute open api --url\n\n" + + " # Use it in a script\n" + + " curl -sS \"$(datumctl compute open api --url)/healthz\"", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := util.NewClient(util.ProjectFromCmd(cmd)) + if err != nil { + return err + } + return run(cmd.Context(), cmd.OutOrStdout(), c, args[0], urlOnly, openBrowser) + }, + ValidArgsFunction: util.CompleteWorkloadNames, + } + + cmd.Flags().BoolVar(&urlOnly, "url", false, "Print the URL instead of opening it") + + return cmd +} + +// run resolves the workload's URL and either prints it or opens it. +// +// Two failures a developer hits here are different problems with different +// fixes, so they get different errors: a workload that does not exist, and a +// workload that exists but has never been published. +func run(ctx context.Context, out io.Writer, c client.Client, workloadName string, urlOnly bool, open opener) error { + if ctx == nil { + ctx = context.Background() + } + + var workload computev1alpha.Workload + if err := c.Get(ctx, types.NamespacedName{Namespace: util.ResourceNamespace, Name: workloadName}, &workload); err != nil { + if k8serrors.IsNotFound(err) { + return fmt.Errorf("workload %q not found — run 'datumctl compute workloads' to see what is deployed", workloadName) + } + return fmt.Errorf("getting workload: %w", err) + } + + info, err := url.ForWorkload(ctx, c, workloadName) + if err != nil { + return err + } + if info == nil { + return fmt.Errorf( + "workload %q has no URL — it does not declare an HTTP port.\nTo publish it: datumctl compute deploy %s --http-port ", + workloadName, workloadName, + ) + } + if info.URL == "" { + return fmt.Errorf("workload %q does not have a URL yet — the platform is still assigning one. Try again in a moment", workloadName) + } + + // Validate before anything else touches the URL, so the bare --url form is + // held to the same standard as the one handed to a browser. + if err := validateURL(info.URL); err != nil { + return err + } + + // The pipe-friendly form: the URL, a newline, nothing else. + if urlOnly { + fmt.Fprintln(out, info.URL) + return nil + } + + if !info.Live() { + fmt.Fprintln(out, "The URL is not fully live yet — it may not respond immediately.") + } + fmt.Fprintf(out, "Opening %s\n", info.URL) + + return open(ctx, info.URL) +} diff --git a/internal/cmd/compute/open/open_test.go b/internal/cmd/compute/open/open_test.go new file mode 100644 index 00000000..fdcf9b2f --- /dev/null +++ b/internal/cmd/compute/open/open_test.go @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package open + +import ( + "context" + "errors" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + testWorkload = "api" + testPortName = "http" + testCanonical = "a1b2c3d4.datumproxy.net" + testURL = "https://" + testCanonical +) + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := computev1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering compute scheme: %v", err) + } + if err := networkingv1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering networking scheme: %v", err) + } + return s +} + +func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithStatusSubresource(&networkingv1alpha.HTTPProxy{}, &networkingv1alpha.NetworkService{}). + WithObjects(objs...). + Build() +} + +func testWorkloadObject(name string) *computev1alpha.Workload { + return &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: util.ResourceNamespace, + UID: "workload-uid", + }, + } +} + +// publishedObjects returns the two objects the platform reports for a workload +// that is serving on the managed hostname. +func publishedObjects(t *testing.T, canonical string, programmed bool) []client.Object { + t.Helper() + w := testWorkloadObject(testWorkload) + + proxy := url.BuildHTTPProxy(w, testPortName, nil) + proxy.Status.CanonicalHostname = canonical + programmedStatus := metav1.ConditionFalse + if programmed { + programmedStatus = metav1.ConditionTrue + } + proxy.Status.Conditions = []metav1.Condition{ + {Type: networkingv1alpha.HTTPProxyConditionProgrammed, Status: programmedStatus, Reason: "Programmed"}, + {Type: networkingv1alpha.HTTPProxyConditionCertificatesReady, Status: metav1.ConditionTrue, Reason: "AllCertificatesReady"}, + } + + svc := url.BuildNetworkService(w, testPortName, 8080) + svc.Status.Summary = networkingv1alpha.NetworkServiceSummary{Locations: 1, Members: 2, Healthy: 2} + + return []client.Object{w, proxy, svc} +} + +// recordingOpener stands in for the browser: it records what it was handed and +// never launches anything. +type recordingOpener struct { + calls []string + err error +} + +func (r *recordingOpener) open(_ context.Context, rawURL string) error { + r.calls = append(r.calls, rawURL) + return r.err +} + +func TestRun(t *testing.T) { + tests := []struct { + name string + // objects seeded into the control plane + objects func(t *testing.T) []client.Object + // arguments + workload string + urlOnly bool + openErr error + + wantErr string + wantOut string + wantContains []string + wantOpened []string + }{ + { + name: "opens the URL in the browser", + objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, true) }, + workload: testWorkload, + wantOut: "Opening " + testURL + "\n", + wantOpened: []string{testURL}, + }, + { + name: "--url prints the bare URL and opens nothing", + objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, true) }, + workload: testWorkload, + urlOnly: true, + wantOut: testURL + "\n", + wantOpened: nil, + }, + { + name: "--url output has no decoration to break a pipe", + objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, false) }, + workload: testWorkload, + urlOnly: true, + wantOut: testURL + "\n", + wantOpened: nil, + }, + { + name: "a URL that is not live yet still opens, with a note", + objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, false) }, + workload: testWorkload, + wantContains: []string{ + "not fully live yet", + "Opening " + testURL, + }, + wantOpened: []string{testURL}, + }, + { + name: "workload that is not published names the command that publishes it", + objects: func(t *testing.T) []client.Object { + return []client.Object{testWorkloadObject(testWorkload)} + }, + workload: testWorkload, + wantErr: "datumctl compute deploy api --http-port", + wantOpened: nil, + }, + { + name: "workload that does not exist is a different error", + objects: func(t *testing.T) []client.Object { + return []client.Object{testWorkloadObject("other")} + }, + workload: testWorkload, + wantErr: `workload "api" not found`, + wantOpened: nil, + }, + { + name: "published but no hostname assigned yet", + objects: func(t *testing.T) []client.Object { + return publishedObjects(t, "", true) + }, + workload: testWorkload, + wantErr: "does not have a URL yet", + wantOpened: nil, + }, + { + name: "a hostname carrying credentials is never opened", + objects: func(t *testing.T) []client.Object { + return publishedObjects(t, "user:password@evil.example.com", true) + }, + workload: testWorkload, + wantErr: wantCredentials, + wantOpened: nil, + }, + { + name: "a malformed hostname is never opened", + objects: func(t *testing.T) []client.Object { + return publishedObjects(t, "not a host", true) + }, + workload: testWorkload, + wantErr: wantMalformed, + wantOpened: nil, + }, + { + name: "a malformed hostname is not printed by --url either", + objects: func(t *testing.T) []client.Object { + return publishedObjects(t, "not a host", true) + }, + workload: testWorkload, + urlOnly: true, + wantErr: wantMalformed, + wantOpened: nil, + }, + { + name: "a browser failure is reported", + objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, true) }, + workload: testWorkload, + openErr: errors.New("no browser"), + wantErr: "no browser", + wantOpened: []string{testURL}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(t, tc.objects(t)...) + opener := &recordingOpener{err: tc.openErr} + var out strings.Builder + + err := run(context.Background(), &out, c, tc.workload, tc.urlOnly, opener.open) + + switch { + case tc.wantErr == "" && err != nil: + t.Fatalf("run() error = %v, want nil", err) + case tc.wantErr != "" && err == nil: + t.Fatalf("run() error = nil, want one containing %q", tc.wantErr) + case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): + t.Fatalf("run() error = %q, want it to contain %q", err, tc.wantErr) + } + + if tc.wantOut != "" && out.String() != tc.wantOut { + t.Errorf("output = %q, want %q", out.String(), tc.wantOut) + } + for _, want := range tc.wantContains { + if !strings.Contains(out.String(), want) { + t.Errorf("output = %q, want it to contain %q", out.String(), want) + } + } + if got, want := strings.Join(opener.calls, ","), strings.Join(tc.wantOpened, ","); got != want { + t.Errorf("opened %q, want %q", got, want) + } + }) + } +} + +// A failing lookup is the platform's problem to report, not something to +// translate into "this workload has no URL". +func TestRunPropagatesLookupErrors(t *testing.T) { + boom := errors.New("connection refused") + c := interceptor.NewClient(newFakeClient(t, testWorkloadObject(testWorkload)), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return boom + }, + }) + + opener := &recordingOpener{} + var out strings.Builder + err := run(context.Background(), &out, c, testWorkload, false, opener.open) + if !errors.Is(err, boom) { + t.Fatalf("run() error = %v, want the transport error to propagate", err) + } + if len(opener.calls) != 0 { + t.Fatalf("opened %v, want nothing", opener.calls) + } + if out.String() != "" { + t.Errorf("output = %q, want nothing", out.String()) + } +} + +func TestCommand(t *testing.T) { + cmd := Command() + + if cmd.Use != "open " { + t.Errorf("Use = %q", cmd.Use) + } + if cmd.ValidArgsFunction == nil { + t.Error("ValidArgsFunction is nil, want util.CompleteWorkloadNames") + } + if cmd.Flags().Lookup("url") == nil { + t.Error("--url flag is not registered") + } + if err := cmd.Args(cmd, []string{}); err == nil { + t.Error("Args accepted no workload name, want exactly one") + } + if err := cmd.Args(cmd, []string{"a", "b"}); err == nil { + t.Error("Args accepted two workload names, want exactly one") + } +} diff --git a/internal/cmd/compute/root.go b/internal/cmd/compute/root.go index d9c25a8c..3f703d3b 100644 --- a/internal/cmd/compute/root.go +++ b/internal/cmd/compute/root.go @@ -8,7 +8,9 @@ import ( "go.datum.net/compute/internal/cmd/compute/build" "go.datum.net/compute/internal/cmd/compute/deploy" "go.datum.net/compute/internal/cmd/compute/destroy" + "go.datum.net/compute/internal/cmd/compute/domains" "go.datum.net/compute/internal/cmd/compute/instances" + "go.datum.net/compute/internal/cmd/compute/open" "go.datum.net/compute/internal/cmd/compute/quota" "go.datum.net/compute/internal/cmd/compute/restart" "go.datum.net/compute/internal/cmd/compute/rollout" @@ -37,7 +39,9 @@ func Command() *cobra.Command { access.Command(), deploy.Command(), destroy.Command(), + domains.Command(), instances.Command(), + open.Command(), quota.Command(), restart.Command(), rollout.Command(), diff --git a/internal/cmd/compute/url/lookup.go b/internal/cmd/compute/url/lookup.go new file mode 100644 index 00000000..932f64c4 --- /dev/null +++ b/internal/cmd/compute/url/lookup.go @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "context" + "fmt" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" + "sigs.k8s.io/controller-runtime/pkg/client" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// Status strings shown for a hostname. A hostname is active or it is not; when +// it is not, the server's own message is shown rather than a CLI translation +// of it — and never the server's condition reason, which is camelCase internal +// state the developer is not meant to read. +const ( + statusActive = "active" + statusPending = "pending" + + certificateValid = "valid" +) + +// Backends totals the instances serving a URL, across every city. +type Backends struct { + // Cities is how many cities the URL has backends in. + Cities int32 `json:"cities"` + // Total is how many backends are registered, healthy or not. + Total int32 `json:"total"` + // Healthy is how many of them are taking traffic. + Healthy int32 `json:"healthy"` +} + +// Location reports the backends a URL has in one city, and whether that city +// is taking traffic. +type Location struct { + // City is the location as the platform reports it. + City string `json:"city"` + // Backends is how many instances in this city back the URL. + Backends int32 `json:"backends"` + // Healthy is how many of them are taking traffic. + Healthy int32 `json:"healthy"` + // Serving reports whether this city is in rotation. + Serving bool `json:"serving"` +} + +// Hostname is the state of one hostname on a URL. +type Hostname struct { + // Hostname is the fully qualified name. + Hostname string `json:"hostname"` + // URL is the hostname as an https:// URL. Datum never serves plaintext. + URL string `json:"url"` + // Managed is true for the platform-assigned hostname, which the user + // neither creates nor removes. + Managed bool `json:"managed"` + // Active is true when this hostname is serving traffic. + Active bool `json:"active"` + // Status is "active", or the server's blocking message verbatim, or + // "pending" when the server has not said anything a human can read. + Status string `json:"status"` + // Certificate is "valid", the server's certificate message verbatim, + // "pending" when it is not ready and the server said nothing readable, or + // empty when the platform has not reported on a certificate at all. + Certificate string `json:"certificate,omitempty"` + // Detail is the server's message for the first blocking condition, shown + // verbatim. Empty when nothing is blocking. + Detail string `json:"detail,omitempty"` + // Conditions are the raw per-hostname conditions, for -o yaml and for + // callers that want to render more than Status. + Conditions []metav1.Condition `json:"-"` +} + +// Info is everything the CLI knows about one workload's URL. A workload that +// has not been published has no Info at all — callers get nil, not a zero +// value. +type Info struct { + // WorkloadName is the workload this URL belongs to. + WorkloadName string `json:"workloadName"` + // URL is the one URL to show the user: the first active custom hostname if + // there is one, otherwise the platform-managed hostname. Always https://. + URL string `json:"url"` + // CanonicalHostname is the platform-managed hostname. Empty until the + // server assigns one. + CanonicalHostname string `json:"canonicalHostname,omitempty"` + // CustomHostnames are the hostnames the user attached, in declared order. + CustomHostnames []string `json:"customHostnames,omitempty"` + // Hostnames carries per-hostname state for every hostname on the URL, + // custom ones first, the managed one last. + Hostnames []Hostname `json:"hostnames,omitempty"` + + // PortName and Port are the port the backends answer on. Protocol is the + // transport, always TCP today. + PortName string `json:"portName,omitempty"` + Port int32 `json:"port,omitempty"` + Protocol string `json:"protocol,omitempty"` + + // Backends totals the serving instances; Locations breaks that down by + // city, which is what makes a multi-city deployment legible. + Backends Backends `json:"backends"` + Locations []Location `json:"locations,omitempty"` + + // EdgeProgrammed is true once the edge is carrying the configuration. + EdgeProgrammed bool `json:"edgeProgrammed"` + // CertificateIssued is true once the primary hostname has a certificate. + CertificateIssued bool `json:"certificateIssued"` + // CertificateKnown is false when the platform has reported nothing about a + // certificate, which is different from reporting that there isn't one. + CertificateKnown bool `json:"-"` + + // ProxyConditions and ServiceConditions are the raw conditions behind the + // fields above. Render whatever the server emits; never branch on a reason. + ProxyConditions []metav1.Condition `json:"-"` + ServiceConditions []metav1.Condition `json:"-"` + + // Proxy and Service are the objects themselves, for `-o yaml`. They are + // the machinery: never name them in normal output. Reach them through + // Objects(), which is the shape structured output renders. + Proxy *networkingv1alpha.HTTPProxy `json:"-"` + Service *networkingv1alpha.NetworkService `json:"-"` +} + +// Objects is the raw platform state behind a URL, in the shape structured +// output renders it. It is the escape hatch the product promises: plain +// language in normal output, the real objects behind `-o yaml`. +// +// Both fields may be nil — a proxy exists for a moment before its backends do, +// and a control plane that does not serve the backend kind reports none. +type Objects struct { + HTTPProxy *networkingv1alpha.HTTPProxy `json:"httpProxy,omitempty"` + NetworkService *networkingv1alpha.NetworkService `json:"networkService,omitempty"` +} + +// Objects returns the real objects behind the URL, for a caller rendering +// `-o yaml` or `-o json`. It returns nil when there is nothing to show, so a +// caller can fall back to the human view without a second check. +// +// This is the only sanctioned way out of this package to the machinery: the +// default human output never names it. +func (i *Info) Objects() *Objects { + if i == nil || (i.Proxy == nil && i.Service == nil) { + return nil + } + return &Objects{HTTPProxy: i.Proxy, NetworkService: i.Service} +} + +// Live reports whether the URL is ready to be handed to the user: it has a +// hostname, the edge is programmed, and a certificate has been issued (or the +// platform reports no certificate state at all, in which case there is nothing +// to wait for). +func (i *Info) Live() bool { + if i == nil { + return false + } + return i.URL != "" && i.EdgeProgrammed && (i.CertificateIssued || !i.CertificateKnown) +} + +// ForWorkload returns the URL info for one workload, or nil when the workload +// has not been published. A workload without a URL is an ordinary state, not +// an error; so is a control plane that does not serve these kinds at all. +// Transport and permission errors propagate. +func ForWorkload(ctx context.Context, c client.Client, workloadName string) (*Info, error) { + proxies, services, err := list(ctx, c, labels.Set{computev1alpha.WorkloadNameLabel: workloadName}) + if err != nil { + return nil, err + } + if len(proxies) == 0 { + return nil, nil + } + return newInfo(workloadName, &proxies[0], serviceFor(services, workloadName)), nil +} + +// ForAll returns URL info for every published workload in the project, keyed +// by workload name. Workloads without a URL are absent from the map. +// +// It costs exactly two List calls no matter how many workloads there are: the +// list view renders a whole project through this. +func ForAll(ctx context.Context, c client.Client) (map[string]*Info, error) { + proxies, services, err := list(ctx, c, nil) + if err != nil { + return nil, err + } + + byWorkload := make(map[string]*networkingv1alpha.NetworkService, len(services)) + for i := range services { + byWorkload[workloadOf(services[i].Labels, services[i].Name)] = &services[i] + } + + infos := make(map[string]*Info, len(proxies)) + for i := range proxies { + name := workloadOf(proxies[i].Labels, proxies[i].Name) + infos[name] = newInfo(name, &proxies[i], byWorkload[name]) + } + return infos, nil +} + +// list fetches both published kinds with one call each. An empty match lists +// every URL the CLI published in the namespace — and only those: an HTTPProxy +// a user wrote by hand is theirs, and is never reported as a workload's URL. +func list(ctx context.Context, c client.Client, match labels.Set) ([]networkingv1alpha.HTTPProxy, []networkingv1alpha.NetworkService, error) { + selector := labels.SelectorFromSet(match) + if len(match) == 0 { + req, err := labels.NewRequirement(computev1alpha.WorkloadNameLabel, selection.Exists, nil) + if err != nil { + return nil, nil, fmt.Errorf("building URL selector: %w", err) + } + selector = labels.NewSelector().Add(*req) + } + + opts := []client.ListOption{ + client.InNamespace(util.ResourceNamespace), + client.MatchingLabelsSelector{Selector: selector}, + } + + var proxyList networkingv1alpha.HTTPProxyList + if err := c.List(ctx, &proxyList, opts...); err != nil { + if notPublished(err) { + return nil, nil, nil + } + return nil, nil, fmt.Errorf("listing URLs: %w", err) + } + + var serviceList networkingv1alpha.NetworkServiceList + if err := c.List(ctx, &serviceList, opts...); err != nil { + if notPublished(err) { + return proxyList.Items, nil, nil + } + return nil, nil, fmt.Errorf("listing URL backends: %w", err) + } + + return proxyList.Items, serviceList.Items, nil +} + +// notPublished reports whether an error means "there is nothing published +// here" rather than a real failure. A control plane that has never had these +// CRDs installed answers with a no-match error, and a client whose scheme +// lacks the kinds answers with a not-registered error; neither is something to +// report to a user who only asked for a URL. +func notPublished(err error) bool { + return err == nil || + k8serrors.IsNotFound(err) || + meta.IsNoMatchError(err) || + runtime.IsNotRegisteredError(err) +} + +// serviceFor picks the NetworkService belonging to a workload out of a list. +func serviceFor(services []networkingv1alpha.NetworkService, workloadName string) *networkingv1alpha.NetworkService { + for i := range services { + if workloadOf(services[i].Labels, services[i].Name) == workloadName { + return &services[i] + } + } + return nil +} + +// workloadOf reads the workload a published object belongs to from its labels, +// falling back to the object's own name, which ResourceName keeps in step. +func workloadOf(objLabels map[string]string, objName string) string { + if name := objLabels[computev1alpha.WorkloadNameLabel]; name != "" { + return name + } + return objName +} + +// newInfo assembles the user-facing view from the two objects. service may be +// nil: a proxy can exist for a moment before its backends do. +func newInfo(workloadName string, proxy *networkingv1alpha.HTTPProxy, service *networkingv1alpha.NetworkService) *Info { + info := &Info{ + WorkloadName: workloadName, + CanonicalHostname: proxy.Status.CanonicalHostname, + ProxyConditions: proxy.Status.Conditions, + Proxy: proxy, + Service: service, + } + + if c := util.FindCondition(proxy.Status.Conditions, networkingv1alpha.HTTPProxyConditionProgrammed); c != nil { + info.EdgeProgrammed = c.Status == metav1.ConditionTrue + } + + for _, h := range proxy.Spec.Hostnames { + info.CustomHostnames = append(info.CustomHostnames, string(h)) + } + + for _, h := range info.CustomHostnames { + info.Hostnames = append(info.Hostnames, hostnameInfo(proxy, h, false)) + } + if info.CanonicalHostname != "" { + info.Hostnames = append(info.Hostnames, hostnameInfo(proxy, info.CanonicalHostname, true)) + } + + info.URL, info.CertificateIssued, info.CertificateKnown = primary(info) + + if service != nil { + info.ServiceConditions = service.Status.Conditions + if len(service.Spec.Ports) > 0 { + p := service.Spec.Ports[0] + info.PortName, info.Port = p.Name, p.Port + info.Protocol = string(p.Protocol) + if info.Protocol == "" { + info.Protocol = string(networkingv1alpha.NetworkServiceProtocolTCP) + } + } + info.Backends = Backends{ + Cities: service.Status.Summary.Locations, + Total: service.Status.Summary.Members, + Healthy: service.Status.Summary.Healthy, + } + for _, l := range service.Status.Locations { + info.Locations = append(info.Locations, Location{ + City: l.Name, + Backends: l.Members, + Healthy: l.Healthy, + Serving: l.Serving, + }) + } + } + + return info +} + +// primary chooses the URL to show and reports the certificate state behind it. +// An active custom hostname is what the user wants to see; until one is +// active, the platform-managed hostname is the one that actually answers. +func primary(info *Info) (url string, certIssued, certKnown bool) { + var fallback *Hostname + var managed *Hostname + + for i := range info.Hostnames { + h := &info.Hostnames[i] + switch { + case h.Managed: + managed = h + case h.Active: + return h.URL, h.Certificate == certificateValid, h.Certificate != "" + case fallback == nil: + fallback = h + } + } + + if managed != nil { + return managed.URL, managed.Certificate == certificateValid, managed.Certificate != "" + } + if fallback != nil { + return fallback.URL, fallback.Certificate == certificateValid, fallback.Certificate != "" + } + return "", false, false +} + +// hostnameInfo derives one hostname's state from that hostname's own entry in +// the proxy's per-hostname statuses. Nothing else can say whether a hostname is +// serving: the proxy-level conditions are a roll-up over every hostname on the +// proxy, so one hostname waiting on a certificate turns the proxy's certificate +// condition False while every other hostname carries on serving normally. +// +// Two rules follow from that, and both matter to a user: +// +// - A custom hostname the platform has published no status for is pending, +// never active. It has just been attached and nothing about it has been +// checked yet, so `domains add` must show the DNS records and wait. +// - The platform-managed hostname may fall back to the proxy-level +// conditions, because it is the hostname the proxy is for. On a control +// plane that publishes per-hostname statuses for other hostnames, only the +// conditions that are True may inform it: a True roll-up covers every +// hostname, while a False one names none of them. +func hostnameInfo(proxy *networkingv1alpha.HTTPProxy, hostname string, managed bool) Hostname { + h := Hostname{ + Hostname: hostname, + URL: "https://" + hostname, + Managed: managed, + Status: statusPending, + } + + conditions := perHostnameConditions(proxy, hostname) + h.Conditions = conditions + + if len(conditions) == 0 { + if !managed { + return h + } + if len(proxy.Status.HostnameStatuses) == 0 { + conditions = proxy.Status.Conditions + } else { + conditions = satisfied(proxy.Status.Conditions) + } + } + + // Certificate state, in the server's own words. + if c := certificateCondition(conditions); c != nil { + if c.Status == metav1.ConditionTrue { + h.Certificate = certificateValid + } else { + h.Certificate = humanReason(c) + } + } + + // A hostname is active when nothing about it is blocking and the edge is + // carrying the configuration. + blocked := firstBlocking(conditions) + programmed := util.FindCondition(proxy.Status.Conditions, networkingv1alpha.HTTPProxyConditionProgrammed) + switch { + case blocked != nil: + h.Status = humanReason(blocked) + h.Detail = blocked.Message + case programmed != nil && programmed.Status == metav1.ConditionTrue: + h.Status = statusActive + h.Active = true + } + + return h +} + +// humanReason renders why a condition is not satisfied, for a user to read. +// +// It is the server's message, which is written for a human, and never the +// condition's reason, which is camelCase internal state. A condition with no +// message says only that the platform has not finished — which is "pending", +// the same plain word an unreported hostname gets. +func humanReason(c *metav1.Condition) string { + if c.Message != "" { + return c.Message + } + return statusPending +} + +// satisfied returns the conditions that are True. Used to let a proxy-level +// roll-up vouch for the managed hostname without letting it blame it. +func satisfied(conditions []metav1.Condition) []metav1.Condition { + var ok []metav1.Condition + for _, c := range conditions { + if c.Status == metav1.ConditionTrue { + ok = append(ok, c) + } + } + return ok +} + +// perHostnameConditions returns the conditions the server published for one +// hostname, or nil when it published none. +func perHostnameConditions(proxy *networkingv1alpha.HTTPProxy, hostname string) []metav1.Condition { + for _, s := range proxy.Status.HostnameStatuses { + if s.Hostname == hostname { + return s.Conditions + } + } + return nil +} + +// certificateCondition finds whichever certificate condition the given set +// carries: per-hostname status uses one type, proxy-level status another. +func certificateCondition(conditions []metav1.Condition) *metav1.Condition { + if c := util.FindCondition(conditions, networkingv1alpha.HostnameConditionCertificateReady); c != nil { + return c + } + return util.FindCondition(conditions, networkingv1alpha.HTTPProxyConditionCertificatesReady) +} + +// firstBlocking returns the first condition that is not True, so its reason and +// message can be shown verbatim. Unknown counts as blocking: the platform has +// not yet said the hostname works. +func firstBlocking(conditions []metav1.Condition) *metav1.Condition { + for i := range conditions { + if conditions[i].Status != metav1.ConditionTrue { + return &conditions[i] + } + } + return nil +} diff --git a/internal/cmd/compute/url/lookup_test.go b/internal/cmd/compute/url/lookup_test.go new file mode 100644 index 00000000..0927f0a3 --- /dev/null +++ b/internal/cmd/compute/url/lookup_test.go @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// Fixtures shared by the tests in this package. +const ( + testWorkloadName = "api" + testPortName = "http" + testCanonical = "a1b2c3d4.datumproxy.net" + testCanonicalURL = "https://" + testCanonical + testCustomHostname = "api.example.com" + testCustomURL = "https://" + testCustomHostname + + kindProxy = "HTTPProxy" + kindService = "NetworkService" +) + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := computev1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering compute scheme: %v", err) + } + if err := networkingv1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering networking scheme: %v", err) + } + return s +} + +func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithStatusSubresource(&networkingv1alpha.HTTPProxy{}, &networkingv1alpha.NetworkService{}). + WithObjects(objs...). + Build() +} + +func cond(condType string, status metav1.ConditionStatus, reason, message string) metav1.Condition { + return metav1.Condition{Type: condType, Status: status, Reason: reason, Message: message} +} + +// publishedProxy returns a proxy in the shape the platform reports once it is +// serving on the managed hostname alone. +func publishedProxy(workload, canonical string) *networkingv1alpha.HTTPProxy { + p := BuildHTTPProxy(workloadNamed(workload), testPortName, nil) + p.Status.CanonicalHostname = canonical + p.Status.Conditions = []metav1.Condition{ + cond(networkingv1alpha.HTTPProxyConditionAccepted, metav1.ConditionTrue, "Accepted", ""), + cond(networkingv1alpha.HTTPProxyConditionProgrammed, metav1.ConditionTrue, "Programmed", ""), + cond(networkingv1alpha.HTTPProxyConditionCertificatesReady, metav1.ConditionTrue, "AllCertificatesReady", ""), + } + return p +} + +func publishedService(workload string, port int32, locations ...networkingv1alpha.NetworkServiceLocationStatus) *networkingv1alpha.NetworkService { + s := BuildNetworkService(workloadNamed(workload), testPortName, port) + var members, healthy int32 + for _, l := range locations { + members += l.Members + healthy += l.Healthy + } + s.Status.Summary = networkingv1alpha.NetworkServiceSummary{ + Locations: int32(len(locations)), + Members: members, + Healthy: healthy, + } + s.Status.Locations = locations + s.Status.Conditions = []metav1.Condition{ + cond(networkingv1alpha.NetworkServiceMembersResolved, metav1.ConditionTrue, "MembersResolved", ""), + cond(networkingv1alpha.NetworkServiceReady, metav1.ConditionTrue, "Ready", ""), + } + return s +} + +func workloadNamed(name string) *computev1alpha.Workload { + w := testWorkload() + w.Name = name + w.UID = types.UID("uid-" + name) + return w +} + +func location(city string, members, healthy int32, serving bool) networkingv1alpha.NetworkServiceLocationStatus { + return networkingv1alpha.NetworkServiceLocationStatus{Name: city, Members: members, Healthy: healthy, Serving: serving} +} + +func TestForWorkloadPublished(t *testing.T) { + c := newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 2, 2, true), location("IAD", 2, 2, true)), + ) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info == nil { + t.Fatal("ForWorkload returned nil for a published workload") + } + + if info.URL != testCanonicalURL { + t.Errorf("URL = %q, want the managed hostname as https", info.URL) + } + if info.CanonicalHostname != testCanonical { + t.Errorf("CanonicalHostname = %q", info.CanonicalHostname) + } + if len(info.CustomHostnames) != 0 { + t.Errorf("CustomHostnames = %v, want none", info.CustomHostnames) + } + if !info.EdgeProgrammed || !info.CertificateIssued || !info.Live() { + t.Errorf("edge=%v cert=%v live=%v, want all true", info.EdgeProgrammed, info.CertificateIssued, info.Live()) + } + if info.Port != 8080 || info.PortName != testPortName || info.Protocol != "TCP" { + t.Errorf("backend port = %d/%s (%s), want 8080/http (TCP)", info.Port, info.PortName, info.Protocol) + } + want := Backends{Cities: 2, Total: 4, Healthy: 4} + if info.Backends != want { + t.Errorf("Backends = %+v, want %+v", info.Backends, want) + } + if len(info.Locations) != 2 || info.Locations[0] != (Location{City: "DFW", Backends: 2, Healthy: 2, Serving: true}) { + t.Errorf("Locations = %+v", info.Locations) + } + if info.Proxy == nil || info.Service == nil { + t.Error("raw objects should be carried for -o yaml") + } + if len(info.ServiceConditions) == 0 || len(info.ProxyConditions) == 0 { + t.Error("conditions should be carried through for rendering") + } +} + +func TestForWorkloadDegraded(t *testing.T) { + c := newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 2, 2, true), location("IAD", 2, 0, false)), + ) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info.Backends.Healthy != 2 || info.Backends.Total != 4 { + t.Errorf("Backends = %+v, want 2 of 4 healthy", info.Backends) + } + // A degraded backend set does not stop the URL from answering. + if !info.Live() { + t.Error("URL should still be live while one city is out of rotation") + } +} + +func TestForWorkloadCustomHostnamePreferredWhenActive(t *testing.T) { + proxy := publishedProxy(testWorkloadName, testCanonical) + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, testCustomHostname) + proxy.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: testCustomHostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionTrue, "Verified", ""), + cond(networkingv1alpha.HostnameConditionDNSRecordProgrammed, metav1.ConditionTrue, "RecordCreated", ""), + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionTrue, "CertificateIssued", ""), + }, + }} + + c := newFakeClient(t, proxy, publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true))) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info.URL != testCustomURL { + t.Errorf("URL = %q, want the custom hostname", info.URL) + } + if len(info.Hostnames) != 2 { + t.Fatalf("Hostnames = %+v, want the custom one and the managed one", info.Hostnames) + } + if info.Hostnames[0].Managed || !info.Hostnames[1].Managed { + t.Errorf("hostname order = %+v, want custom first and managed last", info.Hostnames) + } + if info.Hostnames[0].Status != statusActive || info.Hostnames[0].Certificate != certificateValid { + t.Errorf("custom hostname = %+v, want active with a valid certificate", info.Hostnames[0]) + } +} + +func TestForWorkloadPendingCustomHostnameFallsBackToManaged(t *testing.T) { + proxy := publishedProxy(testWorkloadName, testCanonical) + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, testCustomHostname) + proxy.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: testCustomHostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionFalse, "DomainNotVerified", "waiting for TXT record"), + }, + }} + + c := newFakeClient(t, proxy, publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true))) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info.URL != testCanonicalURL { + t.Errorf("URL = %q, want the managed hostname while the custom one is pending", info.URL) + } + // The server's own words, unedited — its message, never its reason. + if info.Hostnames[0].Status != "waiting for TXT record" || info.Hostnames[0].Detail != "waiting for TXT record" { + t.Errorf("custom hostname = %+v, want the server's message verbatim", info.Hostnames[0]) + } + if info.Hostnames[0].Status == "DomainNotVerified" { + t.Error("a raw camelCase condition reason must never be shown as a status") + } + if info.Hostnames[0].Active { + t.Error("a hostname with a blocking condition is not active") + } +} + +func TestForWorkloadNotPublished(t *testing.T) { + c := newFakeClient(t, publishedProxy("other", "zzz.datumproxy.net")) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("an unpublished workload is not an error, got: %v", err) + } + if info != nil { + t.Errorf("info = %+v, want nil for an unpublished workload", info) + } +} + +func TestForWorkloadWithoutTheCRDsInstalled(t *testing.T) { + // A control plane that never had the networking kinds answers with a + // no-match error. That means "no URLs here", not "the command failed". + s := runtime.NewScheme() + if err := computev1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering compute scheme: %v", err) + } + c := fake.NewClientBuilder().WithScheme(s).Build() + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("a control plane without the kinds is not an error, got: %v", err) + } + if info != nil { + t.Errorf("info = %+v, want nil", info) + } + + all, err := ForAll(context.Background(), c) + if err != nil { + t.Fatalf("ForAll returned error: %v", err) + } + if len(all) != 0 { + t.Errorf("ForAll = %v, want empty", all) + } +} + +func TestForWorkloadPropagatesTransportErrors(t *testing.T) { + boom := errors.New("connection refused") + c := interceptor.NewClient(newFakeClient(t), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return boom + }, + }) + + if _, err := ForWorkload(context.Background(), c, testWorkloadName); !errors.Is(err, boom) { + t.Fatalf("error = %v, want the transport error to propagate", err) + } + if _, err := ForAll(context.Background(), c); !errors.Is(err, boom) { + t.Fatalf("error = %v, want the transport error to propagate", err) + } +} + +func TestForAllRendersAWholeProjectInTwoListCalls(t *testing.T) { + objs := []client.Object{ + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 2, 2, true), location("IAD", 2, 2, true)), + publishedProxy("web", "e5f6a7b8.datumproxy.net"), + publishedService("web", 3000, location("DFW", 1, 1, true)), + publishedProxy("docs", "c9d0e1f2.datumproxy.net"), + publishedService("docs", 80, location("IAD", 1, 0, false)), + } + // A workload with no URL at all: it must simply be absent from the map. + worker := workloadNamed("worker") + + lists := 0 + c := interceptor.NewClient(newFakeClient(t, objs...), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lists++ + return cl.List(ctx, list, opts...) + }, + }) + + infos, err := ForAll(context.Background(), c) + if err != nil { + t.Fatalf("ForAll returned error: %v", err) + } + if lists != 2 { + t.Errorf("List calls = %d, want exactly 2 no matter how many workloads", lists) + } + if len(infos) != 3 { + t.Fatalf("infos = %d entries, want 3", len(infos)) + } + if infos[worker.Name] != nil { + t.Errorf("unpublished workload %q should be absent from the map", worker.Name) + } + + if got := infos[testWorkloadName]; got == nil || got.URL != testCanonicalURL || got.Backends.Healthy != 4 { + t.Errorf("api = %+v", got) + } + if got := infos["web"]; got == nil || got.Port != 3000 || got.Backends.Total != 1 { + t.Errorf("web = %+v", got) + } + if got := infos["docs"]; got == nil || got.Backends.Healthy != 0 || len(got.Locations) != 1 { + t.Errorf("docs = %+v", got) + } + for name, info := range infos { + if info.WorkloadName != name { + t.Errorf("info keyed %q carries workload name %q", name, info.WorkloadName) + } + } +} + +func TestForWorkloadWithoutBackendsYet(t *testing.T) { + // The proxy can exist for a moment before the service does. That is a URL + // with no backends, not a lookup failure. + c := newFakeClient(t, publishedProxy(testWorkloadName, testCanonical)) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info == nil { + t.Fatal("info = nil, want a URL with no backends") + } + if info.Backends != (Backends{}) || info.Service != nil { + t.Errorf("Backends = %+v, Service = %v, want empty", info.Backends, info.Service) + } + if info.Port != 0 { + t.Errorf("Port = %d, want 0 when no backends are known", info.Port) + } +} + +func TestNamespaceIsAlwaysTheProjectNamespace(t *testing.T) { + if util.ResourceNamespace != BuildHTTPProxy(testWorkload(), testPortName, nil).Namespace { + t.Error("published objects must live in the project namespace") + } +} + +func TestLookupsIgnoreHandWrittenProxies(t *testing.T) { + // A proxy the user wrote themselves is theirs. Reporting it as a + // workload's URL would make `destroy` offer to delete it. + handWritten := BuildHTTPProxy(workloadNamed(testWorkloadName), testPortName, nil) + handWritten.Name = "hand-written" + handWritten.Labels = nil + handWritten.OwnerReferences = nil + handWritten.Status.CanonicalHostname = testCanonical + + c := newFakeClient(t, handWritten) + + infos, err := ForAll(context.Background(), c) + if err != nil { + t.Fatalf("ForAll returned error: %v", err) + } + if len(infos) != 0 { + t.Errorf("ForAll = %v, want nothing — that proxy is not a workload URL", infos) + } + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info != nil { + t.Errorf("ForWorkload = %+v, want nil", info) + } +} + +// TestObjectsIsTheEscapeHatchToTheRealState pins the promise the spec makes +// twice over: plain language in normal output, and `-o yaml` showing the real +// objects. Without a way out of this package to them, `domains +// -o yaml` can only re-print the same summary the table already showed. +func TestObjectsIsTheEscapeHatchToTheRealState(t *testing.T) { + c := newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)), + ) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + + objs := info.Objects() + if objs == nil { + t.Fatal("Objects() = nil, want the real objects behind the URL") + } + if objs.HTTPProxy != info.Proxy || objs.NetworkService != info.Service { + t.Error("Objects() must hand back the objects themselves, not a summary of them") + } + + // What `-o yaml` would render: the real spec and status, not the CLI's view. + var out bytes.Buffer + if err := util.PrintYAML(&out, objs); err != nil { + t.Fatalf("rendering the objects: %v", err) + } + got := out.String() + for _, want := range []string{"httpProxy", "networkService", testCanonical, "canonicalHostname"} { + if !strings.Contains(got, want) { + t.Errorf("-o yaml output missing %q:\n%s", want, got) + } + } + + // And the default structured view still carries none of the machinery. + var human bytes.Buffer + if err := util.PrintJSON(&human, info); err != nil { + t.Fatalf("rendering the URL: %v", err) + } + for _, unwanted := range []string{"httpProxy", "networkService"} { + if strings.Contains(human.String(), unwanted) { + t.Errorf("%q leaked into the default view:\n%s", unwanted, human.String()) + } + } + + // Nothing to show reads as nothing, so a caller needs no second check. + var missing *Info + if missing.Objects() != nil { + t.Error("Objects() on a workload with no URL must be nil") + } + if (&Info{}).Objects() != nil { + t.Error("Objects() with neither object must be nil") + } +} diff --git a/internal/cmd/compute/url/publish.go b/internal/cmd/compute/url/publish.go new file mode 100644 index 00000000..28018d3c --- /dev/null +++ b/internal/cmd/compute/url/publish.go @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "context" + "fmt" + "io" + "reflect" + "strings" + "time" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + // pollInterval matches the rollout watcher, so a deploy that publishes and + // a deploy that only rolls out feel the same. + pollInterval = 2 * time.Second + + // blockingGrace is how long to wait before repeating what the server says + // is holding the URL up. Every object starts out reporting "waiting for + // controller", and echoing that immediately is noise, not diagnosis. + blockingGrace = 20 * time.Second + + // labelWidth aligns the progress labels ("Backends", "Edge", + // "Certificate") in a fixed column. + labelWidth = 12 + + // maxReadFailures is how many consecutive failed reads the wait rides out + // before giving up and reporting the last one. + // + // A read that fails once is a blip and the next tick is a better answer + // than failing a deploy that is going fine. A read that fails every time is + // something else — a missing list permission is the everyday one — and + // polling it forever turns `deploy --http-port` into a silent hang. Three + // ticks is a few seconds: long enough for a blip, short enough that a user + // is told what is wrong rather than left watching a cursor. + maxReadFailures = 3 + + // maxWait bounds the whole wait, so no caller can hang forever even while + // the control plane answers happily and simply never finishes. It is far + // longer than publishing takes; reaching it means something is stuck. + maxWait = 15 * time.Minute +) + +// Publish creates or updates the two objects that put a workload on a URL and +// waits until that URL answers, printing progress as the platform reports it: +// +// Backends 4 healthy across DFW, IAD +// Edge programmed +// Certificate issued +// +// It prints progress lines only. The caller prints whatever heading precedes +// them and the final URL — the URL is the deliverable and belongs to the +// command that was asked for it. +// +// The NetworkService is written before the HTTPProxy: a proxy naming a service +// that does not exist yet reports a missing backend, which the user would see +// as a spurious failure. +// +// hostnames are custom hostnames; pass nil for the managed URL alone. +// +// Cancelling ctx (Ctrl-C) detaches: publishing continues on the platform, a +// note says how to pick it up again, and Publish returns (nil, nil). A nil +// Info with a nil error means "still going, we stopped watching" — never an +// error, and never a reason for the caller to fail. +func Publish( + ctx context.Context, + out io.Writer, + c client.Client, + w *computev1alpha.Workload, + portName string, + port int32, + hostnames []string, +) (*Info, error) { + // A write interrupted by Ctrl-C is a detach like any other. Reporting the + // cancelled context as a failure would exit 1 on a keystroke the user was + // told is safe. + if err := Declare(ctx, c, w, portName, port, hostnames); err != nil { + if ctx.Err() != nil { + detached(out, w.Name) + return nil, nil + } + return nil, err + } + + return Wait(ctx, out, c, w.Name) +} + +// Declare writes the two objects that put a workload on a URL and returns +// without waiting for that URL to answer. +// +// It is the half of Publish a caller wants when the URL has to be declared +// early — a deploy declares it alongside the workload so that backends +// register as instances come up, then waits only once the rollout is done. +// It prints nothing: at the point it runs there is nothing to report yet. +// +// The NetworkService is written before the HTTPProxy: a proxy naming a service +// that does not exist yet reports a missing backend, which the user would see +// as a spurious failure. +// +// hostnames are custom hostnames; pass nil for the managed URL alone. +func Declare( + ctx context.Context, + c client.Client, + w *computev1alpha.Workload, + portName string, + port int32, + hostnames []string, +) error { + if err := applyService(ctx, c, BuildNetworkService(w, portName, port)); err != nil { + return err + } + return applyProxy(ctx, c, BuildHTTPProxy(w, portName, hostnames)) +} + +// detached prints the note that says publishing carries on without us, and how +// to pick it back up. +func detached(out io.Writer, workloadName string) { + fmt.Fprintf(out, "\nDetached. Publishing continues in the background.\n") + fmt.Fprintf(out, " Check it with: datumctl compute open %s\n", workloadName) +} + +// Wait polls until the workload's URL is live, printing each stage as it +// lands. It is the half of Publish that a caller which already called Declare +// still needs. +// +// It always terminates: Ctrl-C detaches, a control plane that cannot be read +// gives up after maxReadFailures and returns what it was told, and the whole +// wait ends at maxWait however healthy the polling looks. +// +// Cancelling ctx (Ctrl-C) detaches: a note says how to pick the URL up again +// and Wait returns (nil, nil), which is never an error. +func Wait(ctx context.Context, out io.Writer, c client.Client, workloadName string) (*Info, error) { + p := &progress{out: out, started: time.Now(), seen: map[string]bool{}} + + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + deadline := time.NewTimer(maxWait) + defer deadline.Stop() + + var failures int + for { + info, done, err := p.check(ctx, c, workloadName) + switch { + case done: + // A URL that came up on the same tick the user interrupted is + // still a URL, and worth more to them than a detach note. + return info, nil + + case ctx.Err() != nil: + // Otherwise detaching wins over whatever the last read said: a + // read that was cancelled failed because the user asked it to. + detached(out, workloadName) + return nil, nil + + case err != nil: + failures++ + if failures >= maxReadFailures { + return nil, fmt.Errorf("checking the URL for %q: %w", workloadName, err) + } + + default: + failures = 0 + } + + select { + case <-ctx.Done(): + detached(out, workloadName) + return nil, nil + + case <-deadline.C: + return nil, fmt.Errorf( + "the URL for %q was still not answering after %s — it may yet come up; check it with: datumctl compute open %s", + workloadName, maxWait, workloadName) + + case <-ticker.C: + } + } +} + +// progress prints each stage of publishing exactly once, and repeats nothing. +type progress struct { + out io.Writer + started time.Time + seen map[string]bool +} + +// check reads current state and reports whether the URL is live, along with +// whatever went wrong reading it. The caller decides how much failure to ride +// out; a single failure means nothing, since the objects were written a moment +// ago and the next tick is a better answer than failing a deploy that is going +// fine. +// +// A workload that is simply not published yet is not a failure: there is +// nothing to report and nothing to give up over. +func (p *progress) check(ctx context.Context, c client.Client, workloadName string) (*Info, bool, error) { + info, err := ForWorkload(ctx, c, workloadName) + if err != nil { + return nil, false, err + } + if info == nil { + return nil, false, nil + } + + if info.Backends.Healthy > 0 { + p.line("Backends", fmt.Sprintf("%d healthy across %s", info.Backends.Healthy, strings.Join(servingCities(info), ", "))) + } + if info.EdgeProgrammed { + p.line("Edge", "programmed") + } + if info.CertificateIssued { + p.line("Certificate", "issued") + } + + if info.Live() { + return info, true, nil + } + + if time.Since(p.started) > blockingGrace { + p.blocking(info) + } + return nil, false, nil +} + +// line prints one progress row, skipping rows already printed with the same +// value. A count that changes as instances register is worth reprinting; a +// repeat of the same fact is not. +func (p *progress) line(label, value string) { + key := label + "\x00" + value + if p.seen[key] { + return + } + p.seen[key] = true + fmt.Fprintf(p.out, " %-*s %s\n", labelWidth, label, value) +} + +// blocking echoes, verbatim and once each, whatever the server says is holding +// the URL up. The CLI never interprets a reason: a condition the CLI has never +// heard of shows up here without a release. +func (p *progress) blocking(info *Info) { + p.blockingFrom(info.ServiceConditions, networkingv1alpha.NetworkServiceReady) + p.blockingFrom(info.ProxyConditions, networkingv1alpha.HTTPProxyConditionProgrammed) + for _, h := range info.Hostnames { + if !h.Active && h.Detail != "" { + p.note(fmt.Sprintf("%s: %s", h.Hostname, h.Detail)) + } + } +} + +func (p *progress) blockingFrom(conditions []metav1.Condition, condType string) { + reason, message, blocked := util.ReadinessBlock(conditions, condType) + if !blocked || reason == "" { + return + } + p.note(HumanBlock(reason, message)) +} + +func (p *progress) note(text string) { + if p.seen[text] { + return + } + p.seen[text] = true + fmt.Fprintf(p.out, " %-*s %s\n", labelWidth, "", text) +} + +// servingCities names the cities taking traffic, for the backends line. It +// falls back to every city with members so the line is never empty while the +// platform is still deciding what is in rotation. +func servingCities(info *Info) []string { + serving := make([]string, 0, len(info.Locations)) + all := make([]string, 0, len(info.Locations)) + for _, l := range info.Locations { + all = append(all, l.City) + if l.Serving { + serving = append(serving, l.City) + } + } + if len(serving) > 0 { + return serving + } + return all +} + +// applyService creates the NetworkService, or brings an existing one in line +// with what the workload now declares. +func applyService(ctx context.Context, c client.Client, desired *networkingv1alpha.NetworkService) error { + var existing networkingv1alpha.NetworkService + err := c.Get(ctx, client.ObjectKeyFromObject(desired), &existing) + if k8serrors.IsNotFound(err) { + if err := c.Create(ctx, desired); err != nil { + return fmt.Errorf("publishing backends for %q: %w", desired.Name, err) + } + return nil + } + if err != nil { + return fmt.Errorf("reading published backends for %q: %w", desired.Name, err) + } + + if reflect.DeepEqual(existing.Spec, desired.Spec) && metaCurrent(&existing, desired) { + return nil + } + existing.Spec = desired.Spec + adoptMeta(&existing, desired) + if err := c.Update(ctx, &existing); err != nil { + return fmt.Errorf("updating published backends for %q: %w", desired.Name, err) + } + return nil +} + +// applyProxy creates the HTTPProxy, or brings an existing one in line with +// what the workload now declares. +func applyProxy(ctx context.Context, c client.Client, desired *networkingv1alpha.HTTPProxy) error { + var existing networkingv1alpha.HTTPProxy + err := c.Get(ctx, client.ObjectKeyFromObject(desired), &existing) + if k8serrors.IsNotFound(err) { + if err := c.Create(ctx, desired); err != nil { + return fmt.Errorf("publishing URL for %q: %w", desired.Name, err) + } + return nil + } + if err != nil { + return fmt.Errorf("reading published URL for %q: %w", desired.Name, err) + } + + if reflect.DeepEqual(existing.Spec, desired.Spec) && metaCurrent(&existing, desired) { + return nil + } + existing.Spec = desired.Spec + adoptMeta(&existing, desired) + if err := c.Update(ctx, &existing); err != nil { + return fmt.Errorf("updating published URL for %q: %w", desired.Name, err) + } + return nil +} + +// metaCurrent reports whether an existing object already carries the labels +// and owner reference the desired object declares. +func metaCurrent(existing, desired client.Object) bool { + for k, v := range desired.GetLabels() { + if existing.GetLabels()[k] != v { + return false + } + } + return hasOwner(existing, desired) +} + +// hasOwner reports whether existing already references the desired owner. +func hasOwner(existing, desired client.Object) bool { + owners := desired.GetOwnerReferences() + if len(owners) == 0 { + return true + } + for _, o := range existing.GetOwnerReferences() { + if o.UID == owners[0].UID && o.Kind == owners[0].Kind { + return true + } + } + return false +} + +// adoptMeta merges the labels and owner reference onto an object that already +// exists, without dropping anything a user put there. +func adoptMeta(existing, desired client.Object) { + labels := existing.GetLabels() + if labels == nil { + labels = map[string]string{} + } + for k, v := range desired.GetLabels() { + labels[k] = v + } + existing.SetLabels(labels) + + if !hasOwner(existing, desired) { + existing.SetOwnerReferences(append(existing.GetOwnerReferences(), desired.GetOwnerReferences()...)) + } +} + +// Unpublish removes a workload's URL: the HTTPProxy first, then the +// NetworkService behind it. Deleting the service first would leave the proxy +// reporting a missing backend for as long as the delete takes. +// +// Objects that are not there are not an error — unpublishing something that +// was never published is a no-op, which is what `destroy` needs. +func Unpublish(ctx context.Context, c client.Client, workloadName string) error { + sel := client.MatchingLabels{computev1alpha.WorkloadNameLabel: workloadName} + ns := client.InNamespace(util.ResourceNamespace) + + if err := c.DeleteAllOf(ctx, &networkingv1alpha.HTTPProxy{}, ns, sel); err != nil && !notPublished(err) { + return fmt.Errorf("removing URL for %q: %w", workloadName, err) + } + if err := c.DeleteAllOf(ctx, &networkingv1alpha.NetworkService{}, ns, sel); err != nil && !notPublished(err) { + return fmt.Errorf("removing URL backends for %q: %w", workloadName, err) + } + + return nil +} diff --git a/internal/cmd/compute/url/publish_failure_test.go b/internal/cmd/compute/url/publish_failure_test.go new file mode 100644 index 00000000..413f0b74 --- /dev/null +++ b/internal/cmd/compute/url/publish_failure_test.go @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// failDeleteOf returns interceptor funcs that fail DeleteAllOf for one kind +// and pass everything else through, which is what a partial permission looks +// like from the CLI's side. +func failDeleteOf(kind string, boom error) interceptor.Funcs { + return interceptor.Funcs{ + DeleteAllOf: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { + if kindOf(obj) == kind { + return boom + } + return c.DeleteAllOf(ctx, obj, opts...) + }, + } +} + +func objectExists(t *testing.T, c client.Client, obj client.Object) bool { + t.Helper() + err := c.Get(context.Background(), types.NamespacedName{Namespace: util.ResourceNamespace, Name: testWorkloadName}, obj) + if err == nil { + return true + } + if !notPublished(err) { + t.Fatalf("reading %s back: %v", kindOf(obj), err) + } + return false +} + +// TestUnpublishStopsWhenTheURLCannotBeRemoved: a partial delete has to be +// reported, and it has to stop. Removing the backends out from under a proxy +// that is still routing to them is the one ordering this package exists to +// prevent, so a failure on the proxy must not be followed by deleting the +// service anyway. +func TestUnpublishStopsWhenTheURLCannotBeRemoved(t *testing.T) { + boom := errors.New("forbidden") + c := interceptor.NewClient(newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)), + ), failDeleteOf(kindProxy, boom)) + + err := Unpublish(context.Background(), c, testWorkloadName) + if err == nil { + t.Fatal("a URL that could not be removed must be reported") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure to survive wrapping", err) + } + if !strings.Contains(err.Error(), testWorkloadName) { + t.Errorf("error = %q, want it to name the workload", err) + } + + if !objectExists(t, c, &networkingv1alpha.NetworkService{}) { + t.Error("the backends were deleted behind a proxy that is still routing to them") + } + // And the URL is still findable, so a retry has something to act on. + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil || info == nil { + t.Fatalf("the URL must remain visible for a retry, got info=%v err=%v", info, err) + } +} + +// The other half of a partial delete: the URL is gone and its backends are +// not. This is the one that leaves an object behind with no proxy pointing at +// it, so the error has to say which of the two failed. +func TestUnpublishReportsAFailureToRemoveTheBackends(t *testing.T) { + boom := errors.New("forbidden") + c := interceptor.NewClient(newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)), + ), failDeleteOf(kindService, boom)) + + err := Unpublish(context.Background(), c, testWorkloadName) + if err == nil { + t.Fatal("backends that could not be removed must be reported") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's failure to survive wrapping", err) + } + if !strings.Contains(err.Error(), "backends") { + t.Errorf("error = %q, want it to distinguish the backends from the URL itself", err) + } + + if objectExists(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("the proxy should already be gone: it is deleted first") + } + if !objectExists(t, c, &networkingv1alpha.NetworkService{}) { + t.Fatal("test is not exercising the leftover-backends case") + } + + // The state this leaves behind: nothing that looks up a workload's URL can + // see the leftover backends, because lookups key on the proxy. + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info != nil { + t.Errorf("info = %+v, want nil — the proxy is gone", info) + } +} + +// A second Unpublish after a partial failure has to finish the job. This is +// the retry every caller advertises, and it is idempotent over the half that +// already succeeded. +func TestUnpublishRetryFinishesAPartialDelete(t *testing.T) { + fail := true + c := interceptor.NewClient(newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)), + ), interceptor.Funcs{ + DeleteAllOf: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { + if fail && kindOf(obj) == kindService { + return errors.New("forbidden") + } + return cl.DeleteAllOf(ctx, obj, opts...) + }, + }) + + if err := Unpublish(context.Background(), c, testWorkloadName); err == nil { + t.Fatal("expected the first attempt to fail on the backends") + } + + fail = false + if err := Unpublish(context.Background(), c, testWorkloadName); err != nil { + t.Fatalf("the retry must finish the job, got: %v", err) + } + if objectExists(t, c, &networkingv1alpha.NetworkService{}) { + t.Error("the leftover backends survived the retry") + } +} + +// TestPublishFailsOnTheProxyAfterWritingTheBackends pins what a failed publish +// leaves behind, and that the error names the URL rather than the backends +// that did get written — a user reading it has to know which step failed. +func TestPublishFailsOnTheProxyAfterWritingTheBackends(t *testing.T) { + boom := errors.New("admission webhook denied the request") + c := interceptor.NewClient(newFakeClient(t), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if kindOf(obj) == kindProxy { + return boom + } + return cl.Create(ctx, obj, opts...) + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + var out bytes.Buffer + info, err := Publish(ctx, &out, c, workloadNamed(testWorkloadName), testPortName, 8080, nil) + if err == nil { + t.Fatal("a proxy that could not be created must fail the publish") + } + if !errors.Is(err, boom) { + t.Errorf("error = %v, want the server's message to reach the user", err) + } + if !strings.Contains(err.Error(), "URL") { + t.Errorf("error = %q, want it to say the URL is what failed", err) + } + if info != nil { + t.Errorf("info = %+v, want nil", info) + } + if out.Len() != 0 { + t.Errorf("nothing was published, so no progress may be printed:\n%s", out.String()) + } + + // A retry has to be able to succeed, so the half that was written stays. + if !objectExists(t, c, &networkingv1alpha.NetworkService{}) { + t.Error("the backends should remain, so a retry is an update and not a rebuild") + } +} + +// A failure to write the backends must stop before the proxy is created: a +// proxy naming a service that does not exist reports a broken backend, which +// is the spurious failure the write ordering exists to avoid. +func TestPublishDoesNotCreateAProxyWithoutBackends(t *testing.T) { + boom := errors.New("quota exceeded") + rec := &recorder{} + c := interceptor.NewClient(newFakeClient(t), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + rec.creates = append(rec.creates, kindOf(obj)) + if kindOf(obj) == kindService { + return boom + } + return cl.Create(ctx, obj, opts...) + }, + }) + + if _, err := Publish(context.Background(), &bytes.Buffer{}, c, workloadNamed(testWorkloadName), testPortName, 8080, nil); !errors.Is(err, boom) { + t.Fatalf("error = %v, want the create failure", err) + } + if len(rec.creates) != 1 || rec.creates[0] != kindService { + t.Errorf("creates = %v, want the backends attempted and nothing after", rec.creates) + } + if objectExists(t, c, &networkingv1alpha.HTTPProxy{}) { + t.Error("a proxy was created with no backends to point at") + } +} + +// TestPublishReportsAReadFailureRatherThanOverwriting: an existing object that +// cannot be read is not an object that can safely be replaced. Publishing has +// to stop, not fall through to a blind create or an update built on nothing. +func TestPublishReportsAReadFailureRatherThanOverwriting(t *testing.T) { + boom := errors.New("connection reset") + rec := &recorder{} + c := interceptor.NewClient(newFakeClient(t), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return boom + }, + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + rec.creates = append(rec.creates, kindOf(obj)) + return cl.Create(ctx, obj, opts...) + }, + Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + rec.updates = append(rec.updates, kindOf(obj)) + return cl.Update(ctx, obj, opts...) + }, + }) + + if _, err := Publish(context.Background(), &bytes.Buffer{}, c, workloadNamed(testWorkloadName), testPortName, 8080, nil); !errors.Is(err, boom) { + t.Fatalf("error = %v, want the read failure", err) + } + if len(rec.creates) != 0 || len(rec.updates) != 0 { + t.Errorf("creates = %v, updates = %v, want nothing written on an unreadable control plane", rec.creates, rec.updates) + } +} + +// TestPublishStopsWhenTheURLCanNeverBeRead: publishing treats a read failure +// as transient and polls again, which is right for the blip it was written +// for. Nothing escalates, though, so a failure that is not transient — a +// missing list permission is the everyday one — never becomes anything. +// +// The context here is unbounded on purpose: that is the one a deploy passes, +// and the only reason the rest of this package's tests do not hang on this is +// that they all pass a deadline. +func TestPublishStopsWhenTheURLCanNeverBeRead(t *testing.T) { + c := interceptor.NewClient(newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)), + ), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errors.New("httpproxies.networking.datumapis.com is forbidden") + }, + }) + + type result struct { + info *Info + err error + out string + } + done := make(chan result, 1) + go func() { + var out bytes.Buffer + info, err := Publish(context.Background(), &out, c, workloadNamed(testWorkloadName), testPortName, 8080, nil) + done <- result{info, err, out.String()} + }() + + select { + case got := <-done: + if got.err == nil && !strings.Contains(got.out, "forbidden") { + t.Errorf("publishing gave up silently; output:\n%s", got.out) + } + if got.err != nil && !strings.Contains(got.err.Error(), "forbidden") { + t.Errorf("error = %v, want the server's own words for why it gave up", got.err) + } + if got.info != nil { + t.Errorf("info = %+v, want nil — the URL was never read", got.info) + } + case <-time.After(10 * time.Second): + t.Fatal("Publish never returned: a deploy against a control plane it cannot read hangs indefinitely") + } +} + +// TestPublishRidesOutATransientReadFailure is the other half of giving up: a +// read that fails once must still be a blip. The objects were written a moment +// ago, and failing a deploy on the first hiccup would be worse than the hang +// this bound exists to stop. +func TestPublishRidesOutATransientReadFailure(t *testing.T) { + var lists int + c := interceptor.NewClient(newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)), + ), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lists++ + if lists == 1 { + return errors.New("etcdserver: request timed out") + } + return cl.List(ctx, list, opts...) + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var out bytes.Buffer + info, err := Publish(ctx, &out, c, workloadNamed(testWorkloadName), testPortName, 8080, nil) + if err != nil { + t.Fatalf("one failed read must not fail a publish, got: %v", err) + } + if info == nil || info.URL != testCanonicalURL { + t.Fatalf("info = %+v, want the live URL on the next tick", info) + } + if strings.Contains(out.String(), "timed out") { + t.Errorf("a blip was reported to the user:\n%s", out.String()) + } +} + +// TestPublishDetachesWhenInterruptedMidWrite: Ctrl-C is a detach wherever it +// lands, including during the writes that precede the wait. Returning the +// cancelled context as an error exits 1 on a keystroke the command's own help +// says is safe. +func TestPublishDetachesWhenInterruptedMidWrite(t *testing.T) { + for _, tc := range []struct{ name, kind string }{ + {"during the backends", kindService}, + {"during the URL", kindProxy}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // The interrupt arrives while this write is in flight, so the write + // fails with the cancelled context — exactly as a real client does. + c := interceptor.NewClient(newFakeClient(t), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if kindOf(obj) == tc.kind { + cancel() + return context.Canceled + } + return cl.Create(ctx, obj, opts...) + }, + }) + + var out bytes.Buffer + info, err := Publish(ctx, &out, c, workloadNamed(testWorkloadName), testPortName, 8080, nil) + if err != nil { + t.Fatalf("detaching is not an error, got: %v", err) + } + if info != nil { + t.Errorf("info = %+v, want nil when we stopped watching", info) + } + if !strings.Contains(out.String(), "Detached") { + t.Errorf("output = %q, want the same detach note the wait prints", out.String()) + } + if !strings.Contains(out.String(), "datumctl compute open "+testWorkloadName) { + t.Errorf("output = %q, want a pointer to how to pick it up again", out.String()) + } + }) + } +} diff --git a/internal/cmd/compute/url/publish_test.go b/internal/cmd/compute/url/publish_test.go new file mode 100644 index 00000000..fb7f56ce --- /dev/null +++ b/internal/cmd/compute/url/publish_test.go @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// recorder records the order of writes, which is the part of publishing that +// has to be right: the service exists before anything references it. +type recorder struct { + creates []string + updates []string + deletes []string +} + +func (r *recorder) funcs() interceptor.Funcs { + return interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + r.creates = append(r.creates, kindOf(obj)) + return c.Create(ctx, obj, opts...) + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + r.updates = append(r.updates, kindOf(obj)) + return c.Update(ctx, obj, opts...) + }, + DeleteAllOf: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { + r.deletes = append(r.deletes, kindOf(obj)) + return c.DeleteAllOf(ctx, obj, opts...) + }, + } +} + +func kindOf(obj client.Object) string { + switch obj.(type) { + case *networkingv1alpha.NetworkService: + return kindService + case *networkingv1alpha.HTTPProxy: + return kindProxy + default: + return "other" + } +} + +func TestPublishCreatesBackendsBeforeTheProxy(t *testing.T) { + rec := &recorder{} + c := interceptor.NewClient(newFakeClient(t), rec.funcs()) + + // Nothing ever reports the URL live here, so the wait runs until the + // context is cancelled — the Ctrl-C path. + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + + var out bytes.Buffer + info, err := Publish(ctx, &out, c, workloadNamed(testWorkloadName), testPortName, 8080, nil) + if err != nil { + t.Fatalf("detaching is not an error, got: %v", err) + } + if info != nil { + t.Errorf("info = %+v, want nil when we stopped watching", info) + } + + want := []string{kindService, kindProxy} + if len(rec.creates) != 2 || rec.creates[0] != want[0] || rec.creates[1] != want[1] { + t.Fatalf("creates = %v, want %v — a proxy naming a missing service reports a broken backend", rec.creates, want) + } + + if !strings.Contains(out.String(), "Detached") { + t.Errorf("output = %q, want a detach note", out.String()) + } + if !strings.Contains(out.String(), "datumctl compute open api") { + t.Errorf("output = %q, want a pointer to `datumctl compute open`", out.String()) + } + + var svc networkingv1alpha.NetworkService + if err := c.Get(context.Background(), types.NamespacedName{Namespace: util.ResourceNamespace, Name: testWorkloadName}, &svc); err != nil { + t.Fatalf("network service was not created: %v", err) + } + if svc.Spec.Ports[0].Port != 8080 { + t.Errorf("port = %d, want 8080", svc.Spec.Ports[0].Port) + } + + var proxy networkingv1alpha.HTTPProxy + if err := c.Get(context.Background(), types.NamespacedName{Namespace: util.ResourceNamespace, Name: testWorkloadName}, &proxy); err != nil { + t.Fatalf("proxy was not created: %v", err) + } + if proxy.Spec.Rules[0].Backends[0].NetworkService.Name != testWorkloadName { + t.Errorf("backend = %+v, want a reference to the service", proxy.Spec.Rules[0].Backends[0]) + } +} + +func TestPublishReturnsWhenTheURLIsLive(t *testing.T) { + rec := &recorder{} + c := interceptor.NewClient(newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 2, 2, true), location("IAD", 2, 2, true)), + ), rec.funcs()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var out bytes.Buffer + info, err := Publish(ctx, &out, c, workloadNamed(testWorkloadName), testPortName, 8080, nil) + if err != nil { + t.Fatalf("Publish returned error: %v", err) + } + if info == nil || info.URL != testCanonicalURL { + t.Fatalf("info = %+v, want the live URL", info) + } + + // Already published and unchanged: no writes at all. + if len(rec.creates) != 0 || len(rec.updates) != 0 { + t.Errorf("creates = %v, updates = %v, want none for an unchanged workload", rec.creates, rec.updates) + } + + got := out.String() + for _, want := range []string{ + "Backends 4 healthy across DFW, IAD", + "Edge programmed", + "Certificate issued", + } { + if !strings.Contains(got, want) { + t.Errorf("output missing %q:\n%s", want, got) + } + } + // The URL is the caller's line to print, not this package's. + if strings.Contains(got, "https://") { + t.Errorf("Publish should not print the URL itself:\n%s", got) + } +} + +func TestPublishUpdatesAChangedPort(t *testing.T) { + rec := &recorder{} + c := interceptor.NewClient(newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)), + ), rec.funcs()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if _, err := Publish(ctx, &bytes.Buffer{}, c, workloadNamed(testWorkloadName), testPortName, 9090, nil); err != nil { + t.Fatalf("Publish returned error: %v", err) + } + + var svc networkingv1alpha.NetworkService + if err := c.Get(context.Background(), types.NamespacedName{Namespace: util.ResourceNamespace, Name: testWorkloadName}, &svc); err != nil { + t.Fatalf("getting service: %v", err) + } + if svc.Spec.Ports[0].Port != 9090 { + t.Errorf("port = %d, want the new port 9090", svc.Spec.Ports[0].Port) + } + if len(rec.updates) != 1 || rec.updates[0] != kindService { + t.Errorf("updates = %v, want the service alone", rec.updates) + } +} + +func TestPublishAdoptsAnExistingUnlabelledObject(t *testing.T) { + // An object written before the labels existed must be brought in line, or + // lookups would never find it again. + svc := BuildNetworkService(workloadNamed(testWorkloadName), testPortName, 8080) + svc.Labels = nil + svc.OwnerReferences = nil + proxy := BuildHTTPProxy(workloadNamed(testWorkloadName), testPortName, nil) + proxy.Labels = nil + proxy.OwnerReferences = nil + + c := newFakeClient(t, svc, proxy) + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + if _, err := Publish(ctx, &bytes.Buffer{}, c, workloadNamed(testWorkloadName), testPortName, 8080, nil); err != nil { + t.Fatalf("Publish returned error: %v", err) + } + + var got networkingv1alpha.NetworkService + if err := c.Get(context.Background(), types.NamespacedName{Namespace: util.ResourceNamespace, Name: testWorkloadName}, &got); err != nil { + t.Fatalf("getting service: %v", err) + } + assertPublishedLabels(t, got.Labels, workloadNamed(testWorkloadName)) + assertOwnerRef(t, got.OwnerReferences, workloadNamed(testWorkloadName)) +} + +func TestUnpublishRemovesTheProxyFirst(t *testing.T) { + rec := &recorder{} + c := interceptor.NewClient(newFakeClient(t, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)), + publishedProxy("web", "e5f6a7b8.datumproxy.net"), + publishedService("web", 3000, location("DFW", 1, 1, true)), + ), rec.funcs()) + + if err := Unpublish(context.Background(), c, testWorkloadName); err != nil { + t.Fatalf("Unpublish returned error: %v", err) + } + + want := []string{kindProxy, kindService} + if len(rec.deletes) != 2 || rec.deletes[0] != want[0] || rec.deletes[1] != want[1] { + t.Fatalf("deletes = %v, want %v — removing the backends first leaves the proxy reporting a missing backend", rec.deletes, want) + } + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info != nil { + t.Errorf("api still published: %+v", info) + } + + // Another workload's URL is untouched. + other, err := ForWorkload(context.Background(), c, "web") + if err != nil || other == nil { + t.Fatalf("web should be untouched, got info=%+v err=%v", other, err) + } +} + +func TestUnpublishIsANoOpWhenNothingIsPublished(t *testing.T) { + c := newFakeClient(t) + if err := Unpublish(context.Background(), c, testWorkloadName); err != nil { + t.Fatalf("unpublishing something that was never published is not an error, got: %v", err) + } +} diff --git a/internal/cmd/compute/url/reason.go b/internal/cmd/compute/url/reason.go new file mode 100644 index 00000000..2be4040c --- /dev/null +++ b/internal/cmd/compute/url/reason.go @@ -0,0 +1,73 @@ +package url + +import ( + "strings" + "unicode" +) + +// HumanBlock renders what the server says is holding something up, in words a +// developer can act on. +// +// The platform reports a blocked state as a camelCase reason plus a human +// message. Product principle 4 forbids showing the reason: "NoMatchingInterfaces" +// is internal state, and a developer who has never read the API types cannot do +// anything with it. The message is what was written for them, so it wins +// whenever there is one. +// +// When a condition carries no message there is still something worth saying, so +// the reason is spaced out and lowercased rather than dropped: "no matching +// interfaces" tells a developer more than silence and still never shows them an +// identifier. This is formatting, not interpretation — nothing here branches on +// which reason it was given, per the house rule in util/conditions.go. +func HumanBlock(reason, message string) string { + if message != "" { + return message + } + return humanizeReason(reason) +} + +// humanizeReason turns a camelCase condition reason into a lowercase phrase. +// Runs of capitals are kept together so "CertificateCARequired" reads as +// "certificate CA required" rather than "certificate c a required". +func humanizeReason(reason string) string { + if reason == "" { + return statusPending + } + + runes := []rune(reason) + var b strings.Builder + for i, r := range runes { + if i > 0 && unicode.IsUpper(r) { + prev := runes[i-1] + // A capital after a lowercase always starts a word; a capital that + // ends a run of capitals starts one only if a lowercase follows it. + startsWord := !unicode.IsUpper(prev) || + (i+1 < len(runes) && unicode.IsLower(runes[i+1])) + if startsWord { + b.WriteRune(' ') + } + } + b.WriteRune(r) + } + + words := strings.Fields(b.String()) + for i, w := range words { + // Leave acronyms as the server wrote them; lowercase ordinary words. + if !isAcronym(w) { + words[i] = strings.ToLower(w) + } + } + return strings.Join(words, " ") +} + +func isAcronym(w string) bool { + if len(w) < 2 { + return false + } + for _, r := range w { + if !unicode.IsUpper(r) && !unicode.IsDigit(r) { + return false + } + } + return true +} diff --git a/internal/cmd/compute/url/reason_test.go b/internal/cmd/compute/url/reason_test.go new file mode 100644 index 00000000..69cde865 --- /dev/null +++ b/internal/cmd/compute/url/reason_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import "testing" + +// TestHumanBlockNeverShowsAReason pins product principle 4: a developer sees +// what the server wrote for them, never the identifier it filed it under. +func TestHumanBlockNeverShowsAReason(t *testing.T) { + for _, tc := range []struct { + name string + reason string + message string + want string + }{{ + name: "the message wins whenever there is one", + reason: "NoMatchingInterfaces", + message: "selector matched no network interface", + want: "selector matched no network interface", + }, { + name: "a message-less reason is spaced out, not dropped", + reason: "NoMatchingInterfaces", + want: "no matching interfaces", + }, { + name: "a reason the CLI has never heard of still reads as words", + reason: "SomeReasonTheCLIHasNeverHeardOf", + want: "some reason the CLI has never heard of", + }, { + name: "acronyms survive", + reason: "CertificateCARequired", + want: "certificate CA required", + }, { + name: "a single word", + reason: "Pending", + want: "pending", + }, { + name: "nothing at all still says something", + want: statusPending, + }} { + t.Run(tc.name, func(t *testing.T) { + if got := HumanBlock(tc.reason, tc.message); got != tc.want { + t.Errorf("HumanBlock(%q, %q) = %q, want %q", tc.reason, tc.message, got, tc.want) + } + }) + } +} diff --git a/internal/cmd/compute/url/render.go b/internal/cmd/compute/url/render.go new file mode 100644 index 00000000..deaa9ac9 --- /dev/null +++ b/internal/cmd/compute/url/render.go @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "fmt" + "io" + "strings" + + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// RenderDetail writes the per-URL detail view: where the URL is, what backs +// it, and — the part a multi-city platform owes its users — which city is +// actually serving. +// +// Nothing here names the machinery. When something is wrong, the server's own +// reason and message are printed verbatim, so a condition this CLI has never +// heard of still reaches the user. +// +// A nil Info means the workload has no URL; the caller says so in its own +// words, because only it knows how the user asked. +func RenderDetail(out io.Writer, info *Info) { + if info == nil { + return + } + + renderURLs(out, info) + renderBackendLine(out, info) + fmt.Fprintln(out) + fmt.Fprintf(out, "%-*s %s\n", labelWidth, "Health", health(info)) + + if len(info.Locations) > 0 { + fmt.Fprintln(out) + renderLocations(out, info) + } + + renderDiagnosis(out, info) +} + +// renderURLs lists every hostname the URL answers on, the working one first. +// A hostname that is not serving carries the server's reason beside it. +func renderURLs(out io.Writer, info *Info) { + label := "URL" + if len(info.Hostnames) == 0 { + fmt.Fprintf(out, "%-*s %s\n", labelWidth, label, "—") + return + } + + for _, h := range info.Hostnames { + line := h.URL + if !h.Active { + line += " (" + h.Status + ")" + } + fmt.Fprintf(out, "%-*s %s\n", labelWidth, label, line) + label = "" + } +} + +// renderBackendLine states what the edge forwards to. +func renderBackendLine(out io.Writer, info *Info) { + if info.Port == 0 { + return + } + protocol := strings.ToLower(info.Protocol) + if protocol == "" { + protocol = strings.ToLower(string(networkingv1alpha.NetworkServiceProtocolTCP)) + } + fmt.Fprintf(out, "%-*s port %d/%s\n", labelWidth, "Backend", info.Port, protocol) +} + +// renderLocations prints the per-city breakdown, indented under the label +// column so it reads as part of the health block. +func renderLocations(out io.Writer, info *Info) { + indent := strings.Repeat(" ", labelWidth+1) + tw := util.NewTabWriter(out) + fmt.Fprintf(tw, "%sCITY\tBACKENDS\tHEALTHY\tSERVING\n", indent) + for _, l := range info.Locations { + fmt.Fprintf(tw, "%s%s\t%d\t%d\t%s\n", indent, l.City, l.Backends, l.Healthy, yesNo(l.Serving)) + } + _ = tw.Flush() +} + +// health summarises the URL in one line, from counts rather than from any +// reason string. +func health(info *Info) string { + b := info.Backends + switch { + case b.Total == 0: + return "Unavailable — no backends registered" + case b.Healthy == 0: + return fmt.Sprintf("Unavailable — 0 of %d backends healthy", b.Total) + case b.Healthy < b.Total: + return fmt.Sprintf("Degraded — %d of %d backends healthy", b.Healthy, b.Total) + default: + return fmt.Sprintf("Healthy — %d of %d backends healthy", b.Healthy, b.Total) + } +} + +// renderDiagnosis explains a URL that is not fully healthy and says what to +// run next. A healthy URL gets nothing: there is nothing to do. +func renderDiagnosis(out io.Writer, info *Info) { + var unhealthy []string + for _, l := range info.Locations { + if l.Backends > 0 && l.Healthy == 0 { + unhealthy = append(unhealthy, l.City) + } + } + + blocking := blockingLines(info) + if len(unhealthy) == 0 && len(blocking) == 0 { + return + } + + fmt.Fprintln(out) + indent := strings.Repeat(" ", 7) + + for _, city := range unhealthy { + fmt.Fprintf(out, " %s: no healthy backends — instances are running but not passing health checks.\n", city) + serving := servingCities(info) + switch { + case len(serving) == 0: + fmt.Fprintf(out, "%sNo city is taking traffic, so the URL is not answering.\n", indent) + case len(serving) == 1: + fmt.Fprintf(out, "%sTraffic is being served from %s only.\n", indent, serving[0]) + default: + fmt.Fprintf(out, "%sTraffic is being served from %s.\n", indent, strings.Join(serving, ", ")) + } + } + + for _, line := range blocking { + fmt.Fprintf(out, " %s\n", line) + } + + fmt.Fprintln(out) + fmt.Fprintln(out, " Next steps:") + if len(unhealthy) == 0 { + fmt.Fprintf(out, " Check instances: datumctl compute instances --workload=%s\n", info.WorkloadName) + return + } + for _, city := range unhealthy { + fmt.Fprintf(out, " Check instances: datumctl compute instances --workload=%s --city=%s\n", info.WorkloadName, city) + } +} + +// blockingLines collects what the server says is wrong, in the server's own +// words. The condition reason is never shown — see HumanBlock. +func blockingLines(info *Info) []string { + var lines []string + add := func(reason, message string) { + lines = append(lines, HumanBlock(reason, message)) + } + + if reason, message, blocked := util.ReadinessBlock(info.ServiceConditions, networkingv1alpha.NetworkServiceReady); blocked && reason != "" { + add(reason, message) + } + if reason, message, blocked := util.ReadinessBlock(info.ProxyConditions, networkingv1alpha.HTTPProxyConditionProgrammed); blocked && reason != "" { + add(reason, message) + } + for _, h := range info.Hostnames { + if !h.Active && h.Detail != "" { + lines = append(lines, fmt.Sprintf("%s: %s", h.Hostname, h.Detail)) + } + } + return lines +} + +func yesNo(v bool) string { + if v { + return "yes" + } + return "no" +} diff --git a/internal/cmd/compute/url/render_test.go b/internal/cmd/compute/url/render_test.go new file mode 100644 index 00000000..7b9e9c56 --- /dev/null +++ b/internal/cmd/compute/url/render_test.go @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "bytes" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// row returns the whitespace-separated fields of the first line containing the +// given first field, so assertions do not depend on column widths. +func row(t *testing.T, output, first string) []string { + t.Helper() + for _, line := range strings.Split(output, "\n") { + fields := strings.Fields(line) + if len(fields) > 0 && fields[0] == first { + return fields + } + } + t.Fatalf("no line starting with %q in:\n%s", first, output) + return nil +} + +func TestRenderDetailDegraded(t *testing.T) { + info := newInfo(testWorkloadName, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 2, 2, true), location("IAD", 2, 0, false)), + ) + + var out bytes.Buffer + RenderDetail(&out, info) + got := out.String() + + if f := row(t, got, "URL"); f[1] != testCanonicalURL { + t.Errorf("URL row = %v", f) + } + if f := row(t, got, "Backend"); f[1] != "port" || f[2] != "8080/tcp" { + t.Errorf("Backend row = %v, want port 8080/tcp", f) + } + if !strings.Contains(got, "Health Degraded — 2 of 4 backends healthy") { + t.Errorf("missing the health summary:\n%s", got) + } + + if f := row(t, got, "CITY"); strings.Join(f, " ") != "CITY BACKENDS HEALTHY SERVING" { + t.Errorf("table header = %v", f) + } + if f := row(t, got, "DFW"); strings.Join(f[1:], " ") != "2 2 yes" { + t.Errorf("DFW row = %v, want 2 2 yes", f) + } + if f := row(t, got, "IAD:"); len(f) == 0 { + t.Error("expected a narrative line for the unhealthy city") + } + if f := row(t, got, "IAD"); strings.Join(f[1:], " ") != "2 0 no" { + t.Errorf("IAD row = %v, want 2 0 no", f) + } + + for _, want := range []string{ + "IAD: no healthy backends — instances are running but not passing health checks.", + "Traffic is being served from DFW only.", + "Next steps:", + "datumctl compute instances --workload=api --city=IAD", + } { + if !strings.Contains(got, want) { + t.Errorf("output missing %q:\n%s", want, got) + } + } + + // The user never sees the machinery. + for _, forbidden := range []string{kindService, kindProxy, "http://"} { + if strings.Contains(got, forbidden) { + t.Errorf("output names the machinery %q:\n%s", forbidden, got) + } + } +} + +func TestRenderDetailHealthySaysNothingToDo(t *testing.T) { + info := newInfo(testWorkloadName, + publishedProxy(testWorkloadName, testCanonical), + publishedService(testWorkloadName, 8080, location("DFW", 2, 2, true), location("IAD", 2, 2, true)), + ) + + var out bytes.Buffer + RenderDetail(&out, info) + got := out.String() + + if !strings.Contains(got, "Healthy — 4 of 4 backends healthy") { + t.Errorf("missing the health summary:\n%s", got) + } + if strings.Contains(got, "Next steps") { + t.Errorf("a healthy URL needs no next steps:\n%s", got) + } + if f := row(t, got, "IAD"); strings.Join(f[1:], " ") != "2 2 yes" { + t.Errorf("IAD row = %v", f) + } +} + +func TestRenderDetailBothHostnames(t *testing.T) { + proxy := publishedProxy(testWorkloadName, testCanonical) + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, testCustomHostname) + proxy.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: testCustomHostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionTrue, "Verified", ""), + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionTrue, "CertificateIssued", ""), + }, + }} + + var out bytes.Buffer + RenderDetail(&out, newInfo(testWorkloadName, proxy, publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)))) + got := out.String() + + lines := strings.Split(got, "\n") + if !strings.HasPrefix(lines[0], "URL") || !strings.Contains(lines[0], testCustomURL) { + t.Errorf("first line = %q, want the custom hostname", lines[0]) + } + if strings.Contains(lines[1], "URL") || !strings.Contains(lines[1], testCanonicalURL) { + t.Errorf("second line = %q, want the managed hostname under an empty label", lines[1]) + } +} + +func TestRenderDetailShowsTheServersOwnWords(t *testing.T) { + proxy := publishedProxy(testWorkloadName, testCanonical) + svc := publishedService(testWorkloadName, 8080) + svc.Status.Conditions = []metav1.Condition{ + cond(networkingv1alpha.NetworkServiceReady, metav1.ConditionFalse, + "SomeReasonTheCLIHasNeverHeardOf", "selector matched no network interface"), + } + + var out bytes.Buffer + RenderDetail(&out, newInfo(testWorkloadName, proxy, svc)) + got := out.String() + + // The server's message verbatim, and never its reason: a reason the CLI has + // never heard of is still an identifier, and product principle 4 keeps those + // away from the developer. TestRenderDetailPendingHostnameCarriesItsStatus + // asserts the same rule for hostnames. + if !strings.Contains(got, "selector matched no network interface") { + t.Errorf("the server's message must appear verbatim:\n%s", got) + } + if strings.Contains(got, "SomeReasonTheCLIHasNeverHeardOf") { + t.Errorf("a raw condition reason reached the user:\n%s", got) + } + if !strings.Contains(got, "Unavailable — no backends registered") { + t.Errorf("missing the health summary:\n%s", got) + } + if !strings.Contains(got, "datumctl compute instances --workload=api") { + t.Errorf("missing next steps:\n%s", got) + } +} + +func TestRenderDetailPendingHostnameCarriesItsStatus(t *testing.T) { + proxy := publishedProxy(testWorkloadName, testCanonical) + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, testCustomHostname) + proxy.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: testCustomHostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionFalse, "DomainNotVerified", "waiting for TXT record"), + }, + }} + + var out bytes.Buffer + RenderDetail(&out, newInfo(testWorkloadName, proxy, publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true)))) + got := out.String() + + // The server's message, not its reason: "DomainNotVerified" is internal + // state and a developer must never be shown it. + if !strings.Contains(got, "https://api.example.com (waiting for TXT record)") { + t.Errorf("a pending hostname should carry the server's message:\n%s", got) + } + if strings.Contains(got, "DomainNotVerified") { + t.Errorf("a raw condition reason reached the user:\n%s", got) + } + if !strings.Contains(got, "api.example.com: waiting for TXT record") { + t.Errorf("the server's message should be shown verbatim:\n%s", got) + } +} + +func TestRenderDetailNilWritesNothing(t *testing.T) { + var out bytes.Buffer + RenderDetail(&out, nil) + if out.Len() != 0 { + t.Errorf("output = %q, want nothing — the caller words the no-URL case", out.String()) + } +} diff --git a/internal/cmd/compute/url/resources.go b/internal/cmd/compute/url/resources.go new file mode 100644 index 00000000..296937db --- /dev/null +++ b/internal/cmd/compute/url/resources.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package url owns the single mechanism by which a compute workload becomes a +// public HTTPS URL: a NetworkService that selects the workload's network +// interfaces by label, and an HTTPProxy whose only backend names that service. +// +// Every command that shows, creates, or removes a workload URL goes through +// this package so the two objects are always built, found, and deleted the +// same way. Nothing here prints machinery names — the vocabulary the user sees +// is "URL", "backends", "edge", and "certificate". +package url + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// maxPortNameLength is the DNS-label limit NetworkServicePort.Name enforces. +const maxPortNameLength = 63 + +// ResourceName returns the name shared by the NetworkService and the HTTPProxy +// that publish a workload. Both objects are named after the workload so a +// human reading `datumctl get` output can tell what they belong to; lookups +// never depend on it, they select on labels. +func ResourceName(workloadName string) string { + return workloadName +} + +// PortName derives the NetworkServicePort name for a workload port. +// +// computev1alpha.NamedPort.Name has no pattern constraint, but +// NetworkServicePort.Name (and the backend reference naming it) must be a DNS +// label: lowercase alphanumerics and dashes, starting and ending with an +// alphanumeric, at most 63 characters. The name is sanitized to fit. A name +// with nothing usable in it is an error rather than an object the API server +// would reject with a message about a field the user never typed. +func PortName(p computev1alpha.NamedPort) (string, error) { + if strings.TrimSpace(p.Name) == "" { + return "", fmt.Errorf("port %d has no name — name the port to publish it on a URL", p.Port) + } + + var b strings.Builder + for _, r := range strings.ToLower(p.Name) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + + name := strings.Trim(b.String(), "-") + if len(name) > maxPortNameLength { + name = strings.Trim(name[:maxPortNameLength], "-") + } + if name == "" { + return "", fmt.Errorf("port name %q cannot be used for a URL — use letters, digits and dashes", p.Name) + } + return name, nil +} + +// objectMeta returns the metadata both published objects share: the workload's +// namespace-scoped name, the labels every lookup selects on, and an owner +// reference back to the workload. +// +// The owner reference is belt and braces. Garbage collection in a project +// virtual control plane is unverified, so Unpublish deletes both objects +// explicitly and lookups match on labels; the reference exists so that a +// control plane which does collect owned objects does the right thing. +func objectMeta(w *computev1alpha.Workload) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: ResourceName(w.Name), + Namespace: util.ResourceNamespace, + Labels: map[string]string{ + computev1alpha.WorkloadNameLabel: w.Name, + computev1alpha.WorkloadUIDLabel: string(w.UID), + }, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: computev1alpha.GroupVersion.String(), + Kind: "Workload", + Name: w.Name, + UID: w.UID, + Controller: ptr(false), + BlockOwnerDeletion: ptr(false), + }}, + } +} + +// BuildNetworkService returns the NetworkService that gathers the workload's +// instances into one set of backends. Membership is selected by label, so +// instances appearing, disappearing and moving between cities need no edit. +// +// TrafficDistribution is deliberately left unset: the default serves each +// request from the location nearest the edge that received it, which is what +// is wanted without saying so. +func BuildNetworkService(w *computev1alpha.Workload, portName string, port int32) *networkingv1alpha.NetworkService { + return &networkingv1alpha.NetworkService{ + ObjectMeta: objectMeta(w), + Spec: networkingv1alpha.NetworkServiceSpec{ + NetworkInterfaces: networkingv1alpha.NetworkServiceInterfaceSelector{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + computev1alpha.WorkloadNameLabel: w.Name, + }, + }, + }, + Ports: []networkingv1alpha.NetworkServicePort{{ + Name: portName, + Port: port, + Protocol: networkingv1alpha.NetworkServiceProtocolTCP, + }}, + }, + } +} + +// BuildHTTPProxy returns the HTTPProxy that puts the workload on the internet. +// One rule, one backend, no matches (the CRD defaults to a PathPrefix match on +// "/"), and never any backend TLS: the edge reaches instances over plaintext +// inside the network, and the API rejects backend TLS for this backend form. +// +// hostnames are custom hostnames only. The platform-managed hostname is +// assigned by the server and read back from status. +func BuildHTTPProxy(w *computev1alpha.Workload, portName string, hostnames []string) *networkingv1alpha.HTTPProxy { + proxy := &networkingv1alpha.HTTPProxy{ + ObjectMeta: objectMeta(w), + Spec: networkingv1alpha.HTTPProxySpec{ + Rules: []networkingv1alpha.HTTPProxyRule{{ + Backends: []networkingv1alpha.HTTPProxyRuleBackend{{ + NetworkService: &networkingv1alpha.NetworkServiceBackendRef{ + Name: ResourceName(w.Name), + Port: portName, + }, + }}, + }}, + }, + } + + for _, h := range hostnames { + h = strings.TrimSpace(h) + if h == "" { + continue + } + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, gatewayv1.Hostname(h)) + } + + return proxy +} + +func ptr[T any](v T) *T { return &v } diff --git a/internal/cmd/compute/url/resources_test.go b/internal/cmd/compute/url/resources_test.go new file mode 100644 index 00000000..793296fc --- /dev/null +++ b/internal/cmd/compute/url/resources_test.go @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +func testWorkload() *computev1alpha.Workload { + w := &computev1alpha.Workload{} + w.Name = testWorkloadName + w.Namespace = util.ResourceNamespace + w.UID = types.UID("11111111-2222-3333-4444-555555555555") + return w +} + +func TestPortName(t *testing.T) { + tests := []struct { + name string + port computev1alpha.NamedPort + want string + wantErr bool + }{ + {name: "the common deploy path", port: computev1alpha.NamedPort{Name: "http", Port: 8080}, want: testPortName}, + {name: "uppercase is lowered", port: computev1alpha.NamedPort{Name: "HTTP", Port: 80}, want: testPortName}, + {name: "underscores become dashes", port: computev1alpha.NamedPort{Name: "web_port", Port: 80}, want: "web-port"}, + {name: "dots become dashes", port: computev1alpha.NamedPort{Name: "web.port", Port: 80}, want: "web-port"}, + {name: "leading and trailing junk is trimmed", port: computev1alpha.NamedPort{Name: "_http_", Port: 80}, want: testPortName}, + {name: "digits are kept", port: computev1alpha.NamedPort{Name: "h2c9", Port: 80}, want: "h2c9"}, + {name: "spaces become dashes", port: computev1alpha.NamedPort{Name: "my port", Port: 80}, want: "my-port"}, + { + name: "over-long names are truncated to a DNS label", + port: computev1alpha.NamedPort{Name: strings.Repeat("a", 70), Port: 80}, + want: strings.Repeat("a", 63), + }, + { + // Truncating must not leave a trailing dash, which the API rejects. + name: "truncation does not leave a trailing dash", + port: computev1alpha.NamedPort{Name: strings.Repeat("a", 62) + "-b" + strings.Repeat("c", 10), Port: 80}, + want: strings.Repeat("a", 62), + }, + {name: "empty name is an error", port: computev1alpha.NamedPort{Name: "", Port: 8080}, wantErr: true}, + {name: "whitespace-only name is an error", port: computev1alpha.NamedPort{Name: " ", Port: 8080}, wantErr: true}, + {name: "nothing usable is an error", port: computev1alpha.NamedPort{Name: "___", Port: 8080}, wantErr: true}, + {name: "non-ascii only is an error", port: computev1alpha.NamedPort{Name: "日本", Port: 8080}, wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := PortName(tc.port) + if tc.wantErr { + if err == nil { + t.Fatalf("PortName(%q) = %q, want error", tc.port.Name, got) + } + return + } + if err != nil { + t.Fatalf("PortName(%q) returned error: %v", tc.port.Name, err) + } + if got != tc.want { + t.Errorf("PortName(%q) = %q, want %q", tc.port.Name, got, tc.want) + } + if !dnsLabel(got) { + t.Errorf("PortName(%q) = %q, which the API would reject", tc.port.Name, got) + } + }) + } +} + +// dnsLabel mirrors the pattern NetworkServicePort.Name is validated against. +func dnsLabel(s string) bool { + if s == "" || len(s) > 63 { + return false + } + for i, r := range s { + alnum := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + if !alnum && r != '-' { + return false + } + if (i == 0 || i == len(s)-1) && !alnum { + return false + } + } + return true +} + +func TestResourceNameIsSharedByBothObjects(t *testing.T) { + w := testWorkload() + svc := BuildNetworkService(w, testPortName, 8080) + proxy := BuildHTTPProxy(w, testPortName, nil) + + if svc.Name != ResourceName(w.Name) || proxy.Name != ResourceName(w.Name) { + t.Fatalf("names diverge: service %q, proxy %q, ResourceName %q", svc.Name, proxy.Name, ResourceName(w.Name)) + } + if proxy.Spec.Rules[0].Backends[0].NetworkService.Name != svc.Name { + t.Errorf("backend points at %q, but the service is named %q", + proxy.Spec.Rules[0].Backends[0].NetworkService.Name, svc.Name) + } +} + +func TestBuildNetworkService(t *testing.T) { + w := testWorkload() + svc := BuildNetworkService(w, testPortName, 8080) + + if svc.Namespace != util.ResourceNamespace { + t.Errorf("namespace = %q, want %q", svc.Namespace, util.ResourceNamespace) + } + assertPublishedLabels(t, svc.Labels, w) + assertOwnerRef(t, svc.OwnerReferences, w) + + want := map[string]string{computev1alpha.WorkloadNameLabel: w.Name} + got := svc.Spec.NetworkInterfaces.Selector.MatchLabels + if len(got) != len(want) { + t.Fatalf("selector matchLabels = %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Errorf("selector matchLabels[%s] = %q, want %q", k, got[k], v) + } + } + if len(svc.Spec.NetworkInterfaces.Selector.MatchExpressions) != 0 { + t.Errorf("selector should not use matchExpressions: %v", svc.Spec.NetworkInterfaces.Selector.MatchExpressions) + } + + if len(svc.Spec.Ports) != 1 { + t.Fatalf("ports = %d, want exactly 1", len(svc.Spec.Ports)) + } + p := svc.Spec.Ports[0] + if p.Name != testPortName || p.Port != 8080 || p.Protocol != networkingv1alpha.NetworkServiceProtocolTCP { + t.Errorf("port = %+v, want {http 8080 TCP}", p) + } + + // The type's own documentation says to leave this unset. + if (svc.Spec.TrafficDistribution != networkingv1alpha.NetworkServiceTrafficDistribution{}) { + t.Errorf("trafficDistribution = %+v, want unset", svc.Spec.TrafficDistribution) + } +} + +func TestBuildHTTPProxy(t *testing.T) { + w := testWorkload() + proxy := BuildHTTPProxy(w, testPortName, []string{testCustomHostname, " ", "www.example.com"}) + + if proxy.Namespace != util.ResourceNamespace { + t.Errorf("namespace = %q, want %q", proxy.Namespace, util.ResourceNamespace) + } + assertPublishedLabels(t, proxy.Labels, w) + assertOwnerRef(t, proxy.OwnerReferences, w) + + if len(proxy.Spec.Hostnames) != 2 { + t.Fatalf("hostnames = %v, want the two non-blank entries", proxy.Spec.Hostnames) + } + if string(proxy.Spec.Hostnames[0]) != testCustomHostname || string(proxy.Spec.Hostnames[1]) != "www.example.com" { + t.Errorf("hostnames = %v, want [api.example.com www.example.com]", proxy.Spec.Hostnames) + } + + if len(proxy.Spec.Rules) != 1 { + t.Fatalf("rules = %d, want exactly 1", len(proxy.Spec.Rules)) + } + rule := proxy.Spec.Rules[0] + if len(rule.Matches) != 0 { + t.Errorf("matches = %v, want none so the CRD default (PathPrefix /) applies", rule.Matches) + } + if len(rule.Backends) != 1 { + t.Fatalf("backends = %d, want exactly 1", len(rule.Backends)) + } + + b := rule.Backends[0] + if b.NetworkService == nil { + t.Fatal("backend does not reference a network service") + } + if b.NetworkService.Name != ResourceName(w.Name) || b.NetworkService.Port != testPortName { + t.Errorf("backend ref = %+v, want {api http}", *b.NetworkService) + } + // The API rejects backend TLS for this backend form, and the other backend + // forms are mutually exclusive with it. + if b.TLS != nil { + t.Error("backend TLS is set; the API rejects it for networkService backends") + } + if b.Endpoint != "" || b.Connector != nil || b.Instance != nil { + t.Errorf("backend sets a mutually exclusive field: %+v", b) + } +} + +func TestBuildHTTPProxyWithoutHostnames(t *testing.T) { + proxy := BuildHTTPProxy(testWorkload(), testPortName, nil) + if len(proxy.Spec.Hostnames) != 0 { + t.Errorf("hostnames = %v, want none — the managed hostname comes from status", proxy.Spec.Hostnames) + } +} + +func assertPublishedLabels(t *testing.T, got map[string]string, w *computev1alpha.Workload) { + t.Helper() + if got[computev1alpha.WorkloadNameLabel] != w.Name { + t.Errorf("label %s = %q, want %q", computev1alpha.WorkloadNameLabel, got[computev1alpha.WorkloadNameLabel], w.Name) + } + if got[computev1alpha.WorkloadUIDLabel] != string(w.UID) { + t.Errorf("label %s = %q, want %q", computev1alpha.WorkloadUIDLabel, got[computev1alpha.WorkloadUIDLabel], w.UID) + } +} + +func assertOwnerRef(t *testing.T, refs []metav1.OwnerReference, w *computev1alpha.Workload) { + t.Helper() + if len(refs) != 1 { + t.Fatalf("owner references = %d, want exactly 1", len(refs)) + } + ref := refs[0] + if ref.APIVersion != computev1alpha.GroupVersion.String() { + t.Errorf("owner apiVersion = %q, want %q", ref.APIVersion, computev1alpha.GroupVersion.String()) + } + if ref.Kind != "Workload" || ref.Name != w.Name || ref.UID != w.UID { + t.Errorf("owner ref = %+v, want Workload/%s/%s", ref, w.Name, w.UID) + } + if ref.Controller == nil || *ref.Controller { + t.Errorf("owner controller = %v, want false", ref.Controller) + } + if ref.BlockOwnerDeletion == nil || *ref.BlockOwnerDeletion { + t.Errorf("owner blockOwnerDeletion = %v, want false", ref.BlockOwnerDeletion) + } +} diff --git a/internal/cmd/compute/url/state_test.go b/internal/cmd/compute/url/state_test.go new file mode 100644 index 00000000..99451155 --- /dev/null +++ b/internal/cmd/compute/url/state_test.go @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "context" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// pendingCertProxy is the shape the platform reports in the window between +// `domains add` and the new hostname's certificate being issued: the +// per-hostname status says that one hostname is waiting, and the proxy-level +// certificate roll-up — whose True reason is "AllCertificatesReady" — is +// therefore False for the proxy as a whole. +func pendingCertProxy() *networkingv1alpha.HTTPProxy { + p := publishedProxy(testWorkloadName, testCanonical) + p.Spec.Hostnames = append(p.Spec.Hostnames, testCustomHostname) + p.Status.Conditions = []metav1.Condition{ + cond(networkingv1alpha.HTTPProxyConditionAccepted, metav1.ConditionTrue, "Accepted", ""), + cond(networkingv1alpha.HTTPProxyConditionProgrammed, metav1.ConditionTrue, "Programmed", ""), + cond(networkingv1alpha.HTTPProxyConditionCertificatesReady, metav1.ConditionFalse, + "CertificatePending", "issuing certificate for "+testCustomHostname), + } + p.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: testCustomHostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionTrue, "Verified", ""), + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionFalse, "Pending", "issuing"), + }, + }} + return p +} + +// TestManagedHostnameSurvivesAPendingCustomHostname is the state every user +// who runs `domains add` passes through, and the one no existing test covers: +// one hostname is waiting on a certificate while the platform-managed hostname +// carries on serving exactly as it did before. +// +// The managed hostname has no per-hostname status of its own here — control +// planes only publish HostnameStatuses for hostnames they are working on — and +// the proxy-level conditions are describing the *other* hostname. Nothing the +// platform says about a custom hostname may be attributed to this one. +func TestManagedHostnameSurvivesAPendingCustomHostname(t *testing.T) { + c := newFakeClient(t, pendingCertProxy(), + publishedService(testWorkloadName, 8080, location("DFW", 2, 2, true))) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + + managed := info.Hostnames[len(info.Hostnames)-1] + if !managed.Managed { + t.Fatalf("last hostname = %+v, want the managed one", managed) + } + if !managed.Active || managed.Status != statusActive { + t.Errorf("managed hostname = %+v, want it still active — it was serving before the custom hostname was attached", managed) + } + if managed.Detail != "" { + t.Errorf("managed hostname detail = %q, want nothing: that message is about %s", managed.Detail, testCustomHostname) + } + + // The consequence that costs the most: url.Publish waits on Live(), so a + // redeploy of this workload blocks until an unrelated certificate issues. + if !info.Live() { + t.Errorf("URL is not live, so `deploy` will wait on a hostname the user did not ask about; info = %+v", info) + } +} + +// The custom hostname's own state is read from its own conditions and is +// correct even today — this pins the half that works, so a fix for the managed +// hostname cannot regress it. +func TestPendingCustomHostnameReportsItsOwnReason(t *testing.T) { + c := newFakeClient(t, pendingCertProxy(), + publishedService(testWorkloadName, 8080, location("DFW", 2, 2, true))) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + + custom := info.Hostnames[0] + if custom.Managed || custom.Active { + t.Fatalf("first hostname = %+v, want the custom one, not serving yet", custom) + } + if custom.Status != "issuing" || custom.Certificate != "issuing" { + t.Errorf("custom hostname = %+v, want the server's own message", custom) + } + + // The URL shown is still the managed one: a hostname that is not serving + // must never be the address handed to the user. + if info.URL != testCanonicalURL { + t.Errorf("URL = %q, want the managed hostname while the custom one is pending", info.URL) + } +} + +// TestForWorkloadWhenTheBackendKindIsNotServed covers the half-installed +// control plane: URLs are served, their backends are not. The URL is still +// reported — with no backends — rather than the whole lookup failing. +func TestForWorkloadWhenTheBackendKindIsNotServed(t *testing.T) { + s := runtime.NewScheme() + if err := computev1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering compute scheme: %v", err) + } + // Only the proxy kind, deliberately: listing NetworkServices answers with + // a not-registered error, which reads as "nothing published here". + s.AddKnownTypes(networkingv1alpha.GroupVersion, + &networkingv1alpha.HTTPProxy{}, &networkingv1alpha.HTTPProxyList{}) + metav1.AddToGroupVersion(s, networkingv1alpha.GroupVersion) + + c := fake.NewClientBuilder().WithScheme(s). + WithObjects(publishedProxy(testWorkloadName, testCanonical)).Build() + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("a missing backend kind is not a lookup failure, got: %v", err) + } + if info == nil { + t.Fatal("info = nil, want the URL to still be reported") + } + if info.URL != testCanonicalURL { + t.Errorf("URL = %q, want %q", info.URL, testCanonicalURL) + } + if info.Service != nil || info.Backends != (Backends{}) { + t.Errorf("Service = %v, Backends = %+v, want nothing known about backends", info.Service, info.Backends) + } + + all, err := ForAll(context.Background(), c) + if err != nil { + t.Fatalf("ForAll returned error: %v", err) + } + if len(all) != 1 || all[testWorkloadName] == nil { + t.Errorf("ForAll = %v, want the one URL", all) + } +} + +// TestPrimaryWithoutAManagedHostnameYet: between creating the proxy and the +// server assigning a hostname, a workload that was published with a custom +// hostname has exactly one hostname and it is not serving. Whatever is shown, +// it must be that hostname and not the empty string — an empty URL is what +// `open` turns into "the platform is still assigning one". +func TestPrimaryWithoutAManagedHostnameYet(t *testing.T) { + proxy := BuildHTTPProxy(workloadNamed(testWorkloadName), testPortName, []string{testCustomHostname}) + // No CanonicalHostname, and nothing programmed yet. + proxy.Status.Conditions = []metav1.Condition{ + cond(networkingv1alpha.HTTPProxyConditionAccepted, metav1.ConditionTrue, "Accepted", ""), + cond(networkingv1alpha.HTTPProxyConditionProgrammed, metav1.ConditionFalse, "Pending", "waiting for the edge"), + } + + c := newFakeClient(t, proxy) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info.CanonicalHostname != "" { + t.Fatalf("CanonicalHostname = %q, want none assigned yet", info.CanonicalHostname) + } + if len(info.Hostnames) != 1 || info.Hostnames[0].Managed { + t.Fatalf("Hostnames = %+v, want the custom one alone", info.Hostnames) + } + if info.URL != testCustomURL { + t.Errorf("URL = %q, want the only hostname there is", info.URL) + } + // It is not live, so no command may present it as ready. + if info.Live() { + t.Error("a URL whose edge is not programmed must not report as live") + } +} + +// A URL nothing is known about at all — no hostname of either kind — must +// report an empty URL rather than "https://". +func TestPrimaryWithNoHostnamesAtAll(t *testing.T) { + c := newFakeClient(t, BuildHTTPProxy(workloadNamed(testWorkloadName), testPortName, nil)) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + if info.URL != "" { + t.Errorf("URL = %q, want empty — there is no hostname to show", info.URL) + } + if info.Live() { + t.Error("a URL with no hostname is not live") + } +} + +// TestAFreshlyAttachedHostnameIsPending is the first seconds of `domains add`: +// the hostname is on the spec and the platform has not looked at it yet, so it +// has no entry in the per-hostname statuses. +// +// The proxy is already serving and every proxy-level condition is True, so a +// hostname that borrowed them would read as active and verified the instant it +// was attached — `domains add` would print a checkmark, skip the DNS records +// the user has to create, and exit. +func TestAFreshlyAttachedHostnameIsPending(t *testing.T) { + proxy := publishedProxy(testWorkloadName, testCanonical) + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, testCustomHostname) + // Deliberately no HostnameStatuses: nothing has been reported about it. + + c := newFakeClient(t, proxy, + publishedService(testWorkloadName, 8080, location("DFW", 1, 1, true))) + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("ForWorkload returned error: %v", err) + } + + custom := info.Hostnames[0] + if custom.Managed { + t.Fatalf("first hostname = %+v, want the custom one", custom) + } + if custom.Active || custom.Status != statusPending { + t.Errorf("custom hostname = %+v, want it pending: the platform has said nothing about it", custom) + } + if custom.Certificate != "" { + t.Errorf("certificate = %q, want nothing known — no certificate has been reported for this hostname", custom.Certificate) + } + if len(custom.Conditions) != 0 { + t.Errorf("conditions = %+v, want none: borrowing another hostname's checks is what puts a checkmark on an unverified domain", custom.Conditions) + } + + // The address handed to the user stays the one that answers. + if info.URL != testCanonicalURL { + t.Errorf("URL = %q, want the managed hostname", info.URL) + } + + // And the managed hostname still reads from the proxy-level conditions, + // which is all a control plane that reports nothing per-hostname publishes. + managed := info.Hostnames[1] + if !managed.Active || managed.Certificate != certificateValid { + t.Errorf("managed hostname = %+v, want it active with a valid certificate", managed) + } +} + +// TestABlockingConditionWithNoMessageNeverShowsItsReason: a condition reason is +// camelCase internal state, and the product promises a developer never sees +// one. When the server has no message to show, the CLI says the plain thing +// rather than leaking the reason into a table cell. +func TestABlockingConditionWithNoMessageNeverShowsItsReason(t *testing.T) { + const ( + verifyReason = "UnverifiedHostnamesPresent" + certReason = "CertificateRequestPending" + ) + + proxy := publishedProxy(testWorkloadName, testCanonical) + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, testCustomHostname) + proxy.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: testCustomHostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionFalse, verifyReason, ""), + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionFalse, certReason, ""), + }, + }} + + info := newInfo(testWorkloadName, proxy, nil) + h := info.Hostnames[0] + + if h.Status != statusPending { + t.Errorf("status = %q, want %q — the server gave no message to show", h.Status, statusPending) + } + if h.Certificate != statusPending { + t.Errorf("certificate = %q, want %q", h.Certificate, statusPending) + } + for _, field := range []string{h.Status, h.Certificate, h.Detail} { + for _, reason := range []string{verifyReason, certReason} { + if strings.Contains(field, reason) { + t.Errorf("%q reached the user: condition reasons are internal state", field) + } + } + } +} + +// The same rule with a message to show: the message is what a user reads, and +// the reason still does not appear anywhere. +func TestABlockingConditionShowsTheServersMessage(t *testing.T) { + proxy := publishedProxy(testWorkloadName, testCanonical) + proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, testCustomHostname) + proxy.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ + Hostname: testCustomHostname, + Conditions: []metav1.Condition{ + cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionFalse, + "DomainNotVerified", "no TXT record found at _datum-challenge.api.example.com"), + cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionFalse, + "CertificatePending", "waiting for the domain to verify"), + }, + }} + + h := newInfo(testWorkloadName, proxy, nil).Hostnames[0] + + if h.Status != "no TXT record found at _datum-challenge.api.example.com" { + t.Errorf("status = %q, want the server's message", h.Status) + } + if h.Certificate != "waiting for the domain to verify" { + t.Errorf("certificate = %q, want the server's message", h.Certificate) + } + if strings.Contains(h.Status+h.Certificate, "DomainNotVerified") || + strings.Contains(h.Status+h.Certificate, "CertificatePending") { + t.Errorf("a raw reason reached the user: status=%q certificate=%q", h.Status, h.Certificate) + } +} diff --git a/internal/cmd/compute/workloads/filter_test.go b/internal/cmd/compute/workloads/filter_test.go new file mode 100644 index 00000000..aab70be4 --- /dev/null +++ b/internal/cmd/compute/workloads/filter_test.go @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package workloads + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// The health values the filter selects on, and the workload names the cases +// name, spelled once each. +const ( + healthAvailable = "Available" + healthDegraded = "Degraded" + healthUnavailable = "Unavailable" + + wlAPI = "api" + wlBroken = "broken" + wlSlow = "slow" +) + +// unavailable returns a workload the platform reports as not available. A +// "Degraded" one is different again: available, but short of its desired +// replicas. The filter selects on the first word of either. +func unavailable(name, uid, image string, cities ...string) *computev1alpha.Workload { + w := workload(name, uid, image, cities...) + w.Status.Conditions = []metav1.Condition{{ + Type: computev1alpha.WorkloadAvailable, + Status: metav1.ConditionFalse, + Reason: "InsufficientCapacity", + }} + return w +} + +// TestListWorkloadsHealthFilter: the health filter is the one piece of the +// list view the URL work moved wholesale into a new function, and nothing +// exercised it afterwards. It has to still select on the first word of the +// health, case-insensitively, and the rows it keeps still carry their URLs. +func TestListWorkloadsHealthFilter(t *testing.T) { + objs := []client.Object{ + workload(wlAPI, "uid-api", "ghcr.io/acme/api:1.4.2", "DFW"), + unavailable(wlBroken, "uid-broken", "ghcr.io/acme/broken:1", "DFW"), + workload(wlSlow, "uid-slow", "ghcr.io/acme/slow:1", "DFW"), + deployment("api-dfw", "uid-api", "DFW", 2, 2), + deployment("broken-dfw", "uid-broken", "DFW", 0, 2), + deployment("slow-dfw", "uid-slow", "DFW", 1, 3), + publishedProxy(wlAPI, testCustom), + publishedService(wlAPI), + } + + tests := []struct { + name string + health string + wantNames []string + wantMissing []string + }{ + {name: "no filter lists them all", wantNames: []string{wlAPI, wlBroken, wlSlow}}, + {name: "available only", health: healthAvailable, wantNames: []string{wlAPI}, wantMissing: []string{wlBroken, wlSlow}}, + {name: "the filter is case-insensitive", health: "available", wantNames: []string{wlAPI}, wantMissing: []string{wlBroken}}, + {name: "unavailable only", health: healthUnavailable, wantNames: []string{wlBroken}, wantMissing: []string{wlAPI, wlSlow}}, + {name: "degraded is available but short of replicas", health: healthDegraded, wantNames: []string{wlSlow}, wantMissing: []string{wlAPI, wlBroken}}, + {name: "matching on the whole health string finds nothing", health: healthAvailable + " — all placements at desired replicas", wantMissing: []string{wlAPI, wlBroken, wlSlow}}, + {name: "a health nothing has", health: "Nonsense", wantMissing: []string{wlAPI, wlBroken, wlSlow}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t, objs...) + + opts := listOptions{output: util.OutputJSON, health: tc.health} + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, opts); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + var views []workloadView + if err := json.Unmarshal(out.Bytes(), &views); err != nil { + t.Fatalf("output is not a JSON array: %v\n%s", err, out.String()) + } + got := map[string]workloadView{} + for _, v := range views { + got[v.Name] = v + } + + for _, name := range tc.wantNames { + if _, ok := got[name]; !ok { + t.Errorf("%q missing from the filtered list: %v", name, got) + } + } + for _, name := range tc.wantMissing { + if _, ok := got[name]; ok { + t.Errorf("%q should have been filtered out: %v", name, got) + } + } + if v, ok := got[wlAPI]; ok && v.URL != "https://"+testCustom { + t.Errorf("api url = %q, want the URL to survive filtering", v.URL) + } + }) + } +} + +// TestListWorkloadsFailsWhenWorkloadsCannotBeListed: an unreadable project is +// an error. Only the URL column degrades to "unknown" — the workloads +// themselves are the command. +func TestListWorkloadsFailsWhenWorkloadsCannotBeListed(t *testing.T) { + boom := errors.New("forbidden") + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(projectObjects()...). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*computev1alpha.WorkloadList); ok { + return boom + } + return cl.List(ctx, list, opts...) + }, + }). + Build() + + var out, errOut bytes.Buffer + err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputTable}) + if !errors.Is(err, boom) { + t.Fatalf("error = %v, want the list failure", err) + } +} + +// TestListWorkloadsUnknownURLsInJSON: the table says "?" when the URLs could +// not be read. JSON has no such marker — the field is just empty — so a script +// reading `.url` cannot tell "no URL" from "could not tell". This pins the +// current shape so the gap is visible rather than assumed away. +func TestListWorkloadsUnknownURLsInJSON(t *testing.T) { + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(projectObjects()...). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*networkingv1alpha.HTTPProxyList); ok { + return errors.New("forbidden") + } + return cl.List(ctx, list, opts...) + }, + }). + Build() + + var out, errOut bytes.Buffer + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputJSON}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + var views []workloadView + if err := json.Unmarshal(out.Bytes(), &views); err != nil { + t.Fatalf("output is not a JSON array: %v\n%s", err, out.String()) + } + for _, v := range views { + if v.URL != "" { + t.Errorf("%s url = %q, want empty when the URLs could not be read", v.Name, v.URL) + } + } + // The warning is the only signal a script has, and it goes to stderr. + if !strings.Contains(errOut.String(), "could not read URLs") { + t.Errorf("stderr must carry the warning:\n%s", errOut.String()) + } +} diff --git a/internal/cmd/compute/workloads/workloads.go b/internal/cmd/compute/workloads/workloads.go index 4093ed3b..e2c21d6a 100644 --- a/internal/cmd/compute/workloads/workloads.go +++ b/internal/cmd/compute/workloads/workloads.go @@ -3,6 +3,7 @@ package workloads import ( "context" "fmt" + "io" "strings" "github.com/spf13/cobra" @@ -14,6 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/url" "go.datum.net/compute/internal/cmd/compute/util" ) @@ -23,7 +25,16 @@ func Command() *cobra.Command { Use: "workloads", Short: "List or inspect workloads", Long: `List all workloads in the project, optionally filtered by health or location. -Use the describe subcommand for a unified config + health view of a single workload.`, +Use the describe subcommand for a unified config + health view of a single workload. + +The URL column shows where a workload answers on the public internet; a +workload that declares no HTTP port shows "—", and "?" means the URLs could not +be read at all. JSON and YAML output carry the same value as a "url" field, +alongside the whole workload resource under "workload"; there the two cases are +told apart by omitting "url" and, when the read failed, setting "urlError". + +The LOCATIONS column lists the locations a workload is placed in, shortened when +there are many; JSON and YAML always carry the full list under "locations".`, Example: ` # List all workloads datumctl compute workloads @@ -36,6 +47,9 @@ Use the describe subcommand for a unified config + health view of a single workl # Machine-readable output datumctl compute workloads -o json + # One workload's URL, for scripting + datumctl compute workloads -o json | jq -r '.[] | select(.name=="api") | .url' + # Describe a single workload datumctl compute workloads describe api`, RunE: func(cmd *cobra.Command, args []string) error { @@ -62,85 +76,176 @@ Use the describe subcommand for a unified config + health view of a single workl // workloads list // ----------------------------------------------------------------------- -//nolint:gocyclo // A list command branches over flag parsing, filtering, and several output formats; splitting it would scatter one linear flow. +const ( + // noURL is what the table shows for a workload that is not published. + noURL = "—" + // unknownURL is what the table shows when the URLs could not be read at + // all, which is a different thing from a workload having none. + unknownURL = "?" + // maxTableLocations is how many locations the LOCATIONS column names before + // it summarises the rest; a workload in a dozen locations must not push the + // columns to its right off the terminal. + maxTableLocations = 3 +) + +// listOptions are the parsed flags of the list view. +type listOptions struct { + output util.OutputFormat + health string + location string + noHeaders bool +} + +// workloadRow is one rendered line of the list view. +type workloadRow struct { + name string + health string + healthShort string // first word: the narrow table column, and the filter key + ready string + upToDate string + placements []string + locations []string + image string + age string + instType string + url string // "" when the workload has no URL + workload *computev1alpha.Workload +} + +// workloadView is the machine-readable form of a row. +// +// `-o json` / `-o yaml` used to emit the raw WorkloadList. They now emit a +// top-level array of these, because a workload's URL is not a field of a +// Workload — it lives on separate objects — and the contract is that +// `workloads -o json | jq -r '.[] | select(.name=="api") | .url'` works. +// +// No existing field changed shape or meaning: the raw resource is carried +// whole under `workload`, so `.items[].spec` becomes `.[].workload.spec`. +type workloadView struct { + Name string `json:"name"` + Health string `json:"health"` + Ready string `json:"ready"` + UpToDate string `json:"upToDate"` + Placements []string `json:"placements,omitempty"` + Locations []string `json:"locations,omitempty"` + Image string `json:"image,omitempty"` + InstanceType string `json:"instanceType,omitempty"` + Age string `json:"age,omitempty"` + + // URL is where the workload answers, and is omitted when it has none — + // the structured form of the table's "—". It is also omitted when the + // URLs could not be read, and then URLError carries the server's error, + // which is the structured form of the table's "?". Reporting both cases + // as `"url": ""` would tell a consumer a workload is unpublished when + // all that actually happened is that the lookup failed. + URL string `json:"url,omitempty"` + URLError string `json:"urlError,omitempty"` + + Workload *computev1alpha.Workload `json:"workload,omitempty"` +} + func runList(cmd *cobra.Command, _ []string) error { - ctx := context.Background() project := util.ProjectFromCmd(cmd) + c, err := util.NewClient(project) + if err != nil { + return err + } + outputFlag, _ := cmd.Flags().GetString("output") healthFilter, _ := cmd.Flags().GetString("health") locationFilter, _ := cmd.Flags().GetString("location") noHeaders, _ := cmd.Flags().GetBool("no-headers") - c, err := util.NewClient(project) + return listWorkloads(context.Background(), cmd.OutOrStdout(), cmd.ErrOrStderr(), c, project, listOptions{ + output: util.OutputFormat(outputFlag), + health: healthFilter, + location: locationFilter, + noHeaders: noHeaders, + }) +} + +// listWorkloads renders the list view. The client is a parameter so the whole +// view can be rendered against a fake one in tests. +func listWorkloads(ctx context.Context, out, errOut io.Writer, c client.Client, project string, opts listOptions) error { + result, err := collectRows(ctx, errOut, c, opts) if err != nil { return err } - var wlList computev1alpha.WorkloadList - if err := c.List(ctx, &wlList, client.InNamespace(util.ResourceNamespace)); err != nil { - return fmt.Errorf("listing workloads: %w", err) - } - - // JSON/YAML: emit raw API resource and return early. - switch util.OutputFormat(outputFlag) { + switch opts.output { case util.OutputJSON: - return util.PrintJSON(cmd.OutOrStdout(), &wlList) + return util.PrintJSON(out, viewsOf(result)) case util.OutputYAML: - return util.PrintYAML(cmd.OutOrStdout(), &wlList) + return util.PrintYAML(out, viewsOf(result)) + } + + if len(result.rows) == 0 { + printNoRows(out, project, opts) + return nil + } + + renderTable(out, result, opts) + return nil +} + +// listing is what reading the project produced: the rows the filters left +// standing, plus the URL lookup's own failure if it had one. A non-nil urlErr +// means every row's URL is unknown, which is not the same claim as a workload +// having none — both the table and the structured output draw that line. +type listing struct { + rows []workloadRow + urlErr error +} + +// urlsKnown reports whether the project's URLs could be read at all. +func (l listing) urlsKnown() bool { return l.urlErr == nil } + +// collectRows reads the project and assembles the rows the filters left +// standing. Its error is the command failing outright; a URL lookup that fails +// is carried on the listing instead, because the URL is a column, not the +// command. +func collectRows(ctx context.Context, errOut io.Writer, c client.Client, opts listOptions) (listing, error) { + var wlList computev1alpha.WorkloadList + if err := c.List(ctx, &wlList, client.InNamespace(util.ResourceNamespace)); err != nil { + return listing{}, fmt.Errorf("listing workloads: %w", err) } - // For table output we need deployment data to compute READY counts. var deployList computev1alpha.WorkloadDeploymentList if err := c.List(ctx, &deployList, client.InNamespace(util.ResourceNamespace)); err != nil { - return fmt.Errorf("listing deployments: %w", err) + return listing{}, fmt.Errorf("listing deployments: %w", err) } - // Build map: workloadUID → []WorkloadDeployment. + // URLs come in one pass for the whole project rather than a lookup per + // workload. A project whose URLs cannot be read still lists its workloads. + urls, urlErr := url.ForAll(ctx, c) + if urlErr != nil { + fmt.Fprintf(errOut, "Warning: could not read URLs: %v\n", urlErr) + } + + // workloadUID → its deployments, and the set of UIDs with a deployment in + // the requested city. deploysByWorkload := make(map[string][]computev1alpha.WorkloadDeployment) + locationFilteredUIDs := map[string]bool{} for _, d := range deployList.Items { wUID := d.Labels[computev1alpha.WorkloadUIDLabel] deploysByWorkload[wUID] = append(deploysByWorkload[wUID], d) - } - - // Location filter: collect workload UIDs with a deployment in the requested location. - locationFilteredUIDs := map[string]bool{} - if locationFilter != "" { - for _, d := range deployList.Items { - if d.Spec.LocationRef.Name == locationFilter { - wUID := d.Labels[computev1alpha.WorkloadUIDLabel] - locationFilteredUIDs[wUID] = true - } + if opts.location != "" && d.Spec.LocationRef.Name == opts.location { + locationFilteredUIDs[wUID] = true } } - type workloadRow struct { - name string - health string - healthShort string // first word, for filter comparison - readyStr string - upToDateStr string - placements string - image string - age string - instType string // wide only - } - - wide := util.OutputFormat(outputFlag) == util.OutputWide - var rows []workloadRow - for _, wl := range wlList.Items { - wUID := string(wl.UID) + for i := range wlList.Items { + wl := &wlList.Items[i] - // Location filter. - if locationFilter != "" && !locationFilteredUIDs[wUID] { + if opts.location != "" && !locationFilteredUIDs[string(wl.UID)] { continue } - deps := deploysByWorkload[wUID] var totalReady, totalUpdated, totalDesired int32 - for _, d := range deps { + for _, d := range deploysByWorkload[string(wl.UID)] { totalReady += d.Status.ReadyReplicas totalUpdated += d.Status.UpdatedReplicas totalDesired += d.Status.DesiredReplicas @@ -149,114 +254,215 @@ func runList(cmd *cobra.Command, _ []string) error { health := util.WorkloadHealth(wl.Status.Conditions, totalReady, totalDesired) healthShort := strings.SplitN(health, " ", 2)[0] // e.g. "Available", "Degraded" - // Health filter. - if healthFilter != "" && !strings.EqualFold(healthShort, healthFilter) { + if opts.health != "" && !strings.EqualFold(healthShort, opts.health) { continue } - // Placement names. - var placementNames []string - for _, p := range wl.Spec.Placements { - placementNames = append(placementNames, p.Name) - } - placements := strings.Join(placementNames, ", ") - if placements == "" { - placements = "(none)" - } - - // Image from first container. - image := "(vm)" + image := "" if wl.Spec.Template.Spec.Runtime.Sandbox != nil && len(wl.Spec.Template.Spec.Runtime.Sandbox.Containers) > 0 { - image = truncateImage(wl.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image) + image = wl.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image } - readyStr := fmt.Sprintf("%d/%d", totalReady, totalDesired) - upToDateStr := fmt.Sprintf("%d/%d", totalUpdated, totalDesired) - instType := wl.Spec.Template.Spec.Runtime.Resources.InstanceType - rows = append(rows, workloadRow{ name: wl.Name, health: health, healthShort: healthShort, - readyStr: readyStr, - upToDateStr: upToDateStr, - placements: placements, + ready: fmt.Sprintf("%d/%d", totalReady, totalDesired), + upToDate: fmt.Sprintf("%d/%d", totalUpdated, totalDesired), + placements: placementNames(wl), + locations: placementLocations(wl), image: image, age: util.RelativeAge(wl.CreationTimestamp), - instType: instType, + instType: wl.Spec.Template.Spec.Runtime.Resources.InstanceType, + url: urlOf(urls[wl.Name]), + workload: wl, }) } - // Tally health counts from the filtered rows (W9: count after filtering). - healthCounts := map[string]int{ - "Available": 0, - "Degraded": 0, - "Unavailable": 0, - "Unknown": 0, + return listing{rows: rows, urlErr: urlErr}, nil +} + +// urlOf is the one URL to show for a workload, or "" when it has none. The +// url package has already picked between a custom hostname and the +// platform-managed one; a nil Info is an unpublished workload. +func urlOf(info *url.Info) string { + if info == nil { + return "" } - for _, r := range rows { - switch r.healthShort { - case "Available": - healthCounts["Available"]++ - case "Degraded": - healthCounts["Degraded"]++ - case "Unavailable": - healthCounts["Unavailable"]++ - default: - healthCounts["Unknown"]++ + return info.URL +} + +// placementNames lists the workload's placements in declared order. +func placementNames(wl *computev1alpha.Workload) []string { + names := make([]string, 0, len(wl.Spec.Placements)) + for _, p := range wl.Spec.Placements { + names = append(names, p.Name) + } + return names +} + +// placementLocations lists every location the workload places into, +// deduplicated, in declared order. +// +// A placement can name its locations, resolve them through a topology +// selector, or still carry city codes stored before placement moved to +// locations. A selector cannot be expanded without asking the server, so it is +// reported as the selector itself — the same thing describe shows. +func placementLocations(wl *computev1alpha.Workload) []string { + var locations []string + seen := map[string]bool{} + add := func(name string) { + if name != "" && !seen[name] { + seen[name] = true + locations = append(locations, name) + } + } + for _, p := range wl.Spec.Placements { + for _, ref := range p.Locations { + add(ref.Name) + } + if len(p.Locations) == 0 { + if p.LocationSelector != nil { + add("selector: " + metav1.FormatLabelSelector(p.LocationSelector)) + continue + } + for _, city := range p.CityCodes { + add(city) + } } } + return locations +} - out := cmd.OutOrStdout() +// viewsOf converts rows to their machine-readable form. It always returns a +// non-nil slice so an empty project encodes as [] rather than null — `jq` over +// an empty project should iterate nothing, not fail. +func viewsOf(l listing) []workloadView { + // The lookup failed for the project, so it failed for every row: none of + // their URLs is known, and saying nothing at all would read as "none". + urlError := "" + if l.urlErr != nil { + urlError = l.urlErr.Error() + } - if len(rows) == 0 { - if healthFilter != "" { - fmt.Fprintf(out, "No workloads in project %s match health=%s.\n", project, healthFilter) - } else if locationFilter != "" { - fmt.Fprintf(out, "No workloads in project %s have a placement in location %s.\n", project, locationFilter) - } else { - fmt.Fprintf(out, "No workloads found in project %s.\n\n", project) - fmt.Fprintf(out, "Get started:\n") - fmt.Fprintf(out, " datumctl compute deploy api --image=ghcr.io/acme/api:v1.0.0 --location=us-east-1\n") - } - return nil + views := make([]workloadView, 0, len(l.rows)) + for _, r := range l.rows { + views = append(views, workloadView{ + Name: r.name, + Health: r.health, + Ready: r.ready, + UpToDate: r.upToDate, + Placements: r.placements, + Locations: r.locations, + Image: r.image, + InstanceType: r.instType, + Age: r.age, + URL: r.url, + URLError: urlError, + Workload: r.workload, + }) } + return views +} + +func renderTable(out io.Writer, l listing, opts listOptions) { + wide := opts.output == util.OutputWide + urlsKnown := l.urlsKnown() tw := util.NewTabWriter(out) - if !noHeaders { + if !opts.noHeaders { if wide { - fmt.Fprintf(tw, "NAME\tHEALTH\tREADY\tUP-TO-DATE\tPLACEMENTS\tIMAGE\tAGE\tINSTANCE TYPE\n") + fmt.Fprintf(tw, "NAME\tLOCATIONS\tHEALTH\tREADY\tUP-TO-DATE\tPLACEMENTS\tIMAGE\tAGE\tINSTANCE TYPE\tURL\n") } else { - fmt.Fprintf(tw, "NAME\tHEALTH\tREADY\tUP-TO-DATE\tPLACEMENTS\tIMAGE\tAGE\n") + fmt.Fprintf(tw, "NAME\tLOCATIONS\tHEALTH\tREADY\tUP-TO-DATE\tPLACEMENTS\tIMAGE\tAGE\tURL\n") } } - for _, r := range rows { + for _, r := range l.rows { if wide { - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - r.name, r.healthShort, r.readyStr, r.upToDateStr, r.placements, r.image, r.age, r.instType) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + r.name, locationsColumn(r.locations), r.healthShort, r.ready, r.upToDate, + columnOrNone(r.placements), truncateImage(r.image), r.age, r.instType, + urlColumn(r.url, urlsKnown)) } else { - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - r.name, r.healthShort, r.readyStr, r.upToDateStr, r.placements, r.image, r.age) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + r.name, locationsColumn(r.locations), r.healthShort, r.ready, r.upToDate, + columnOrNone(r.placements), truncateImage(r.image), r.age, + urlColumn(r.url, urlsKnown)) } } _ = tw.Flush() - // Footer summary. - fmt.Fprintf(out, "\n%d workloads — %d Available, %d Degraded, %d Unavailable, %d Unknown\n", - len(rows), - healthCounts["Available"], - healthCounts["Degraded"], - healthCounts["Unavailable"], - healthCounts["Unknown"], - ) + fmt.Fprintf(out, "\n%s\n", healthSummary(l.rows)) +} - return nil +// locationsColumn renders the LOCATIONS cell. A workload can be placed in more +// locations than a terminal column can hold, so past maxTableLocations the cell +// names the first few and counts the rest; `-o json` still carries all of +// them under "locations". +func locationsColumn(locations []string) string { + if len(locations) <= maxTableLocations { + return columnOrNone(locations) + } + return fmt.Sprintf("%s (+%d)", strings.Join(locations[:maxTableLocations], ", "), len(locations)-maxTableLocations) +} + +// urlColumn renders the URL cell: the URL, "—" when the workload has none, and +// "?" when the project's URLs could not be read at all. +func urlColumn(u string, urlsKnown bool) string { + switch { + case u != "": + return u + case !urlsKnown: + return unknownURL + default: + return noURL + } +} + +// columnOrNone joins a list for a table cell, or says so when it is empty. +func columnOrNone(values []string) string { + if len(values) == 0 { + return "(none)" + } + return strings.Join(values, ", ") +} + +// healthSummary tallies health across the rows that survived filtering. +func healthSummary(rows []workloadRow) string { + counts := map[string]int{} + for _, r := range rows { + switch r.healthShort { + case "Available", "Degraded", "Unavailable": + counts[r.healthShort]++ + default: + counts["Unknown"]++ + } + } + return fmt.Sprintf("%d workloads — %d Available, %d Degraded, %d Unavailable, %d Unknown", + len(rows), counts["Available"], counts["Degraded"], counts["Unavailable"], counts["Unknown"]) +} + +// printNoRows explains an empty table in terms of whatever the user asked for. +func printNoRows(out io.Writer, project string, opts listOptions) { + switch { + case opts.health != "": + fmt.Fprintf(out, "No workloads in project %s match health=%s.\n", project, opts.health) + case opts.location != "": + fmt.Fprintf(out, "No workloads in project %s have a placement in location %s.\n", project, opts.location) + default: + fmt.Fprintf(out, "No workloads found in project %s.\n\n", project) + fmt.Fprintf(out, "Get started:\n") + fmt.Fprintf(out, " datumctl compute deploy api --image=ghcr.io/acme/api:v1.0.0 --location=us-east-1\n") + } } // truncateImage strips the registry host from an image reference so the table // column stays compact. "ghcr.io/acme/api:v1" → "acme/api:v1". func truncateImage(image string) string { + if image == "" { + return "(vm)" + } parts := strings.SplitN(image, "/", 2) if len(parts) == 2 { // Only strip the first component if it looks like a registry host @@ -441,6 +647,23 @@ func runDescribe(cmd *cobra.Command, args []string) error { return nil } +// placementLocationsSummary says where a placement runs: the locations it +// names, or the topology selector it resolves through. +func placementLocationsSummary(p computev1alpha.WorkloadPlacement) string { + if p.LocationSelector != nil { + return "selector: " + metav1.FormatLabelSelector(p.LocationSelector) + } + if len(p.CityCodes) > 0 { + // Stored before placement moved to locations and not yet rewritten. + return "cities: " + strings.Join(p.CityCodes, ", ") + } + names := make([]string, 0, len(p.Locations)) + for _, ref := range p.Locations { + names = append(names, ref.Name) + } + return "locations: " + strings.Join(names, ", ") +} + // degradedAnnotation returns a short annotation for a per-location line when the // deployment is not fully ready. It reads the blocking reason+message from the // deployment's own Available condition, which the server rolls up from the @@ -471,20 +694,3 @@ func formatEnvVar(e corev1.EnvVar) string { } return fmt.Sprintf("%-20s %s", e.Name, e.Value) } - -// placementLocationsSummary says where a placement runs: the locations it -// names, or the topology selector it resolves through. -func placementLocationsSummary(p computev1alpha.WorkloadPlacement) string { - if p.LocationSelector != nil { - return "selector: " + metav1.FormatLabelSelector(p.LocationSelector) - } - if len(p.CityCodes) > 0 { - // Stored before placement moved to locations and not yet rewritten. - return "cities: " + strings.Join(p.CityCodes, ", ") - } - names := make([]string, 0, len(p.Locations)) - for _, ref := range p.Locations { - names = append(names, ref.Name) - } - return "locations: " + strings.Join(names, ", ") -} diff --git a/internal/cmd/compute/workloads/workloads_test.go b/internal/cmd/compute/workloads/workloads_test.go new file mode 100644 index 00000000..7cd79b7e --- /dev/null +++ b/internal/cmd/compute/workloads/workloads_test.go @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package workloads + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + testProject = "acme-prod" + locEast = "us-east-1" + locWest = "eu-west-1" + locFra = "ap-south-1" + locLhr = "sa-east-1" + workerName = "worker" + testCanonical = "a1b2c3d4.datumproxy.net" + testCustom = "api.example.com" +) + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := computev1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering compute scheme: %v", err) + } + if err := networkingv1alpha.AddToScheme(s); err != nil { + t.Fatalf("registering networking scheme: %v", err) + } + return s +} + +// workload returns a sandbox workload with one placement in the given locations. +func workload(name, uid, image string, locations ...string) *computev1alpha.Workload { + return &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: util.ResourceNamespace, + UID: types.UID(uid), + }, + Spec: computev1alpha.WorkloadSpec{ + Template: computev1alpha.InstanceTemplateSpec{ + Spec: computev1alpha.InstanceSpec{ + Runtime: computev1alpha.InstanceRuntimeSpec{ + Sandbox: &computev1alpha.SandboxRuntime{ + Containers: []computev1alpha.SandboxContainer{{Name: name, Image: image}}, + }, + }, + }, + }, + Placements: []computev1alpha.WorkloadPlacement{{ + Name: "default", + Locations: locationRefs(locations...), + }}, + }, + Status: computev1alpha.WorkloadStatus{ + Conditions: []metav1.Condition{{ + Type: computev1alpha.WorkloadAvailable, + Status: metav1.ConditionTrue, + Reason: "Available", + }}, + }, + } +} + +// locationRefs turns location names into the references a placement stores. +func locationRefs(names ...string) []locationsv1alpha1.LocationReference { + refs := make([]locationsv1alpha1.LocationReference, 0, len(names)) + for _, n := range names { + refs = append(refs, locationsv1alpha1.LocationReference{Name: n}) + } + return refs +} + +// deployment returns a deployment for a workload in one location. +func deployment(name, workloadUID, location string, ready, desired int32) *computev1alpha.WorkloadDeployment { + return &computev1alpha.WorkloadDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: util.ResourceNamespace, + Labels: map[string]string{computev1alpha.WorkloadUIDLabel: workloadUID}, + }, + Spec: computev1alpha.WorkloadDeploymentSpec{ + LocationRef: locationsv1alpha1.LocationReference{Name: location}, + PlacementName: "default", + }, + Status: computev1alpha.WorkloadDeploymentStatus{ + ReadyReplicas: ready, + UpdatedReplicas: ready, + DesiredReplicas: desired, + }, + } +} + +// publishedProxy is the proxy the platform reports for a live URL. +func publishedProxy(workloadName string, customHostnames ...string) *networkingv1alpha.HTTPProxy { + hostnames := make([]gatewayv1.Hostname, 0, len(customHostnames)) + // A custom hostname serves on the strength of its own status entry and + // nothing else: the proxy-level conditions are a roll-up over every + // hostname and cannot vouch for any one of them. + statuses := make([]networkingv1alpha.HostnameStatus, 0, len(customHostnames)) + for _, h := range customHostnames { + hostnames = append(hostnames, gatewayv1.Hostname(h)) + statuses = append(statuses, networkingv1alpha.HostnameStatus{ + Hostname: h, + Conditions: []metav1.Condition{ + {Type: networkingv1alpha.HostnameConditionVerified, Status: metav1.ConditionTrue, Reason: "Verified"}, + {Type: networkingv1alpha.HostnameConditionDNSRecordProgrammed, Status: metav1.ConditionTrue, Reason: "RecordCreated"}, + {Type: networkingv1alpha.HostnameConditionCertificateReady, Status: metav1.ConditionTrue, Reason: "CertificateIssued"}, + }, + }) + } + return &networkingv1alpha.HTTPProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: workloadName, + Namespace: util.ResourceNamespace, + Labels: map[string]string{computev1alpha.WorkloadNameLabel: workloadName}, + }, + Spec: networkingv1alpha.HTTPProxySpec{Hostnames: hostnames}, + Status: networkingv1alpha.HTTPProxyStatus{ + CanonicalHostname: testCanonical, + HostnameStatuses: statuses, + Conditions: []metav1.Condition{ + {Type: networkingv1alpha.HTTPProxyConditionProgrammed, Status: metav1.ConditionTrue, Reason: "Programmed"}, + {Type: networkingv1alpha.HTTPProxyConditionCertificatesReady, Status: metav1.ConditionTrue, Reason: "Issued"}, + }, + }, + } +} + +func publishedService(workloadName string) *networkingv1alpha.NetworkService { + return &networkingv1alpha.NetworkService{ + ObjectMeta: metav1.ObjectMeta{ + Name: workloadName, + Namespace: util.ResourceNamespace, + Labels: map[string]string{computev1alpha.WorkloadNameLabel: workloadName}, + }, + Spec: networkingv1alpha.NetworkServiceSpec{ + Ports: []networkingv1alpha.NetworkServicePort{{Name: "http", Port: 8080}}, + }, + Status: networkingv1alpha.NetworkServiceStatus{ + Summary: networkingv1alpha.NetworkServiceSummary{Locations: 2, Members: 4, Healthy: 4}, + }, + } +} + +func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch { + t.Helper() + return fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(objs...).Build() +} + +// project returns the objects for a project with a published wlAPI and an +// unpublished workerName. +func projectObjects() []client.Object { + return []client.Object{ + workload(wlAPI, "uid-api", "ghcr.io/acme/api:1.4.2", locEast, locWest), + workload(workerName, "uid-worker", "ghcr.io/acme/worker:2.0", locEast), + deployment("api-dfw", "uid-api", locEast, 2, 2), + deployment("api-iad", "uid-api", locWest, 2, 2), + deployment("worker-dfw", "uid-worker", locEast, 1, 1), + publishedProxy(wlAPI, testCustom), + publishedService(wlAPI), + } +} + +func TestListWorkloadsTable(t *testing.T) { + tests := []struct { + name string + objs []client.Object + opts listOptions + wantLines []string + wantMissing []string + }{ + { + name: "url column carries the custom hostname, unpublished shows a dash", + objs: projectObjects(), + opts: listOptions{output: util.OutputTable}, + wantLines: []string{ + "NAME", "URL", + wlAPI, "https://" + testCustom, + workerName, noURL, + }, + }, + { + name: "managed hostname when there is no custom one", + objs: []client.Object{ + workload(wlAPI, "uid-api", "ghcr.io/acme/api:1.4.2", locEast), + deployment("api-dfw", "uid-api", locEast, 2, 2), + publishedProxy(wlAPI), + publishedService(wlAPI), + }, + opts: listOptions{output: util.OutputTable}, + wantLines: []string{"https://" + testCanonical}, + }, + { + name: "no-headers drops the header row but keeps the url", + objs: projectObjects(), + opts: listOptions{output: util.OutputTable, noHeaders: true}, + wantLines: []string{"https://" + testCustom}, + wantMissing: []string{"UP-TO-DATE"}, + }, + { + name: "wide keeps the url last", + objs: projectObjects(), + opts: listOptions{output: util.OutputWide}, + wantLines: []string{"INSTANCE TYPE", "URL", "https://" + testCustom}, + }, + { + name: "location filter still resolves urls", + objs: projectObjects(), + opts: listOptions{output: util.OutputTable, location: locWest}, + wantLines: []string{wlAPI, "https://" + testCustom}, + wantMissing: []string{workerName}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t, tc.objs...) + + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, tc.opts); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + got := out.String() + for _, want := range tc.wantLines { + if !strings.Contains(got, want) { + t.Errorf("output missing %q:\n%s", want, got) + } + } + for _, missing := range tc.wantMissing { + if strings.Contains(got, missing) { + t.Errorf("output should not contain %q:\n%s", missing, got) + } + } + if errOut.Len() != 0 { + t.Errorf("unexpected stderr: %s", errOut.String()) + } + }) + } +} + +// TestListWorkloadsURLField pins the scripting contract from the spec: +// `workloads -o json | jq -r '.[] | select(.name==wlAPI) | .url'`. +func TestListWorkloadsURLField(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t, projectObjects()...) + + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputJSON}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + var views []map[string]any + if err := json.Unmarshal(out.Bytes(), &views); err != nil { + t.Fatalf("output is not a JSON array: %v\n%s", err, out.String()) + } + if len(views) != 2 { + t.Fatalf("got %d views, want 2:\n%s", len(views), out.String()) + } + + byName := map[string]map[string]any{} + for _, v := range views { + name, _ := v["name"].(string) + byName[name] = v + } + + if got := byName[wlAPI]["url"]; got != "https://"+testCustom { + t.Errorf("api url = %v, want https://%s", got, testCustom) + } + // An unpublished workload carries no url field at all — see + // TestListWorkloadsJSONNoURLVersusUnknown for why "" would be a lie. + if _, ok := byName[workerName]["url"]; ok { + t.Errorf("unpublished worker should carry no url field:\n%s", out.String()) + } + + // The raw resource is still there, whole, under "workload". + raw, ok := byName[wlAPI]["workload"].(map[string]any) + if !ok { + t.Fatalf("api view has no workload object:\n%s", out.String()) + } + if raw["spec"] == nil { + t.Errorf("workload object lost its spec:\n%s", out.String()) + } +} + +func TestListWorkloadsYAMLURLField(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t, projectObjects()...) + + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputYAML}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + if !strings.Contains(out.String(), "url: https://"+testCustom) { + t.Errorf("yaml missing url field:\n%s", out.String()) + } +} + +func TestListWorkloadsEmptyJSONIsArray(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t) + + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputJSON}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + if got := strings.TrimSpace(out.String()); got != "[]" { + t.Errorf("empty project encoded as %q, want []", got) + } +} + +// TestListWorkloadsURLsUnreadable: a project whose URLs cannot be read still +// lists its workloads. The column says "unknown", not "none". +func TestListWorkloadsURLsUnreadable(t *testing.T) { + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(projectObjects()...). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*networkingv1alpha.HTTPProxyList); ok { + return errors.New("forbidden") + } + return cl.List(ctx, list, opts...) + }, + }). + Build() + + var out, errOut bytes.Buffer + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputTable}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + if !strings.Contains(out.String(), wlAPI) { + t.Errorf("workloads should still list when their URLs cannot be read:\n%s", out.String()) + } + // Both rows say "unknown", and neither says "none" — an unreadable URL is + // not the same claim as a workload having no URL. + if got := strings.Count(out.String(), unknownURL); got != 2 { + t.Errorf("got %d unknown URL cells, want 2:\n%s", got, out.String()) + } + for _, line := range strings.Split(out.String(), "\n") { + if strings.HasSuffix(strings.TrimSpace(line), noURL) { + t.Errorf("unreadable URLs must not read as no URL:\n%s", out.String()) + } + } + if !strings.Contains(errOut.String(), "could not read URLs") { + t.Errorf("stderr should warn about the URL read:\n%s", errOut.String()) + } +} + +func TestURLColumn(t *testing.T) { + tests := []struct { + name string + url string + urlsKnown bool + want string + }{ + {"published", "https://api.example.com", true, "https://api.example.com"}, + {"unpublished", "", true, noURL}, + {"unknown", "", false, unknownURL}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := urlColumn(tc.url, tc.urlsKnown); got != tc.want { + t.Errorf("urlColumn(%q, %v) = %q, want %q", tc.url, tc.urlsKnown, got, tc.want) + } + }) + } +} + +// manyLocations returns a workload placed in n locations, so the LOCATIONS column has +// to decide what to do with a list that does not fit. +func manyLocations(name, uid string, locations ...string) *computev1alpha.Workload { + return workload(name, uid, "ghcr.io/acme/"+name+":1.0", locations...) +} + +// TestListWorkloadsLocationsColumn pins the spec's list view: the locations a +// workload runs in are a table column, not a JSON-only field. +func TestListWorkloadsLocationsColumn(t *testing.T) { + for _, output := range []util.OutputFormat{util.OutputTable, util.OutputWide} { + t.Run(string(output), func(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t, projectObjects()...) + + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: output}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + got := out.String() + if !strings.Contains(got, "LOCATIONS") { + t.Errorf("table has no LOCATIONS header:\n%s", got) + } + if !strings.Contains(got, locEast+", "+locWest) { + t.Errorf("api row does not name its locations:\n%s", got) + } + // The mock's column order: NAME ... LOCATIONS ... READY ... IMAGE ... URL. + nameAt := strings.Index(got, "NAME") + locationsAt := strings.Index(got, "LOCATIONS") + readyAt := strings.Index(got, "READY") + urlAt := strings.Index(got, "URL") + if nameAt >= locationsAt || locationsAt >= readyAt || readyAt >= urlAt { + t.Errorf("columns out of spec order (name=%d locations=%d ready=%d url=%d):\n%s", + nameAt, locationsAt, readyAt, urlAt, got) + } + }) + } +} + +// TestLocationsColumnTruncates: a workload in many locations must not blow out the +// column, and the cell has to say that it is not the whole list. +func TestLocationsColumnTruncates(t *testing.T) { + tests := []struct { + name string + locations []string + want string + }{ + {"none", nil, "(none)"}, + {"one", []string{locEast}, locEast}, + {"at the limit", []string{locEast, locWest, locFra}, "us-east-1, eu-west-1, ap-south-1"}, + {"over the limit", []string{locEast, locWest, locFra, locLhr}, "us-east-1, eu-west-1, ap-south-1 (+1)"}, + {"well over", []string{locEast, locWest, locFra, locLhr, "af-south-1", "me-south-1"}, "us-east-1, eu-west-1, ap-south-1 (+3)"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := locationsColumn(tc.locations); got != tc.want { + t.Errorf("locationsColumn(%v) = %q, want %q", tc.locations, got, tc.want) + } + }) + } +} + +// TestListWorkloadsLocationsJSONKeepsFullList: truncation is a table concern. +// `-o json` still carries every location. +func TestListWorkloadsLocationsJSONKeepsFullList(t *testing.T) { + all := []string{locEast, locWest, locFra, locLhr, "af-south-1", "me-south-1"} + objs := []client.Object{manyLocations(wlAPI, "uid-api", all...)} + + var tableOut, jsonOut, errOut bytes.Buffer + c := newFakeClient(t, objs...) + if err := listWorkloads(context.Background(), &tableOut, &errOut, c, testProject, listOptions{output: util.OutputTable}); err != nil { + t.Fatalf("listWorkloads (table): %v", err) + } + if !strings.Contains(tableOut.String(), "us-east-1, eu-west-1, ap-south-1 (+3)") { + t.Errorf("table column should name the first locations and count the rest:\n%s", tableOut.String()) + } + if strings.Contains(tableOut.String(), "me-south-1") { + t.Errorf("table column was not truncated:\n%s", tableOut.String()) + } + + c = newFakeClient(t, objs...) + if err := listWorkloads(context.Background(), &jsonOut, &errOut, c, testProject, listOptions{output: util.OutputJSON}); err != nil { + t.Fatalf("listWorkloads (json): %v", err) + } + var views []struct { + Cities []string `json:"locations"` + } + if err := json.Unmarshal(jsonOut.Bytes(), &views); err != nil { + t.Fatalf("output is not a JSON array: %v\n%s", err, jsonOut.String()) + } + if len(views) != 1 { + t.Fatalf("got %d views, want 1:\n%s", len(views), jsonOut.String()) + } + if strings.Join(views[0].Cities, ",") != strings.Join(all, ",") { + t.Errorf("json locations = %v, want %v", views[0].Cities, all) + } +} + +// TestListWorkloadsJSONNoURLVersusUnknown: the table draws "—" for a workload +// with no URL and "?" for URLs that could not be read. The structured output +// has to draw the same distinction — a consumer cannot be told both as `""`. +func TestListWorkloadsJSONNoURLVersusUnknown(t *testing.T) { + t.Run("no url omits the field entirely", func(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t, projectObjects()...) + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputJSON}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + byName := viewsByName(t, out.Bytes()) + if _, ok := byName[workerName]["url"]; ok { + t.Errorf("unpublished workload should not carry a url field:\n%s", out.String()) + } + if _, ok := byName[workerName]["urlError"]; ok { + t.Errorf("unpublished workload is not an error:\n%s", out.String()) + } + if got := byName[wlAPI]["url"]; got != "https://"+testCustom { + t.Errorf("api url = %v, want https://%s", got, testCustom) + } + }) + + t.Run("unreadable urls carry an explicit signal", func(t *testing.T) { + c := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(projectObjects()...). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*networkingv1alpha.HTTPProxyList); ok { + return errors.New("forbidden") + } + return cl.List(ctx, list, opts...) + }, + }). + Build() + + var out, errOut bytes.Buffer + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputJSON}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + + byName := viewsByName(t, out.Bytes()) + for _, name := range []string{wlAPI, workerName} { + if _, ok := byName[name]["url"]; ok { + t.Errorf("%s: url must not be reported when it could not be read:\n%s", name, out.String()) + } + msg, ok := byName[name]["urlError"].(string) + if !ok || msg == "" { + t.Errorf("%s: expected a urlError signal:\n%s", name, out.String()) + } + if !strings.Contains(msg, "forbidden") { + t.Errorf("%s: urlError = %q, want the server's error in it", name, msg) + } + } + }) +} + +// TestListWorkloadsYAMLNoURLVersusUnknown: the YAML form draws the same +// distinction as the JSON one. +func TestListWorkloadsYAMLNoURLVersusUnknown(t *testing.T) { + var out, errOut bytes.Buffer + c := newFakeClient(t, projectObjects()...) + if err := listWorkloads(context.Background(), &out, &errOut, c, testProject, listOptions{output: util.OutputYAML}); err != nil { + t.Fatalf("listWorkloads: %v", err) + } + if strings.Contains(out.String(), `url: ""`) { + t.Errorf("an unpublished workload must not read as an empty url:\n%s", out.String()) + } +} + +func viewsByName(t *testing.T, raw []byte) map[string]map[string]any { + t.Helper() + var views []map[string]any + if err := json.Unmarshal(raw, &views); err != nil { + t.Fatalf("output is not a JSON array: %v\n%s", err, string(raw)) + } + byName := map[string]map[string]any{} + for _, v := range views { + name, _ := v["name"].(string) + byName[name] = v + } + return byName +} From 87d021554f28cee7595dc49302bcace509d181c9 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 18:38:47 -0500 Subject: [PATCH 08/12] test(cli): cover redeploy idempotence for a workload's URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy calls Declare on every run, so the common case is a workload that already has a URL. Pin that a redeploy neither recreates it nor churns it when nothing changed, that a changed port is written through, and that the canonical hostname survives — reissuing it would silently break every reference a developer has already shared. Co-Authored-By: Claude Opus 5 --- internal/cmd/compute/url/redeploy_test.go | 147 ++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 internal/cmd/compute/url/redeploy_test.go diff --git a/internal/cmd/compute/url/redeploy_test.go b/internal/cmd/compute/url/redeploy_test.go new file mode 100644 index 00000000..364a9655 --- /dev/null +++ b/internal/cmd/compute/url/redeploy_test.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package url + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "go.datum.net/compute/internal/cmd/compute/util" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// writeCounts records what a redeploy actually sends to the API server. +type writeCounts struct { + creates int + updates int +} + +func countingClient(t *testing.T, counts *writeCounts, objs ...client.Object) client.WithWatch { + t.Helper() + return interceptor.NewClient(newFakeClient(t, objs...), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + counts.creates++ + return cl.Create(ctx, obj, opts...) + }, + Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + counts.updates++ + return cl.Update(ctx, obj, opts...) + }, + }) +} + +// TestDeclareIsIdempotent: redeploying an unchanged workload must not try to +// create a URL that already exists, and must not churn the objects either. The +// first Declare creates both; a second identical one writes nothing at all. +// +// This is the shape of every redeploy — `deploy` calls Declare on each run, +// including the runs that only change the image. +func TestDeclareIsIdempotent(t *testing.T) { + var counts writeCounts + w := workloadNamed(testWorkloadName) + c := countingClient(t, &counts) + + if err := Declare(context.Background(), c, w, "http", 8080, nil); err != nil { + t.Fatalf("first declare: %v", err) + } + if counts.creates != 2 { + t.Fatalf("creates = %d, want 2 (the backends and the URL)", counts.creates) + } + if counts.updates != 0 { + t.Errorf("updates = %d on a first declare, want 0", counts.updates) + } + + counts = writeCounts{} + if err := Declare(context.Background(), c, w, "http", 8080, nil); err != nil { + t.Fatalf("second declare: %v", err) + } + if counts.creates != 0 { + t.Errorf("creates = %d on redeploy, want 0 — a URL that exists must not be created again", counts.creates) + } + if counts.updates != 0 { + t.Errorf("updates = %d on an unchanged redeploy, want 0 — nothing changed, so nothing should be written", counts.updates) + } +} + +// TestDeclareUpdatesAChangedPort: the counterpart. Idempotence must not mean +// inertness — a workload that moves to another port has to take its URL with +// it, or the URL keeps routing to a port nothing answers on. +func TestDeclareUpdatesAChangedPort(t *testing.T) { + var counts writeCounts + w := workloadNamed(testWorkloadName) + c := countingClient(t, &counts) + + if err := Declare(context.Background(), c, w, "http", 8080, nil); err != nil { + t.Fatalf("first declare: %v", err) + } + + counts = writeCounts{} + if err := Declare(context.Background(), c, w, "http", 9090, nil); err != nil { + t.Fatalf("redeclare on a new port: %v", err) + } + if counts.creates != 0 { + t.Errorf("creates = %d, want 0 — the objects already exist", counts.creates) + } + if counts.updates == 0 { + t.Error("a changed port must be written through to the backends") + } + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("reading back: %v", err) + } + if info == nil { + t.Fatal("the workload lost its URL on redeploy") + } + if info.Port != 9090 { + t.Errorf("port = %d, want 9090", info.Port) + } +} + +// TestDeclareKeepsTheCanonicalHostnameAcrossRedeploys: the managed URL is the +// one a developer has already shared and scripted against. A redeploy that +// replaced the HTTPProxy rather than updating it would issue a new +// .datumproxy.net and silently break every existing reference. +func TestDeclareKeepsTheCanonicalHostnameAcrossRedeploys(t *testing.T) { + w := workloadNamed(testWorkloadName) + c := newFakeClient(t) + + if err := Declare(context.Background(), c, w, "http", 8080, nil); err != nil { + t.Fatalf("first declare: %v", err) + } + + // Stand in for the platform assigning the canonical hostname. + var proxy networkingv1alpha.HTTPProxy + key := types.NamespacedName{Namespace: util.ResourceNamespace, Name: ResourceName(testWorkloadName)} + if err := c.Get(context.Background(), key, &proxy); err != nil { + t.Fatalf("reading the URL back: %v", err) + } + proxy.Status.CanonicalHostname = testCanonical + if err := c.Status().Update(context.Background(), &proxy); err != nil { + t.Fatalf("seeding the canonical hostname: %v", err) + } + + if err := Declare(context.Background(), c, w, "http", 9090, nil); err != nil { + t.Fatalf("redeclare: %v", err) + } + + info, err := ForWorkload(context.Background(), c, testWorkloadName) + if err != nil { + t.Fatalf("reading back: %v", err) + } + if info == nil || info.CanonicalHostname != testCanonical { + t.Fatalf("canonical hostname = %q, want %q — a redeploy must not reissue the URL", + infoCanonical(info), testCanonical) + } +} + +func infoCanonical(i *Info) string { + if i == nil { + return "" + } + return i.CanonicalHostname +} From 38b91f677ce66f7f5bd86c7a4edb30a43a2ac782 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 19:07:00 -0500 Subject: [PATCH 09/12] refactor(cli): drop the domains and open commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Advanced proxy configuration — custom hostnames, DNS verification, path routing, certificates — belongs to dedicated ALB tooling, not the compute plugin. This plugin publishes a URL and stops there. Removes `compute domains` and `compute open`. The per-city backend health view moves into `workloads describe`, which already exists and is where a developer debugging their own workload looks; its health row is labelled Serving so it reads distinctly from the workload's own Health line. deploy still reads and carries forward the custom hostnames already on the proxy, and still fails closed if it cannot read them. That matters more now, not less: hostnames are configured out of band, so a redeploy that clobbered them would break something this plugin never saw being set up. Co-Authored-By: Claude Opus 5 --- docs/enhancements/datumctl-compute-domains.md | 291 --------- docs/enhancements/datumctl-compute-urls.md | 177 +++++ internal/cmd/compute/deploy/deploy.go | 12 +- internal/cmd/compute/domains/add.go | 577 ---------------- internal/cmd/compute/domains/add_test.go | 616 ------------------ internal/cmd/compute/domains/domains.go | 255 -------- internal/cmd/compute/domains/domains_test.go | 449 ------------- internal/cmd/compute/domains/hostname.go | 100 --- internal/cmd/compute/domains/hostname_test.go | 108 --- internal/cmd/compute/domains/remove.go | 193 ------ internal/cmd/compute/domains/remove_test.go | 208 ------ .../cmd/compute/domains/wait_failure_test.go | 113 ---- internal/cmd/compute/domains/write_test.go | 204 ------ internal/cmd/compute/open/browser.go | 83 --- internal/cmd/compute/open/browser_test.go | 91 --- internal/cmd/compute/open/custom_test.go | 133 ---- internal/cmd/compute/open/open.go | 104 --- internal/cmd/compute/open/open_test.go | 283 -------- internal/cmd/compute/root.go | 4 - internal/cmd/compute/url/publish.go | 4 +- .../cmd/compute/url/publish_failure_test.go | 2 +- internal/cmd/compute/url/publish_test.go | 4 +- internal/cmd/compute/url/render.go | 6 +- internal/cmd/compute/url/render_test.go | 4 +- internal/cmd/compute/workloads/workloads.go | 10 + 25 files changed, 206 insertions(+), 3825 deletions(-) delete mode 100644 docs/enhancements/datumctl-compute-domains.md create mode 100644 docs/enhancements/datumctl-compute-urls.md delete mode 100644 internal/cmd/compute/domains/add.go delete mode 100644 internal/cmd/compute/domains/add_test.go delete mode 100644 internal/cmd/compute/domains/domains.go delete mode 100644 internal/cmd/compute/domains/domains_test.go delete mode 100644 internal/cmd/compute/domains/hostname.go delete mode 100644 internal/cmd/compute/domains/hostname_test.go delete mode 100644 internal/cmd/compute/domains/remove.go delete mode 100644 internal/cmd/compute/domains/remove_test.go delete mode 100644 internal/cmd/compute/domains/wait_failure_test.go delete mode 100644 internal/cmd/compute/domains/write_test.go delete mode 100644 internal/cmd/compute/open/browser.go delete mode 100644 internal/cmd/compute/open/browser_test.go delete mode 100644 internal/cmd/compute/open/custom_test.go delete mode 100644 internal/cmd/compute/open/open.go delete mode 100644 internal/cmd/compute/open/open_test.go diff --git a/docs/enhancements/datumctl-compute-domains.md b/docs/enhancements/datumctl-compute-domains.md deleted file mode 100644 index a5faee81..00000000 --- a/docs/enhancements/datumctl-compute-domains.md +++ /dev/null @@ -1,291 +0,0 @@ -# `datumctl compute` — URLs and Domains - -**Status:** Draft -**Companion:** [`datumctl-compute-dx.md`](./datumctl-compute-dx.md) (the DX arc this extends), [`../scoping/alb-workload-exposure.md`](../scoping/alb-workload-exposure.md) (technical scoping) - ---- - -## Summary - -`datumctl compute deploy --port=8080` runs a container across multiple cities and gives the developer nothing to point a browser at. This proposes closing that gap by adopting the conventions every comparable platform already uses, rather than inventing a Datum-specific vocabulary for it. - -Three breaking changes, stated up front: - -1. **`--port` becomes `--http-port`.** The name declares the contract: this is an HTTP service, and it gets a URL. -2. **There is no `expose` / `unexpose` verb.** Custom hostnames live under `domains`, matching Heroku, Vercel, and Railway. -3. **The managed URL is automatic and permanent** for any workload that declares an HTTP port. It is not a resource the developer creates or deletes. - -Plus one addition that costs almost nothing and that four of five comparable CLIs ship: **`datumctl compute open`**. - ---- - -## What the industry actually does - -| Platform | How you declare "this is a web service" | Platform URL | Custom domain | Open in browser | -|---|---|---|---|---| -| **Heroku** | `web:` process type in `Procfile` | automatic | `heroku domains:add` | `heroku open` | -| **Fly.io** | `[http_service]` block in `fly.toml` | automatic | `fly certs add` | `fly open` | -| **Railway** | port detection, then Generate Domain | on request | `railway domain` | `railway open` | -| **Render** | service type: Web Service vs Private Service | automatic | dashboard | — | -| **Cloud Run** | everything is HTTP | automatic | `gcloud run domain-mappings create` | — | -| **Kubernetes** | `Service` + `Ingress`/`Gateway` | none | — | — | - -Three patterns hold across all of them, and they are the whole basis of this proposal. - -**1. The declared *role* determines exposure — never a port number.** Heroku routes the `web:` process and not `worker:`. Render makes you pick Web Service or Private Service. Fly has an `[http_service]` block whose port field is literally named `internal_port`. Not one of these platforms treats "a port exists" as "publish it." - -This is the answer to whether `--port` is intuitive: it isn't, and neither is adding `--public` next to it. `--port` says *what is listening*. Every platform above makes the developer say *what the thing is*. `--http-port` does that in one flag — you cannot type `--http-port 5432` for a Postgres and be surprised by the result, because the flag name states the contract. - -**2. The platform URL is free, automatic, and not a managed object.** Nobody runs `heroku expose`. The URL exists because the service is a web service. This kills `expose` and `unexpose` as concepts: to stop serving, you stop declaring an HTTP port, exactly as you would delete an `[http_service]` block on Fly. - -**3. "Domain" is the standard word — "expose" is not.** `heroku domains:add`, `vercel domains add`, `railway domain`. The one platform that says "expose" is Kubernetes, where `kubectl expose` defaults to a **ClusterIP** — internal. Borrowing that verb for a public-internet action would be borrowing the least familiar option *and* the one that means the opposite thing. - -Datum Compute is general-purpose — sandboxes, VMs, workers, multi-city placement — so it cannot follow Cloud Run and assume everything is HTTP. It sits closest to Fly and Render, and it should follow them: **make the role explicit, then make everything after it automatic.** - ---- - -## Product principles - -**1. Declaring an HTTP port is declaring a web service.** One flag, one meaning, no confirmation flag stacked on top of it. - -**2. The URL is free, so it is automatic.** A managed URL costs the developer nothing, which is what licenses issuing one without asking. Every platform in the table above makes the same trade. - -**3. The URL is the deliverable.** Last line of output, on its own, copy-pasteable. - -**4. Never name the machinery.** The developer sees "URL," "backends," "certificate." Never `NetworkService`, `HTTPProxy`, or a condition reason. Per the DX doc, raw platform state stays behind `describe` and `-o yaml`. - -**5. Never invent resources that don't exist.** No `endpoint/api created` for a kind nobody can `datumctl get`. Plain language in normal output; `-o yaml` shows the real objects. - -**6. One workload, one URL.** Path routing, multiple hostnames, and multi-backend fan-out are manifest territory. - ---- - -## Workflows - -### 1. Deploy and get a URL - -``` -$ datumctl compute deploy api \ - --image=ghcr.io/acme/api:1.4.2 \ - --city=DFW,IAD \ - --min=2 \ - --http-port=8080 - -Resolving workload "api" in project acme-prod... - Placement "default": cities=[DFW, IAD], min=2 - HTTP service: port 8080 → Datum-managed URL - -Apply? (Y/n): y - workload/api created -Saved workload.yaml - -Waiting for rollout. Ctrl-C to detach (rollout continues in background). - - PLACEMENT CITY UPDATED READY OLD PHASE - default DFW 2 0 0 Pending - default IAD 2 0 0 Pending - default DFW 2 2 0 Done - default IAD 2 2 0 Done - -Rollout complete in 47s. - -Publishing... - Backends 4 healthy across DFW, IAD - Edge programmed - Certificate issued - - https://a1b2c3d4.datumproxy.net -``` - -The URL objects are created alongside the workload, not after the rollout, so backends register as instances come up and the URL is ready within a second or two of the last city reaching `Done`. Creating them afterward would add a visible stall to every deploy. - -A workload with no `--http-port` deploys exactly as it does today and says so, so the developer is never left guessing: - -``` -Rollout complete in 47s. - - No HTTP port declared — this workload is not reachable from the internet. - To publish it: datumctl compute deploy api --http-port 8080 -``` - -### 2. Open it - -``` -$ datumctl compute open api -Opening https://a1b2c3d4.datumproxy.net -``` - -`--url` prints without opening, for piping. This is `heroku open` / `fly open` / `railway open`, and it is the single most-used command in that family after deploy. - -### 3. Find the URL again - -There is no `status` command in this CLI today (the DX doc proposes one; it was never built), so -the URL surfaces in the two places a developer already looks — the list view: - -``` -$ datumctl compute workloads - - NAME CITIES READY IMAGE URL - api DFW, IAD 4/4 ghcr.io/acme/api:1.4.2 https://api.example.com - worker DFW 1/1 ghcr.io/acme/worker:2.0 — -``` - -and as a field for scripting, so `| jq -r .url` works: - -``` -$ datumctl compute workloads -o json | jq -r '.[] | select(.name=="api") | .url' -$ datumctl compute open api --url # the single-workload shortcut -``` - -When a `status` command does land, the URL belongs in its header — but nothing here should wait on it. - -### 4. Diagnose a URL that isn't working - -Passing a workload to `domains` gives the per-URL detail view. This is what makes multi-city legible — the thing a developer is paying Datum for and currently cannot see: - -``` -$ datumctl compute domains api - -URL https://api.example.com - https://a1b2c3d4.datumproxy.net -Backend port 8080/tcp - -Health Degraded — 2 of 4 backends healthy - - CITY BACKENDS HEALTHY SERVING - DFW 2 2 yes - IAD 2 0 no - - IAD: no healthy backends — instances are running but not passing health checks. - Traffic is being served from DFW only. - - Next steps: - Check instances: datumctl compute instances --workload=api --city=IAD - Check logs: datumctl compute logs api --city=IAD -``` - -Per the DX doc's rule, the CLI renders whatever reason and message the server emits rather than branching on reason strings, so new platform conditions surface without a CLI release. - -### 5. Add a custom domain *(phase 2)* - -Modeled directly on `heroku domains:add` and `fly certs add`: print the records, then poll. - -``` -$ datumctl compute domains add api api.example.com - -Verifying example.com... - - Add these DNS records: - - TYPE NAME VALUE - TXT _datum.example.com datum-verify=8f3a91c2b7 - CNAME api.example.com a1b2c3d4.datumproxy.net - -Waiting for DNS. Ctrl-C to detach — run 'datumctl compute domains' to check. - - Domain verified ✓ - DNS record programmed ✓ - Certificate issued ✓ - - https://api.example.com -``` - -``` -$ datumctl compute domains - - WORKLOAD DOMAIN STATUS CERTIFICATE - api api.example.com active valid - api a1b2c3d4.datumproxy.net active valid (Datum-managed) - web www.acme.io pending waiting for DNS -``` - -``` -$ datumctl compute domains remove api api.example.com -``` - -The managed URL always appears in this list and cannot be removed. It is a stable fallback and a useful health-check target. - -### 6. Stop serving - -There is no `unexpose`, for the same reason Heroku has no such command. Remove the HTTP port: - -``` -$ datumctl compute deploy api --image=... --city=DFW,IAD --no-http - - HTTP service: removed — https://a1b2c3d4.datumproxy.net will stop responding -Apply? (Y/n): -``` - -And `destroy` states the consequence rather than silently dropping it: - -``` -$ datumctl compute destroy api - -Workload: api -Placements: 1 Cities: DFW, IAD -Min replicas: 2 -URLs: https://api.example.com, https://a1b2c3d4.datumproxy.net - -This will delete the workload, all its instances, and its URLs. Continue? (y/N): -``` - ---- - -## Command reference - -New: - -``` -datumctl compute open Open the workload's URL in a browser (--url to print) -datumctl compute domains List domains across the project -datumctl compute domains Per-URL detail: hostnames, certificates, per-city backends -datumctl compute domains add Attach a custom hostname (phase 2) -datumctl compute domains remove Detach a custom hostname (phase 2) -``` - -Changed: - -``` -datumctl compute deploy --port → --http-port (breaking); --no-http removes it -datumctl compute workloads gains a URL column and a `url` field in json/yaml output -datumctl compute destroy lists URLs in its summary and deletes them -``` - -### On breaking `--port` - -`--port` is removed, not silently aliased. Silently mapping it to `--http-port` would publish every existing CI workload on the next plugin upgrade — precisely the surprise this design exists to avoid. For one release it remains registered and **errors**: - -``` -Error: --port has been replaced by --http-port, which publishes the workload on a -public HTTPS URL. Use --http-port 8080 to publish, or --no-http to keep it internal. -``` - -That is a loud, one-line migration, and the plugin is activation-gated and alpha — the audience for this break is small and reachable. - ---- - -## What this deliberately does not do - -- **No path or header routing.** One backend at `/`. Anyone needing `/api` and `/admin` split across workloads writes an `HTTPProxy` manifest. -- **No TCP or UDP exposure.** HTTP/HTTPS only, matching the platform's proxy. -- **No plaintext HTTP.** Always `https://`. No `--insecure`. -- **No backend TLS.** The edge reaches instances over plaintext inside the network. A container terminating TLS itself will not work, and the CLI must say so clearly rather than producing a broken URL. -- **No domain deletion on `destroy`.** A verified domain is a project asset that outlives any one workload. - ---- - -## Phasing - -**Phase 1 — the URL.** `--http-port`, automatic managed URL, `open`, URL in `status` / `workloads` / `destroy`, `--url` diagnostics, the `--port` migration error. This is the entire product promise. - -**Phase 2 — domains.** `domains` list/add/remove, verification UX, certificate and DNS progress. - -**Phase 3 — driven by usage.** Multiple ports, health-check configuration, request metrics on the URL view. - ---- - -## Open questions - -1. **What should the managed hostname look like?** `.datumproxy.net` is stable and collision-free but unmemorable and awkward to share. Heroku moved off `.herokuapp.com` to random-word names for exactly the squatting reason; Fly uses `.fly.dev` and accepts the collision namespace. This is a platform decision the CLI inherits, but it shapes the first-run experience more than anything else in this document, and PR 411 is still a draft — worth pushing on now. - -2. **How does `deploy -f workload.yaml` declare an HTTP service?** The manifest path has no `--http-port` equivalent. Fly's answer is that the manifest *is* the declaration (`[http_service]`), which suggests the right answer here is a field on the workload spec rather than CLI-only sugar. That is an API conversation with the compute team, not a CLI one, and until it happens the flag and manifest paths do not converge the way the DX doc claims they do. diff --git a/docs/enhancements/datumctl-compute-urls.md b/docs/enhancements/datumctl-compute-urls.md new file mode 100644 index 00000000..29ca8abf --- /dev/null +++ b/docs/enhancements/datumctl-compute-urls.md @@ -0,0 +1,177 @@ +# `datumctl compute` — Workload URLs + +**Status:** Draft +**Companion:** [`datumctl-compute-dx.md`](./datumctl-compute-dx.md) (the DX arc this extends) + +--- + +## Summary + +`datumctl compute deploy --port=8080` ran a container across multiple cities and gave the developer nothing to point a browser at. This closes that gap with the smallest surface that does it: **declaring an HTTP port publishes the workload on a Datum-managed HTTPS URL.** + +One breaking change — `--port` becomes `--http-port` — and no new commands. + +--- + +## Scope: this plugin publishes a URL, it does not configure a proxy + +Advanced proxy configuration belongs to dedicated ALB tooling: custom hostnames and their DNS verification, path and header routing, multiple backends, certificates, header rewriting, timeouts. This plugin deliberately owns none of it. + +What it owns is the zero-config path, because that is the part a *compute* user needs and the part that was missing: run a container, get a working URL. Anything beyond that is a proxy configuration question, and answering it here would mean this plugin growing a second product inside it. + +That boundary is why there is no `compute domains` command group and no `compute open`. It also sets one hard requirement in the other direction: **because hostnames are configured out of band, publishing must never clobber them.** `deploy` reads the custom hostnames already on the proxy and carries them forward on every redeploy, and fails closed if it cannot read them — a redeploy that silently detached someone's production hostname would be far worse than a redeploy that stops and says why. + +--- + +## Product principles + +**1. Declaring an HTTP port is declaring a web service.** One flag, one meaning. `--port` said *what is listening*, not *who can reach it*; in Kubernetes, which this platform is, `containerPort` exposes nothing at all. Every comparable platform makes the declared role decide exposure instead — Heroku routes `web:`, Render makes you pick Web Service or Private Service, Fly's port field lives inside `[http_service]`. `--http-port` carries that contract in the name, so no second confirmation flag is needed. + +**2. The URL is free, so it is automatic.** A managed URL costs the developer nothing, which is what licenses issuing one without asking. + +**3. The URL is the deliverable.** Last line of output, on its own, copy-pasteable. + +**4. Never name the machinery.** The developer sees "URL", "backends", "certificate" — never `NetworkService`, `HTTPProxy`, or a raw condition reason. Blocking text is routed through `url.HumanBlock`, which shows the server's message and never its reason. + +**5. Never invent resources that don't exist.** No `endpoint/api created` for a kind nobody can `datumctl get`. + +--- + +## The experience + +### Deploy and get a URL + +``` +$ datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --city=DFW,IAD --min=2 --http-port=8080 + +Resolving workload "api" in project acme-prod... + Placement "default": cities=[DFW, IAD], min=2 + HTTP service: port 8080 → Datum-managed URL + +Apply? (Y/n): y + workload/api created + +Waiting for rollout. Ctrl-C to detach (rollout continues in background). + + PLACEMENT CITY UPDATED READY OLD PHASE + default DFW 2 2 0 Done + default IAD 2 2 0 Done + +Rollout complete in 47s. + +Publishing... + Backends 4 healthy across DFW, IAD + Edge programmed + Certificate issued + + https://a1b2c3d4.datumproxy.net +``` + +The URL objects are written alongside the workload, not after the rollout, so backends register as instances come up and the URL answers moments after the last city reaches `Done`. + +A workload with no `--http-port` deploys as it always did, and says so rather than leaving the developer guessing: + +``` +Rollout complete in 47s. + + No HTTP port declared — this workload is not reachable from the internet. + To publish it: datumctl compute deploy api --http-port 8080 +``` + +### Find the URL again + +There is no `status` command in this CLI (the DX doc proposes one that was never built), so the URL surfaces where a developer already looks. In the list: + +``` +$ datumctl compute workloads + + NAME CITIES READY IMAGE URL + api DFW, IAD 4/4 ghcr.io/acme/api:1.4.2 https://api.example.com + worker DFW 1/1 ghcr.io/acme/worker:2.0 — +``` + +As a field, so `| jq -r .url` works: `datumctl compute workloads -o json`. + +And in `describe`, with per-city backend health — the view that makes multi-city routing visible, which it previously was not: + +``` +$ datumctl compute workloads describe api + +Workload api project: acme-prod +Type sandbox/datumcloud/d1-standard-2 +Updated 4m ago + +Health Available + +URL https://api.example.com +Backend port 8080/tcp + +Serving Degraded — 2 of 4 backends healthy + + CITY BACKENDS HEALTHY SERVING + DFW 2 2 yes + IAD 2 0 no + + IAD: no healthy backends — instances are running but not passing health checks. + Traffic is being served from DFW only. + + Next steps: + Check instances: datumctl compute instances --workload=api --city=IAD +``` + +### Stop serving + +`--no-http` removes the HTTP service and the URL with it, naming what will stop answering before the prompt. `destroy` does the same as part of its summary: + +``` +$ datumctl compute destroy api + +Workload: api +Placements: 1 Cities: DFW, IAD +Min replicas: 2 +URLs: https://api.example.com, https://a1b2c3d4.datumproxy.net + +This will delete the workload, all its instances, and its URLs. Continue? (y/N): +``` + +The URL resources are deleted explicitly rather than by owner-reference GC, which is unverified in project virtual control planes. Domain objects are left alone — a verified domain is a project asset that outlives any one workload. + +--- + +## Command surface + +Changed, and nothing added: + +``` +datumctl compute deploy --port → --http-port (breaking); --no-http removes it +datumctl compute workloads URL column, and a `url` field in -o json/yaml +datumctl compute workloads describe URL, backend port, and per-city backend health +datumctl compute destroy lists URLs in its summary and deletes them +``` + +### On breaking `--port` + +`--port` is removed, not silently aliased. Aliasing would publish every existing workload on the next plugin upgrade — precisely the surprise this design exists to avoid. For one release it stays registered and hard-errors: + +``` +Error: --port has been replaced by --http-port, which publishes the workload on a public +HTTPS URL. Use --http-port 8080 to publish, or --no-http to keep it internal +``` + +Redeploys are idempotent: an unchanged redeploy writes nothing, omitting `--http-port` inherits the port the workload already declares, and the managed hostname is never reissued — anything already pointing at that URL keeps working. + +--- + +## What this deliberately does not do + +- **No proxy configuration.** Custom hostnames, path and header routing, multiple backends, certificates, rewrites, timeouts. All ALB tooling's job; see the scope section. +- **No TCP or UDP exposure.** HTTP/HTTPS only, matching the platform's proxy. +- **No plaintext HTTP.** Always `https://`. No `--insecure`. +- **No backend TLS.** The edge reaches instances over plaintext inside the network, and the platform rejects backend TLS for this backend form. A container terminating TLS itself will not work, and the CLI says so on the publishing path. +- **One workload, one URL.** + +--- + +## Open question + +**What should the managed hostname look like?** `.datumproxy.net` is stable and collision-free but unmemorable and awkward to share. Heroku moved off `.herokuapp.com` for squatting reasons; Fly uses `.fly.dev` and accepts the collision namespace. This is a platform decision the CLI inherits, but it shapes the first-run experience more than anything else here. diff --git a/internal/cmd/compute/deploy/deploy.go b/internal/cmd/compute/deploy/deploy.go index d2bfb644..bac21477 100644 --- a/internal/cmd/compute/deploy/deploy.go +++ b/internal/cmd/compute/deploy/deploy.go @@ -558,18 +558,18 @@ func publish(ctx context.Context, out io.Writer, c client.Client, w *computev1al // existingHostnames returns the custom hostnames already attached to the // workload's URL, so republishing carries them forward. // -// Publishing rewrites the proxy spec wholesale. Without this, every redeploy -// of a workload would silently detach the hostnames 'datumctl compute domains -// add' put there, and the custom domain would stop answering — an unpublish -// the user never asked for. Detaching a hostname is 'domains remove' and -// nothing else. +// Publishing rewrites the proxy spec wholesale. Custom hostnames are not set by +// this plugin — they are configured out of band, by the ALB tooling that owns +// advanced proxy configuration — so without this every redeploy would silently +// detach them and the custom domain would stop answering. That matters more, +// not less, for hostnames this plugin cannot see itself having added. // // It fails closed. A workload that has never been published has no hostnames // and that is a nil with no error, but a control plane that cannot be read is // an error the caller must stop on: the two calls use different verbs on the // same object — a List here, a Get in the apply — so a control plane that // refuses one and answers the other would otherwise rewrite spec.Hostnames to -// nothing and detach every custom domain while the deploy reported success. +// nothing and report success. func existingHostnames(ctx context.Context, c client.Client, workloadName string) ([]string, error) { info, err := url.ForWorkload(ctx, c, workloadName) if err != nil { diff --git a/internal/cmd/compute/domains/add.go b/internal/cmd/compute/domains/add.go deleted file mode 100644 index fd767a3d..00000000 --- a/internal/cmd/compute/domains/add.go +++ /dev/null @@ -1,577 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "context" - "errors" - "fmt" - "io" - "os" - "os/signal" - "strings" - "time" - - "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - - "go.datum.net/compute/internal/cmd/compute/url" - "go.datum.net/compute/internal/cmd/compute/util" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" -) - -const ( - // pollInterval matches the rollout watcher and the publish watcher, so - // every wait in this CLI ticks at the same rate. - pollInterval = 2 * time.Second - - // maxReadFailures is how many consecutive failed reads the wait rides out - // before giving up and saying why, matching the publish watcher. - maxReadFailures = 3 - - // maxWait bounds the whole wait so no path hangs, however healthy the - // polling looks. - maxWait = 15 * time.Minute - - // blockingGrace is how long to wait before repeating what the server says - // is holding a hostname up. Every hostname starts out unverified; saying so - // in the first second is noise, not diagnosis. - blockingGrace = 20 * time.Second - - // recordWindow is how long the platform gets to work out what it needs - // from the user before this command stops waiting quietly and says so. - // - // A hostname is attached the instant the update lands, which is before the - // platform has created the domain it belongs to or computed the record - // that proves the user owns it. Printing the records block in that moment - // would print it without the one record the whole flow turns on, and there - // is no second chance: the block is printed once. So nothing is announced - // until the records are known — and if they never are, that is said out - // loud rather than waited out in silence. - recordWindow = 60 * time.Second - - // stepWidth aligns the checkmarks in a fixed column. - stepWidth = 24 -) - -// stepLabels turns the per-hostname condition types the platform publishes -// into the words a developer thinks in. This maps condition *types*, which are -// API constants; the reasons behind them are never interpreted, only echoed. -var stepLabels = map[string]string{ - networkingv1alpha.HostnameConditionVerified: "Domain verified", - networkingv1alpha.HostnameConditionAvailable: "Hostname available", - networkingv1alpha.HostnameConditionDNSRecordProgrammed: "DNS record programmed", - networkingv1alpha.HostnameConditionCertificateReady: "Certificate issued", -} - -// stepOrder is the order the steps are reported in when several land at once. -// Conditions outside this list are shown after it, under their own names, so a -// check the platform adds tomorrow reaches the user without a CLI release. -var stepOrder = []string{ - networkingv1alpha.HostnameConditionVerified, - networkingv1alpha.HostnameConditionAvailable, - networkingv1alpha.HostnameConditionDNSRecordProgrammed, - networkingv1alpha.HostnameConditionCertificateReady, -} - -func addCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "add ", - Short: "Attach a custom hostname to a workload", - Long: "Attach a custom hostname to a workload's URL.\n\n" + - "Datum prints the DNS records to create, then waits for the domain to verify,\n" + - "for DNS to resolve, and for a certificate to be issued. Ctrl-C detaches — the\n" + - "work continues, and 'datumctl compute domains' shows where it got to.\n\n" + - "A workload may serve several custom hostnames. Attaching one never replaces\n" + - "another: every hostname on a workload routes to that same workload, and the\n" + - "workload keeps its Datum-managed hostname besides. To stop serving a hostname,\n" + - "detach it with 'datumctl compute domains remove'.\n\n" + - "Serving different content per hostname — path routing, host-based routing,\n" + - "several backends behind one name — is manifest territory, not this command's.", - Example: " # Attach a hostname\n" + - " datumctl compute domains add api api.example.com\n\n" + - " # A second hostname, alongside the first\n" + - " datumctl compute domains add api www.example.com", - Args: cobra.ExactArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - c, err := util.NewClient(util.ProjectFromCmd(cmd)) - if err != nil { - return err - } - // Ctrl-C detaches from the wait. It never cancels the attach: the - // hostname is already on the proxy by the time we are waiting. - ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) - defer cancel() - return runAdd(ctx, cmd.OutOrStdout(), c, util.ProjectFromCmd(cmd), args[0], args[1]) - }, - ValidArgsFunction: util.CompleteWorkloadNames, - } - - return cmd -} - -// runAdd appends a hostname to the workload's URL and waits for it to answer. -// -// Attaching the same hostname twice is not an error: the wait is the useful -// part of this command, and a user re-running it after a Ctrl-C wants to pick -// the wait back up, not to be told off. -func runAdd(ctx context.Context, out io.Writer, c client.Client, project, workloadName, rawHostname string) error { - if ctx == nil { - ctx = context.Background() - } - - hostname, err := validateHostname(rawHostname) - if err != nil { - return err - } - - info, err := urlFor(ctx, c, project, workloadName) - if err != nil { - return err - } - - if sameHostname(info.CanonicalHostname, hostname) { - return fmt.Errorf("%s is already the Datum-managed hostname for workload %q", hostname, workloadName) - } - - fmt.Fprintln(out) - - if attached(info, hostname) { - fmt.Fprintf(out, "%s is already attached to workload %q.\n", hostname, workloadName) - } else { - proxy := info.Proxy.DeepCopy() - proxy.Spec.Hostnames = append(proxy.Spec.Hostnames, gatewayv1.Hostname(hostname)) - if err := c.Update(ctx, proxy); err != nil { - return fmt.Errorf("attaching %s to workload %q: %w", hostname, workloadName, err) - } - - // Attaching is the one part of this command that is already done by - // the time the waiting starts, so it is reported before the wait - // rather than after it. A workload that now answers to more than one - // custom hostname is told so plainly: nothing was replaced. - fmt.Fprintf(out, "%s attached to workload %q.\n", hostname, workloadName) - if total := len(info.CustomHostnames) + 1; total > 1 { - fmt.Fprintf(out, "Workload %q now serves %d custom hostnames; all of them route to it.\n", - workloadName, total) - } - } - - fmt.Fprintln(out) - - return waitForHostname(ctx, out, c, workloadName, hostname) -} - -// attached reports whether a hostname is already on the URL. -func attached(info *url.Info, hostname string) bool { - for _, h := range info.CustomHostnames { - if sameHostname(h, hostname) { - return true - } - } - return false -} - -// waitForHostname polls until the hostname is verified, resolving and covered -// by a certificate, printing each step as the platform reports it. -func waitForHostname(ctx context.Context, out io.Writer, c client.Client, workloadName, hostname string) error { - w := &hostnameWatch{ - out: out, - hostname: hostname, - started: time.Now(), - seen: map[string]bool{}, - } - - // A read that keeps failing is not going to start working, and a wait that - // hides it hangs forever; a single blip is ridden out. Same budget and same - // reasoning as the publish watcher, which had this exact defect. - failures := 0 - step := func() (bool, error) { - done, err := w.check(ctx, c, workloadName) - var re readError - if errors.As(err, &re) { - failures++ - if failures >= maxReadFailures { - return false, fmt.Errorf("checking %s on workload %q: %w", hostname, workloadName, re.err) - } - return false, nil - } - failures = 0 - return done, err - } - - done, err := step() - if done || err != nil { - return err - } - - ticker := time.NewTicker(pollInterval) - defer ticker.Stop() - - // Nothing bounds how long a developer waits on DNS they control, so the - // deadline is the backstop that keeps the command from hanging rather than - // a judgement about how long verification should take. - deadline := time.NewTimer(maxWait) - defer deadline.Stop() - - for { - select { - case <-ctx.Done(): - fmt.Fprintf(out, "\nDetached. Verification continues in the background.\n") - fmt.Fprintln(out, " Check it with: datumctl compute domains") - return nil - - case <-deadline.C: - return fmt.Errorf("%s on workload %q was not verified within %s — check it with: datumctl compute domains", - hostname, workloadName, maxWait) - - case <-ticker.C: - done, err := step() - if done || err != nil { - return err - } - } - } -} - -// readError marks a failure to read the current state, as opposed to a failure -// that means waiting longer is pointless. Reads are retried; anything else ends -// the wait immediately. -type readError struct{ err error } - -func (e readError) Error() string { return e.err.Error() } -func (e readError) Unwrap() error { return e.err } - -// hostnameWatch prints the progress of one hostname exactly once per fact. -type hostnameWatch struct { - out io.Writer - hostname string - started time.Time - seen map[string]bool -} - -// check reads the current state and reports whether the hostname is live, and -// whether waiting any longer would be pointless. -// -// A read failure is reported as a readError rather than swallowed. One is -// nothing — the hostname was just written and the next tick is a better answer -// — but a control plane this command cannot read is not going to start -// answering, and a wait that hides that hangs forever. waitForHostname decides -// how many to ride out. The other error this returns is the one from announce: -// the platform never told the user what to create, so there is nothing left for -// a wait to do. -func (w *hostnameWatch) check(ctx context.Context, c client.Client, workloadName string) (bool, error) { - info, err := url.ForWorkload(ctx, c, workloadName) - if err != nil { - return false, readError{err: err} - } - if info == nil { - return false, nil - } - h := hostnameFrom(info, w.hostname) - if h == nil { - return false, nil - } - - if hostnameLive(h) { - w.steps(h) - fmt.Fprintf(w.out, "\n %s\n", h.URL) - return true, nil - } - - domain := domainFor(ctx, c, w.hostname) - announced, err := w.announce(info, h, domain) - if err != nil { - return false, err - } - if !announced { - // Checkmarks and blocking notes belong under the heading that has not - // been printed yet. Nothing is said about a hostname before the user - // has been told what the hostname is waiting for. - return false, nil - } - - w.steps(h) - - if time.Since(w.started) > blockingGrace { - w.blocking(h, domain) - } - return false, nil -} - -// announce prints the records the user has to create and the note that says -// Ctrl-C is safe, and reports whether it has done so yet. -// -// It waits for the platform to have worked out what it needs. A hostname is -// attached the moment the update lands, which is before the domain behind it -// exists and before the record that proves ownership has been computed; a -// records block printed then is a block with the TXT record missing, and the -// user is left waiting on DNS they were never shown. So until the records are -// known this prints nothing and asks to be called again — with one exception: -// once recordWindow has passed, whatever the platform did publish is printed -// along with a note about what it has not, and if that is nothing at all the -// wait ends with an error rather than continuing in silence. -// -// Nothing here is printed twice, and a record that only turns up later is -// still printed when it does. -func (w *hostnameWatch) announce(info *url.Info, h *url.Hostname, domain *networkingv1alpha.Domain) (bool, error) { - records := recordsFor(info, h, domain) - known := recordsKnown(info, h, domain) - - if !known && time.Since(w.started) < recordWindow { - return false, nil - } - if !known && len(records) == 0 { - return false, fmt.Errorf( - "%s is attached to workload %q, but Datum has published nothing to create for it after %s — check it with: datumctl compute domains", - w.hostname, info.WorkloadName, recordWindow) - } - - name := w.hostname - if domain != nil && domain.Spec.DomainName != "" { - name = domain.Spec.DomainName - } - w.once("verifying", func() { fmt.Fprintf(w.out, "Verifying %s...\n", name) }) - - w.records(records) - - if !known { - w.once("incomplete", func() { - fmt.Fprintf(w.out, "\n Datum has not published a verification record for %s yet.\n", name) - fmt.Fprintln(w.out, " It will appear here if one is needed.") - }) - } - - w.once("waiting", func() { - fmt.Fprintln(w.out, "\nWaiting for DNS. Ctrl-C to detach — run 'datumctl compute domains' to check.") - fmt.Fprintln(w.out) - }) - return true, nil -} - -// records prints the records the user has not been shown yet. Under normal -// conditions that is every record at once, in one block, because announce -// waits for them all; a record the platform only worked out afterwards gets a -// block of its own rather than going unmentioned. -func (w *hostnameWatch) records(records []record) { - var unseen []record - for _, r := range records { - key := "record:" + r.Type + " " + r.Name + " " + r.Value - if w.seen[key] { - continue - } - w.seen[key] = true - unseen = append(unseen, r) - } - if len(unseen) == 0 { - return - } - - fmt.Fprintln(w.out, "\n Add these DNS records:") - fmt.Fprintln(w.out) - tw := util.NewTabWriter(w.out) - fmt.Fprintln(tw, " TYPE\tNAME\tVALUE") - for _, r := range unseen { - fmt.Fprintf(tw, " %s\t%s\t%s\n", r.Type, r.Name, r.Value) - } - _ = tw.Flush() -} - -// once runs f the first time it is asked to, and never again. -func (w *hostnameWatch) once(key string, f func()) { - if w.seen[key] { - return - } - w.seen[key] = true - f() -} - -// recordsKnown reports whether the platform has finished working out what it -// needs from the user for this hostname: whether ownership has to be proved -// and, if so, with which record, and what the hostname should be pointed at. -// -// It is deliberately about the platform having answered rather than about -// there being records to print: a domain already verified in this project -// needs no TXT record, and that is an answer. -func recordsKnown(info *url.Info, h *url.Hostname, domain *networkingv1alpha.Domain) bool { - return verificationKnown(domain) && targetKnown(info, h) -} - -// verificationKnown reports whether the platform has settled how ownership of -// the domain is proved. No Domain at all means it has not started: the -// platform creates one the first time it tries to program a hostname. -func verificationKnown(domain *networkingv1alpha.Domain) bool { - if domain == nil { - return false - } - if conditionTrue(domain.Status.Conditions, networkingv1alpha.DomainConditionVerified) { - return true - } - v := domain.Status.Verification - return v != nil && v.DNSRecord.Type != "" && v.DNSRecord.Name != "" && v.DNSRecord.Content != "" -} - -// targetKnown reports whether the hostname can be pointed somewhere: either -// the platform has assigned the hostname to point at, or it has programmed the -// record itself and there is nothing for the user to point. -func targetKnown(info *url.Info, h *url.Hostname) bool { - return info.CanonicalHostname != "" || - conditionTrue(h.Conditions, networkingv1alpha.HostnameConditionDNSRecordProgrammed) -} - -// steps prints a checkmark for each check the platform reports as passed. -func (w *hostnameWatch) steps(h *url.Hostname) { - for _, condType := range stepOrder { - if c := util.FindCondition(h.Conditions, condType); c != nil && c.Status == metav1.ConditionTrue { - w.step(condType) - } - } - for _, c := range h.Conditions { - if c.Status == metav1.ConditionTrue { - w.step(c.Type) - } - } -} - -// step prints one checkmark row, under the condition's own type when this CLI -// has no friendlier name for it. -func (w *hostnameWatch) step(condType string) { - if w.seen["step:"+condType] { - return - } - w.seen["step:"+condType] = true - - label, ok := stepLabels[condType] - if !ok { - label = condType - } - fmt.Fprintf(w.out, " %-*s ✓\n", stepWidth, label) -} - -// blocking echoes, verbatim and once each, whatever the server says is holding -// the hostname up — including the domain's own verification message, which is -// where "we looked for the TXT record and did not find it" shows up. -func (w *hostnameWatch) blocking(h *url.Hostname, domain *networkingv1alpha.Domain) { - for _, c := range h.Conditions { - if c.Status != metav1.ConditionTrue && c.Reason != "" { - w.note(c.Reason, c.Message) - } - } - if domain == nil { - return - } - if c := util.FindCondition(domain.Status.Conditions, networkingv1alpha.DomainConditionVerified); c != nil && - c.Status != metav1.ConditionTrue && c.Reason != "" { - w.note(c.Reason, c.Message) - } -} - -func (w *hostnameWatch) note(reason, message string) { - text := url.HumanBlock(reason, message) - if w.seen["note:"+text] { - return - } - w.seen["note:"+text] = true - fmt.Fprintf(w.out, " %-*s %s\n", stepWidth, "", text) -} - -// hostnameLive reports whether a hostname is ready to hand to the user: it is -// serving, and its certificate has been issued (or the platform has not -// reported a certificate check for it, in which case there is nothing to wait -// for). -func hostnameLive(h *url.Hostname) bool { - if h == nil || !h.Active { - return false - } - if c := util.FindCondition(h.Conditions, networkingv1alpha.HostnameConditionCertificateReady); c != nil { - return c.Status == metav1.ConditionTrue - } - return true -} - -// hostnameFrom finds one hostname's state in a workload's URL info. -func hostnameFrom(info *url.Info, hostname string) *url.Hostname { - for i := range info.Hostnames { - if sameHostname(info.Hostnames[i].Hostname, hostname) { - return &info.Hostnames[i] - } - } - return nil -} - -// record is one DNS record the user has to create, exactly as the platform -// reports it. Nothing in this file synthesizes a record value: a record the -// server has not told us about is a record we do not print. -type record struct { - Type string - Name string - Value string -} - -// recordsFor returns the records still outstanding for a hostname: the TXT the -// platform wants for proof of ownership, and the CNAME that points the hostname -// at the platform's own. -// -// Each is dropped once the corresponding condition goes True — a domain that is -// already verified needs no TXT, and a hostname whose record the platform -// programmed itself needs no CNAME. -func recordsFor(info *url.Info, h *url.Hostname, domain *networkingv1alpha.Domain) []record { - var records []record - - if domain != nil && !conditionTrue(domain.Status.Conditions, networkingv1alpha.DomainConditionVerified) { - if v := domain.Status.Verification; v != nil { - r := v.DNSRecord - if r.Type != "" && r.Name != "" && r.Content != "" { - records = append(records, record{Type: r.Type, Name: r.Name, Value: r.Content}) - } - } - } - - if info.CanonicalHostname != "" && !conditionTrue(h.Conditions, networkingv1alpha.HostnameConditionDNSRecordProgrammed) { - records = append(records, record{Type: "CNAME", Name: h.Hostname, Value: info.CanonicalHostname}) - } - - return records -} - -func conditionTrue(conditions []metav1.Condition, condType string) bool { - c := util.FindCondition(conditions, condType) - return c != nil && c.Status == metav1.ConditionTrue -} - -// domainFor returns the Domain resource covering a hostname. -// -// The platform creates one automatically the first time it tries to program a -// hostname, so there may not be one yet, and it is matched by suffix: a Domain -// for example.com covers api.example.com. Nothing here creates one — doing so -// would duplicate an object the platform owns. A control plane that does not -// serve Domains at all reads the same as one with no Domain yet: there is -// simply nothing extra to show. -func domainFor(ctx context.Context, c client.Client, hostname string) *networkingv1alpha.Domain { - var list networkingv1alpha.DomainList - if err := c.List(ctx, &list, client.InNamespace(util.ResourceNamespace)); err != nil { - return nil - } - - var best *networkingv1alpha.Domain - for i := range list.Items { - name := normalizeHostname(list.Items[i].Spec.DomainName) - if name == "" || !covers(name, hostname) { - continue - } - // The most specific Domain wins: a project may hold both example.com - // and api.example.com, and the latter is the one being verified. - if best == nil || len(name) > len(normalizeHostname(best.Spec.DomainName)) { - best = &list.Items[i] - } - } - return best -} - -// covers reports whether a domain name covers a hostname, by the platform's own -// rule: an exact match, or a suffix match on a label boundary. -func covers(domainName, hostname string) bool { - h := normalizeHostname(hostname) - return h == domainName || strings.HasSuffix(h, "."+domainName) -} diff --git a/internal/cmd/compute/domains/add_test.go b/internal/cmd/compute/domains/add_test.go deleted file mode 100644 index 8c2999e8..00000000 --- a/internal/cmd/compute/domains/add_test.go +++ /dev/null @@ -1,616 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "bytes" - "context" - "io" - "strings" - "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "go.datum.net/compute/internal/cmd/compute/url" - "go.datum.net/compute/internal/cmd/compute/util" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" -) - -// unverifiedDomain returns a Domain in the shape the platform reports before -// the user has published the TXT record it asked for. -func unverifiedDomain(domainName, recordName, content string) *networkingv1alpha.Domain { - return &networkingv1alpha.Domain{ - ObjectMeta: metav1.ObjectMeta{ - Name: strings.ReplaceAll(domainName, ".", "-"), - Namespace: util.ResourceNamespace, - }, - Spec: networkingv1alpha.DomainSpec{DomainName: domainName}, - Status: networkingv1alpha.DomainStatus{ - Verification: &networkingv1alpha.DomainVerificationStatus{ - DNSRecord: networkingv1alpha.DNSVerificationRecord{ - Name: recordName, - Type: "TXT", - Content: content, - }, - }, - Conditions: []metav1.Condition{ - cond(networkingv1alpha.DomainConditionVerified, metav1.ConditionFalse, "RecordNotFound", "verification record not found"), - }, - }, - } -} - -// cancelled returns a context that is already done, so a wait runs exactly one -// poll and then takes the detach path. -func cancelled() context.Context { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - return ctx -} - -func TestAddAttachesHostname(t *testing.T) { - // The platform has already checked this hostname out — the user is - // re-attaching one it knows — so the wait ends on its first poll and the - // command reaches its URL. A hostname the platform has said nothing about - // is pending, and that wait is TestAddPrintsWhatTheServerReports. - proxy := verifiedProxy(testWorkload, testCanonical, testCustom) - proxy.Spec.Hostnames = nil - - c := newFakeClient(t, - proxy, - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), - ) - - var out bytes.Buffer - if err := runAdd(context.Background(), &out, c, testProject, testWorkload, "HTTPS://API.Example.com./"); err != nil { - t.Fatalf("runAdd: %v", err) - } - - // Stored normalized, so the platform sees the name it can validate. - if got := hostnamesOf(t, c); len(got) != 1 || got[0] != testCustom { - t.Errorf("hostnames = %v, want [%s]", got, testCustom) - } - - // The URL is the deliverable, and it is the last thing printed. - if !strings.Contains(out.String(), "https://"+testCustom) { - t.Errorf("missing the URL:\n%s", out.String()) - } -} - -func TestAddIsIdempotent(t *testing.T) { - c := newFakeClient(t, - verifiedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), - ) - - var out bytes.Buffer - if err := runAdd(context.Background(), &out, c, testProject, testWorkload, testCustom); err != nil { - t.Fatalf("runAdd: %v", err) - } - - if got := hostnamesOf(t, c); len(got) != 1 { - t.Errorf("hostnames = %v, want no duplicate", got) - } - if !strings.Contains(out.String(), "already attached") { - t.Errorf("re-adding should say so plainly:\n%s", out.String()) - } - if !strings.Contains(out.String(), "https://"+testCustom) { - t.Errorf("re-adding should still end at the URL:\n%s", out.String()) - } -} - -func TestAddErrors(t *testing.T) { - tests := []struct { - name string - objects []client.Object - workload string - hostname string - want string - }{ - { - name: "the Datum-managed hostname is not a custom domain", - objects: []client.Object{programmedProxy(testWorkload, testCanonical)}, - workload: testWorkload, - hostname: "https://" + strings.ToUpper(testCanonical) + "/", - want: "already the Datum-managed hostname", - }, - { - name: "wildcard", - objects: []client.Object{programmedProxy(testWorkload, testCanonical)}, - workload: testWorkload, - hostname: "*.example.com", - want: "wildcard hostnames are not supported", - }, - { - name: "not a hostname", - objects: []client.Object{programmedProxy(testWorkload, testCanonical)}, - workload: testWorkload, - hostname: "localhost", - want: "not a fully qualified hostname", - }, - { - name: "workload has no URL to attach to", - objects: []client.Object{workload(testWorkload)}, - workload: testWorkload, - hostname: testCustom, - want: wantPublishHint, - }, - { - name: "workload does not exist", - objects: nil, - workload: "ghost", - hostname: testCustom, - want: "not found in project " + testProject, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - c := newFakeClient(t, tc.objects...) - var out bytes.Buffer - err := runAdd(context.Background(), &out, c, testProject, tc.workload, tc.hostname) - if err == nil { - t.Fatalf("expected an error, got output:\n%s", out.String()) - } - if !strings.Contains(err.Error(), tc.want) { - t.Errorf("error = %q, want it to mention %q", err, tc.want) - } - }) - } -} - -// A hostname that is not live yet prints the records the server asked for, the -// checkmarks for the checks that have passed, and nothing it made up. -func TestAddPrintsWhatTheServerReports(t *testing.T) { - c := newFakeClient(t, - pendingProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), - unverifiedDomain(testApex, "_datum."+testApex, "datum-verify=8f3a91c2b7"), - ) - - var out bytes.Buffer - if err := waitForHostname(cancelled(), &out, c, testWorkload, testCustom); err != nil { - t.Fatalf("waitForHostname: %v", err) - } - got := out.String() - - if !strings.Contains(got, "Verifying "+testApex+"...") { - t.Errorf("should name the domain being verified:\n%s", got) - } - if !strings.Contains(got, "Add these DNS records:") { - t.Errorf("missing the records block:\n%s", got) - } - if f := row(t, got, "TYPE"); strings.Join(f, " ") != "TYPE NAME VALUE" { - t.Errorf("records header = %v", f) - } - if f := row(t, got, "TXT"); strings.Join(f[1:], " ") != "_datum."+testApex+" datum-verify=8f3a91c2b7" { - t.Errorf("TXT row = %v, want the server's own record", f) - } - if f := row(t, got, "CNAME"); strings.Join(f[1:], " ") != testCustom+" "+testCanonical { - t.Errorf("CNAME row = %v, want it to point at the platform hostname", f) - } - - if !strings.Contains(got, "Ctrl-C to detach") { - t.Errorf("should say Ctrl-C is safe:\n%s", got) - } - - // Only the check that passed gets a checkmark. - if !strings.Contains(got, "Domain verified") { - t.Errorf("missing the verified checkmark:\n%s", got) - } - if strings.Contains(got, "Certificate issued") { - t.Errorf("a pending certificate must not be reported as issued:\n%s", got) - } - // And the URL is not offered before it works. - if strings.Contains(got, "\n https://"+testCustom) { - t.Errorf("a hostname that is not live should not be printed as a URL:\n%s", got) - } -} - -func TestAddDetachOnInterruptIsNotAnError(t *testing.T) { - c := newFakeClient(t, - pendingProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), - ) - - var out bytes.Buffer - if err := waitForHostname(cancelled(), &out, c, testWorkload, testCustom); err != nil { - t.Fatalf("detaching returned an error: %v", err) - } - got := out.String() - - if !strings.Contains(got, "Detached.") { - t.Errorf("missing the detach note:\n%s", got) - } - if !strings.Contains(got, "datumctl compute domains") { - t.Errorf("detaching should point at the command that shows the state:\n%s", got) - } -} - -func TestRecordsFor(t *testing.T) { - verified := unverifiedDomain(testApex, "_datum."+testApex, "token") - verified.Status.Conditions = []metav1.Condition{ - cond(networkingv1alpha.DomainConditionVerified, metav1.ConditionTrue, "Verified", ""), - } - - programmedHostname := &url.Hostname{ - Hostname: testCustom, - Conditions: []metav1.Condition{ - cond(networkingv1alpha.HostnameConditionDNSRecordProgrammed, metav1.ConditionTrue, "RecordCreated", ""), - }, - } - - tests := []struct { - name string - info *url.Info - hostname *url.Hostname - domain *networkingv1alpha.Domain - want []string - }{ - { - name: "both records outstanding", - info: &url.Info{CanonicalHostname: testCanonical}, - hostname: &url.Hostname{Hostname: testCustom}, - domain: unverifiedDomain(testApex, "_datum."+testApex, "token"), - want: []string{"TXT _datum." + testApex + " token", "CNAME " + testCustom + " " + testCanonical}, - }, - { - name: "an already verified domain needs no TXT", - info: &url.Info{CanonicalHostname: testCanonical}, - hostname: &url.Hostname{Hostname: testCustom}, - domain: verified, - want: []string{"CNAME " + testCustom + " " + testCanonical}, - }, - { - name: "a record the platform programmed itself needs no CNAME", - info: &url.Info{CanonicalHostname: testCanonical}, - hostname: programmedHostname, - domain: verified, - want: nil, - }, - { - name: "no Domain resource yet", - info: &url.Info{CanonicalHostname: testCanonical}, - hostname: &url.Hostname{Hostname: testCustom}, - domain: nil, - want: []string{"CNAME " + testCustom + " " + testCanonical}, - }, - { - name: "no platform hostname assigned yet, so nothing to point at", - info: &url.Info{}, - hostname: &url.Hostname{Hostname: testCustom}, - domain: nil, - want: nil, - }, - { - name: "a Domain with no verification record reported", - info: &url.Info{}, - hostname: &url.Hostname{Hostname: testCustom}, - domain: &networkingv1alpha.Domain{ - Spec: networkingv1alpha.DomainSpec{DomainName: testApex}, - }, - want: nil, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var got []string - for _, r := range recordsFor(tc.info, tc.hostname, tc.domain) { - got = append(got, strings.Join([]string{r.Type, r.Name, r.Value}, " ")) - } - if strings.Join(got, "|") != strings.Join(tc.want, "|") { - t.Errorf("records = %v, want %v", got, tc.want) - } - }) - } -} - -func TestDomainForPicksTheMostSpecific(t *testing.T) { - c := newFakeClient(t, - unverifiedDomain(testApex, "_datum."+testApex, "apex"), - unverifiedDomain(testCustom, "_datum."+testCustom, "sub"), - unverifiedDomain("other.com", "_datum.other.com", "other"), - ) - - got := domainFor(context.Background(), c, "https://API.Example.com./") - if got == nil || got.Spec.DomainName != testCustom { - t.Fatalf("domainFor = %v, want %s", got, testCustom) - } - - if got := domainFor(context.Background(), c, "www."+testApex); got == nil || got.Spec.DomainName != testApex { - t.Errorf("domainFor(www.%s) = %v, want the apex Domain", testApex, got) - } - if got := domainFor(context.Background(), c, "nothing.test"); got != nil { - t.Errorf("domainFor(nothing.test) = %v, want nil", got) - } -} - -func TestCovers(t *testing.T) { - tests := []struct { - domain, hostname string - want bool - }{ - {testApex, testApex, true}, - {testApex, testCustom, true}, - {testApex, "foo." + testCustom, true}, - {testApex, "test-example.com", false}, - {testApex, "notexample.com", false}, - {testCustom, testApex, false}, - } - - for _, tc := range tests { - t.Run(tc.domain+"/"+tc.hostname, func(t *testing.T) { - if got := covers(tc.domain, tc.hostname); got != tc.want { - t.Errorf("covers(%q, %q) = %v, want %v", tc.domain, tc.hostname, got, tc.want) - } - }) - } -} - -func TestHostnameLive(t *testing.T) { - tests := []struct { - name string - h *url.Hostname - want bool - }{ - {"nil", nil, false}, - {"not serving", &url.Hostname{Active: false}, false}, - { - name: "serving, certificate issued", - h: &url.Hostname{Active: true, Conditions: []metav1.Condition{ - cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionTrue, "CertificateIssued", ""), - }}, - want: true, - }, - { - name: "serving, certificate still coming", - h: &url.Hostname{Active: true, Conditions: []metav1.Condition{ - cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionFalse, "Pending", ""), - }}, - want: false, - }, - { - name: "serving, and the platform said nothing about a certificate", - h: &url.Hostname{Active: true}, - want: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := hostnameLive(tc.h); got != tc.want { - t.Errorf("hostnameLive = %v, want %v", got, tc.want) - } - }) - } -} - -// ----------------------------------------------------------------------- -// what gets announced, and when -// ----------------------------------------------------------------------- - -// TestAddPrintsNothingBeforeThePlatformHasAnythingToSay: the first poll runs -// moments after the attach, before the platform has created the domain behind -// the hostname or computed the record that proves the user owns it. Announcing -// then prints a records block with the verification record missing from it — -// and the block is printed once, so that record would never reach the user. -func TestAddPrintsNothingBeforeThePlatformHasAnythingToSay(t *testing.T) { - // No Domain object: the platform has not got to this hostname yet. - c := newFakeClient(t, - pendingProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), - ) - - var out bytes.Buffer - if err := waitForHostname(cancelled(), &out, c, testWorkload, testCustom); err != nil { - t.Fatalf("waitForHostname: %v", err) - } - got := out.String() - - for _, unwanted := range []string{"Verifying", "Add these DNS records", "Waiting for DNS", "Domain verified"} { - if strings.Contains(got, unwanted) { - t.Errorf("nothing has been published yet, so nothing should be announced — found %q in:\n%s", unwanted, got) - } - } - if !strings.Contains(got, "Detached.") { - t.Errorf("missing the detach note:\n%s", got) - } -} - -// TestAddPrintsTheVerificationRecordWhenItAppears: the TXT record is the whole -// custom-domain flow. It shows up a poll or two after the attach, and it has to -// be printed when it does — being told to wait for DNS records that were never -// shown is the failure this guards. -func TestAddPrintsTheVerificationRecordWhenItAppears(t *testing.T) { - info := &url.Info{WorkloadName: testWorkload, CanonicalHostname: testCanonical} - h := &url.Hostname{Hostname: testCustom, URL: "https://" + testCustom} - - var out bytes.Buffer - w := &hostnameWatch{out: &out, hostname: testCustom, started: time.Now(), seen: map[string]bool{}} - - // First poll: the platform has not created the Domain yet. - announced, err := w.announce(info, h, nil) - if err != nil { - t.Fatalf("announce: %v", err) - } - if announced { - t.Errorf("announced before the platform had published anything:\n%s", out.String()) - } - if out.Len() != 0 { - t.Errorf("nothing to announce yet, but printed:\n%s", out.String()) - } - - // The platform catches up, and asks for the record it needs. - domain := unverifiedDomain(testApex, "_datum."+testApex, "datum-verify=8f3a91c2b7") - announced, err = w.announce(info, h, domain) - if err != nil { - t.Fatalf("announce: %v", err) - } - if !announced { - t.Fatalf("the records are published — they must be announced:\n%s", out.String()) - } - got := out.String() - - if !strings.Contains(got, "Verifying "+testApex+"...") { - t.Errorf("should name the domain being verified:\n%s", got) - } - if f := row(t, got, "TXT"); strings.Join(f[1:], " ") != "_datum."+testApex+" datum-verify=8f3a91c2b7" { - t.Errorf("TXT row = %v, want the record the platform asked for", f) - } - if f := row(t, got, "CNAME"); strings.Join(f[1:], " ") != testCustom+" "+testCanonical { - t.Errorf("CNAME row = %v", f) - } - if !strings.Contains(got, "Ctrl-C to detach") { - t.Errorf("should say Ctrl-C is safe:\n%s", got) - } - - // And nothing is said twice on the polls that follow. - before := out.Len() - if _, err := w.announce(info, h, domain); err != nil { - t.Fatalf("announce: %v", err) - } - if out.Len() != before { - t.Errorf("announced twice:\n%s", out.String()[before:]) - } -} - -// TestAddSaysWhenNoVerificationRecordArrives: waiting quietly is only defensible -// while the platform is still working. Past that, whatever it did publish is -// printed along with the fact that the rest has not been. -func TestAddSaysWhenNoVerificationRecordArrives(t *testing.T) { - info := &url.Info{WorkloadName: testWorkload, CanonicalHostname: testCanonical} - h := &url.Hostname{Hostname: testCustom, URL: "https://" + testCustom} - - var out bytes.Buffer - w := &hostnameWatch{ - out: &out, - hostname: testCustom, - started: time.Now().Add(-2 * recordWindow), - seen: map[string]bool{}, - } - - announced, err := w.announce(info, h, nil) - if err != nil { - t.Fatalf("announce: %v", err) - } - if !announced { - t.Fatalf("there is a record to print, so it must be printed:\n%s", out.String()) - } - got := out.String() - - if f := row(t, got, "CNAME"); strings.Join(f[1:], " ") != testCustom+" "+testCanonical { - t.Errorf("CNAME row = %v, want what the platform did publish", f) - } - if !strings.Contains(got, "has not published a verification record") { - t.Errorf("a missing verification record must be said out loud:\n%s", got) - } - - // And when it finally turns up, it is still printed. - domain := unverifiedDomain(testApex, "_datum."+testApex, "late") - if _, err := w.announce(info, h, domain); err != nil { - t.Fatalf("announce: %v", err) - } - if f := row(t, out.String(), "TXT"); strings.Join(f[1:], " ") != "_datum."+testApex+" late" { - t.Errorf("TXT row = %v, want the record printed once it exists", f) - } -} - -// TestAddStopsWhenThePlatformPublishesNothing: a wait with nothing to wait for -// and nothing to show is a hang. It ends, and says why. -func TestAddStopsWhenThePlatformPublishesNothing(t *testing.T) { - // No managed hostname to point at, and no Domain: there is no record this - // command could print, so there is nothing for the user to do. - info := &url.Info{WorkloadName: testWorkload} - h := &url.Hostname{Hostname: testCustom, URL: "https://" + testCustom} - - var out bytes.Buffer - w := &hostnameWatch{ - out: &out, - hostname: testCustom, - started: time.Now().Add(-2 * recordWindow), - seen: map[string]bool{}, - } - - announced, err := w.announce(info, h, nil) - if err == nil { - t.Fatalf("a wait with nothing to wait for must end, got output:\n%s", out.String()) - } - if announced { - t.Errorf("nothing was published, so nothing was announced") - } - for _, want := range []string{testCustom, testWorkload, "datumctl compute domains"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("error = %q, want it to mention %q", err, want) - } - } -} - -// TestAddWaitFailsWhenNothingIsEverPublished: and the failure reaches the -// command, rather than being swallowed by the poll loop. -func TestAddWaitFailsWhenNothingIsEverPublished(t *testing.T) { - proxy := programmedProxy(testWorkload, "", testCustom) - c := newFakeClient(t, proxy, serviceWith(testWorkload, 8080, location("DFW", 1, 1, true))) - - w := &hostnameWatch{ - out: io.Discard, - hostname: testCustom, - started: time.Now().Add(-2 * recordWindow), - seen: map[string]bool{}, - } - done, err := w.check(context.Background(), c, testWorkload) - if err == nil { - t.Fatal("the poll loop must surface a wait that can never finish") - } - if done { - t.Error("the hostname is not live") - } -} - -// ----------------------------------------------------------------------- -// how many custom hostnames a workload may have -// ----------------------------------------------------------------------- - -// TestAddAllowsSeveralCustomHostnames: principle 6 is one workload, one URL — -// one routing target, not a cap on names. Every comparable CLI appends, the -// Datum-managed hostname already sits alongside whatever the user attached, and -// replacing a production hostname because someone added a staging alias would -// be a surprise no platform in the table commits. So `add` appends, and says -// what the workload serves now. -func TestAddAllowsSeveralCustomHostnames(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - if err := runAdd(cancelled(), &out, c, testProject, testWorkload, "www."+testApex); err != nil { - t.Fatalf("runAdd: %v", err) - } - got := out.String() - - if want := testCustom + ",www." + testApex; strings.Join(hostnamesOf(t, c), ",") != want { - t.Fatalf("hostnames = %v, want %q — the first one is not replaced", hostnamesOf(t, c), want) - } - if !strings.Contains(got, "www."+testApex+" attached to workload") { - t.Errorf("attaching must say it attached:\n%s", got) - } - if !strings.Contains(got, "2 custom hostnames") { - t.Errorf("a workload serving more than one hostname must say so:\n%s", got) - } -} - -// TestAddHelpAgreesThatSeveralHostnamesAreAllowed: the behavior above is only -// coherent if the help says it. A user who expects `add` to replace has to be -// told otherwise before they run it, not after. -func TestAddHelpAgreesThatSeveralHostnamesAreAllowed(t *testing.T) { - long := addCommand().Long - - for _, want := range []string{"several custom hostnames", "never replaces", "domains remove"} { - if !strings.Contains(long, want) { - t.Errorf("help must mention %q:\n%s", want, long) - } - } -} diff --git a/internal/cmd/compute/domains/domains.go b/internal/cmd/compute/domains/domains.go deleted file mode 100644 index 4bb79094..00000000 --- a/internal/cmd/compute/domains/domains.go +++ /dev/null @@ -1,255 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -// Package domains implements `datumctl compute domains`: the project-wide list -// of every hostname a workload answers on, the per-URL detail view a developer -// reaches for when a URL is not working, and the two commands that attach and -// detach a custom hostname. -// -// Everything here reads and writes through the url package, which owns how a -// workload becomes a URL. Nothing here names the machinery: the vocabulary is -// "domain", "certificate", "backends". -package domains - -import ( - "context" - "fmt" - "io" - "sort" - - "github.com/spf13/cobra" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - computev1alpha "go.datum.net/compute/api/v1alpha" - "go.datum.net/compute/internal/cmd/compute/url" - "go.datum.net/compute/internal/cmd/compute/util" -) - -// managedMarker flags the hostname the platform assigned. It is the one row in -// the list a user cannot act on, so it says so on the row rather than only in -// the error they would get if they tried. -const managedMarker = "(Datum-managed)" - -// dash is what an unreported value renders as, so a column is never blank. -const dash = "—" - -// Command returns the "domains" command group. -func Command() *cobra.Command { - var output string - var noHeaders bool - - cmd := &cobra.Command{ - Use: "domains [workload]", - Short: "List the domains in a project, or show one workload's URL in detail", - Long: "List every domain in the project, or pass a workload to see its URL in detail.\n\n" + - "The detail view is the one to reach for when a URL is not answering: it shows\n" + - "every hostname the workload serves, its certificate, and how many backends are\n" + - "healthy in each city.\n\n" + - "Every workload that declares an HTTP port gets a permanent Datum-managed domain.\n" + - "It always appears in this list and cannot be removed.", - Example: " # Every domain in the project\n" + - " datumctl compute domains\n\n" + - " # One workload's URL, in detail\n" + - " datumctl compute domains api\n\n" + - " # Attach a custom hostname\n" + - " datumctl compute domains add api api.example.com\n\n" + - " # Machine-readable output\n" + - " datumctl compute domains -o json", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - c, err := util.NewClient(util.ProjectFromCmd(cmd)) - if err != nil { - return err - } - format := util.OutputFormat(output) - if len(args) == 1 { - return runDetail(cmd.Context(), cmd.OutOrStdout(), c, util.ProjectFromCmd(cmd), args[0], format) - } - return runList(cmd.Context(), cmd.OutOrStdout(), c, util.ProjectFromCmd(cmd), format, noHeaders) - }, - ValidArgsFunction: util.CompleteWorkloadNames, - } - - cmd.Flags().StringVarP(&output, "output", "o", "table", "Output format: table, json, yaml") - cmd.Flags().BoolVar(&noHeaders, "no-headers", false, "Omit the table header row (table only)") - _ = cmd.RegisterFlagCompletionFunc("output", util.CompleteOutputFormats("table", "json", "yaml")) - - cmd.AddCommand(addCommand(), removeCommand()) - - return cmd -} - -// ----------------------------------------------------------------------- -// domains — the project-wide list -// ----------------------------------------------------------------------- - -// listRow is one hostname in the project-wide list. It is also the JSON and -// YAML shape, so `-o json | jq -r '.[].domain'` is a list of hostnames and -// nothing more needs to be parsed out of a table. -type listRow struct { - // Workload is the workload serving this domain. - Workload string `json:"workload"` - // Domain is the bare hostname; URL is the same thing as an https:// URL. - Domain string `json:"domain"` - URL string `json:"url"` - // Managed is true for the permanent hostname the platform assigned, which - // cannot be removed. - Managed bool `json:"managed"` - // Status is "active", or whatever the server says is blocking it. - Status string `json:"status"` - // Certificate is "valid", or the server's reason, or empty when the - // platform has not reported on one. - Certificate string `json:"certificate,omitempty"` - // Detail is the server's message for the first blocking condition. - Detail string `json:"detail,omitempty"` -} - -// runList renders every domain in the project. It costs two List calls in -// total, however many workloads there are, because url.ForAll does. -func runList(ctx context.Context, out io.Writer, c client.Client, project string, format util.OutputFormat, noHeaders bool) error { - if ctx == nil { - ctx = context.Background() - } - - infos, err := url.ForAll(ctx, c) - if err != nil { - return err - } - rows := listRows(infos) - - switch format { - case util.OutputJSON: - return util.PrintJSON(out, rows) - case util.OutputYAML: - return util.PrintYAML(out, rows) - } - - renderList(out, rows, project, noHeaders) - return nil -} - -// listRows flattens the per-workload URL info into one row per hostname, -// ordered by workload so the list is stable between runs. Within a workload -// the custom hostnames come first and the managed one last, which is the order -// url.Info already carries: the name the user chose is the one they are -// looking for. -func listRows(infos map[string]*url.Info) []listRow { - names := make([]string, 0, len(infos)) - for name := range infos { - names = append(names, name) - } - sort.Strings(names) - - var rows []listRow - for _, name := range names { - for _, h := range infos[name].Hostnames { - rows = append(rows, listRow{ - Workload: name, - Domain: h.Hostname, - URL: h.URL, - Managed: h.Managed, - Status: h.Status, - Certificate: h.Certificate, - Detail: h.Detail, - }) - } - } - return rows -} - -// renderList writes the table. A project with no domains gets a sentence that -// says how to get one rather than an empty table. -func renderList(out io.Writer, rows []listRow, project string, noHeaders bool) { - fmt.Fprintln(out) - - if len(rows) == 0 { - fmt.Fprintf(out, "No domains in project %s.\n\n", project) - fmt.Fprintln(out, " A workload gets a permanent URL when it declares an HTTP port:") - fmt.Fprintln(out, " datumctl compute deploy --image= --http-port 8080") - return - } - - tw := util.NewTabWriter(out) - if !noHeaders { - fmt.Fprintln(tw, " WORKLOAD\tDOMAIN\tSTATUS\tCERTIFICATE") - } - for _, r := range rows { - certificate := r.Certificate - if certificate == "" { - certificate = dash - } - // The marker rides on the end of the last column rather than forming a - // column of its own: it is an annotation on the row, and a column of - // mostly-blank cells would pad every other row out to its width. - if r.Managed { - certificate += " " + managedMarker - } - fmt.Fprintf(tw, " %s\t%s\t%s\t%s\n", r.Workload, r.Domain, r.Status, certificate) - } - _ = tw.Flush() -} - -// ----------------------------------------------------------------------- -// domains — the per-URL detail view -// ----------------------------------------------------------------------- - -// runDetail renders one workload's URL: its hostnames, its certificate, and -// the per-city backend table that makes a multi-city deployment legible. -func runDetail(ctx context.Context, out io.Writer, c client.Client, project, workloadName string, format util.OutputFormat) error { - if ctx == nil { - ctx = context.Background() - } - - info, err := url.ForWorkload(ctx, c, workloadName) - if err != nil { - return err - } - if info == nil { - return notPublished(ctx, c, project, workloadName) - } - - switch format { - case util.OutputJSON: - // JSON is the scripting view: the summary a caller wants to read a URL - // or a health count out of. - return util.PrintJSON(out, info) - case util.OutputYAML: - // YAML is the escape hatch the spec promises — the real platform - // objects, for when the summary is not enough to debug with. - return util.PrintYAML(out, info.Objects()) - } - - fmt.Fprintln(out) - url.RenderDetail(out, info) - return nil -} - -// notPublished explains that a workload has no URL. A workload that does not -// exist and a workload that exists but never declared an HTTP port are -// different problems with different fixes, so they get different errors. -func notPublished(ctx context.Context, c client.Client, project, workloadName string) error { - var workload computev1alpha.Workload - err := c.Get(ctx, types.NamespacedName{Namespace: util.ResourceNamespace, Name: workloadName}, &workload) - if k8serrors.IsNotFound(err) { - return fmt.Errorf("workload %q not found in project %s — run 'datumctl compute workloads' to see what is deployed", workloadName, project) - } - return fmt.Errorf( - "workload %q has no URL — it does not declare an HTTP port.\nTo publish it: datumctl compute deploy %s --http-port ", - workloadName, workloadName, - ) -} - -// urlFor loads a workload's URL for a command that is about to change it, -// turning "no URL" into the error that says how to get one. Both add and -// remove need exactly this. -func urlFor(ctx context.Context, c client.Client, project, workloadName string) (*url.Info, error) { - info, err := url.ForWorkload(ctx, c, workloadName) - if err != nil { - return nil, err - } - if info == nil || info.Proxy == nil { - return nil, notPublished(ctx, c, project, workloadName) - } - return info, nil -} diff --git a/internal/cmd/compute/domains/domains_test.go b/internal/cmd/compute/domains/domains_test.go deleted file mode 100644 index e1a6fbd5..00000000 --- a/internal/cmd/compute/domains/domains_test.go +++ /dev/null @@ -1,449 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "bytes" - "context" - "strings" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - - computev1alpha "go.datum.net/compute/api/v1alpha" - "go.datum.net/compute/internal/cmd/compute/url" - "go.datum.net/compute/internal/cmd/compute/util" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" -) - -// Fixtures shared by the tests in this package. -const ( - testProject = "acme-prod" - testWorkload = "api" - testPortName = "http" - - testCanonical = "a1b2c3d4.datumproxy.net" - testApex = "example.com" - testCustom = "api." + testApex - - // wantPublishHint is what an unpublished workload's error has to point at: - // the flag that gives it a URL in the first place. - wantPublishHint = "--http-port" - - // wantNoHostname is the error for a command called without a hostname. - wantNoHostname = "no hostname given" -) - -func testScheme(t *testing.T) *runtime.Scheme { - t.Helper() - s := runtime.NewScheme() - if err := computev1alpha.AddToScheme(s); err != nil { - t.Fatalf("registering compute scheme: %v", err) - } - if err := networkingv1alpha.AddToScheme(s); err != nil { - t.Fatalf("registering networking scheme: %v", err) - } - return s -} - -func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch { - t.Helper() - return fake.NewClientBuilder(). - WithScheme(testScheme(t)). - WithStatusSubresource(&networkingv1alpha.HTTPProxy{}, &networkingv1alpha.NetworkService{}, &networkingv1alpha.Domain{}). - WithObjects(objs...). - Build() -} - -func workload(name string) *computev1alpha.Workload { - return &computev1alpha.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: util.ResourceNamespace, - UID: types.UID("uid-" + name), - }, - } -} - -func cond(condType string, status metav1.ConditionStatus, reason, message string) metav1.Condition { - return metav1.Condition{Type: condType, Status: status, Reason: reason, Message: message} -} - -// programmedProxy returns a proxy in the shape the platform reports once it is -// serving: accepted, programmed, and holding a certificate. -func programmedProxy(name, canonical string, hostnames ...string) *networkingv1alpha.HTTPProxy { - p := url.BuildHTTPProxy(workload(name), testPortName, hostnames) - p.Status.CanonicalHostname = canonical - p.Status.Conditions = []metav1.Condition{ - cond(networkingv1alpha.HTTPProxyConditionAccepted, metav1.ConditionTrue, "Accepted", ""), - cond(networkingv1alpha.HTTPProxyConditionProgrammed, metav1.ConditionTrue, "Programmed", ""), - cond(networkingv1alpha.HTTPProxyConditionCertificatesReady, metav1.ConditionTrue, "AllCertificatesReady", ""), - } - return p -} - -// verifiedProxy returns a proxy whose custom hostname the platform has already -// checked and is serving: verified, its DNS record programmed, its certificate -// issued. A hostname is only ever active on the strength of its own entry -// here — the proxy-level conditions are a roll-up over every hostname and -// cannot vouch for any one of them. -func verifiedProxy(name, canonical, hostname string) *networkingv1alpha.HTTPProxy { - p := programmedProxy(name, canonical, hostname) - p.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ - Hostname: hostname, - Conditions: []metav1.Condition{ - cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionTrue, "Verified", ""), - cond(networkingv1alpha.HostnameConditionAvailable, metav1.ConditionTrue, "Claimed", ""), - cond(networkingv1alpha.HostnameConditionDNSRecordProgrammed, metav1.ConditionTrue, "RecordCreated", ""), - cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionTrue, "CertificateIssued", ""), - }, - }} - return p -} - -// pendingProxy returns a proxy whose custom hostname is still waiting on DNS. -func pendingProxy(name, canonical, hostname string) *networkingv1alpha.HTTPProxy { - p := programmedProxy(name, canonical, hostname) - p.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ - Hostname: hostname, - Conditions: []metav1.Condition{ - cond(networkingv1alpha.HostnameConditionVerified, metav1.ConditionTrue, "Verified", ""), - cond(networkingv1alpha.HostnameConditionDNSRecordProgrammed, metav1.ConditionFalse, "Pending", "waiting for DNS"), - cond(networkingv1alpha.HostnameConditionCertificateReady, metav1.ConditionFalse, "ChallengeInProgress", "ACME challenge in progress"), - }, - }} - return p -} - -func serviceWith(name string, port int32, locations ...networkingv1alpha.NetworkServiceLocationStatus) *networkingv1alpha.NetworkService { - s := url.BuildNetworkService(workload(name), testPortName, port) - var members, healthy int32 - for _, l := range locations { - members += l.Members - healthy += l.Healthy - } - s.Status.Summary = networkingv1alpha.NetworkServiceSummary{ - Locations: int32(len(locations)), - Members: members, - Healthy: healthy, - } - s.Status.Locations = locations - s.Status.Conditions = []metav1.Condition{ - cond(networkingv1alpha.NetworkServiceMembersResolved, metav1.ConditionTrue, "MembersResolved", ""), - cond(networkingv1alpha.NetworkServiceReady, metav1.ConditionTrue, "Ready", ""), - } - return s -} - -func location(city string, members, healthy int32, serving bool) networkingv1alpha.NetworkServiceLocationStatus { - return networkingv1alpha.NetworkServiceLocationStatus{Name: city, Members: members, Healthy: healthy, Serving: serving} -} - -// row returns the whitespace-separated fields of the first line whose fields -// begin with the given ones, so assertions never depend on column widths. -func row(t *testing.T, output string, first ...string) []string { - t.Helper() - for _, line := range strings.Split(output, "\n") { - fields := strings.Fields(line) - if len(fields) < len(first) { - continue - } - match := true - for i, want := range first { - if fields[i] != want { - match = false - break - } - } - if match { - return fields - } - } - t.Fatalf("no line starting with %v in:\n%s", first, output) - return nil -} - -// ----------------------------------------------------------------------- -// the list view -// ----------------------------------------------------------------------- - -func TestListRendersEveryHostname(t *testing.T) { - c := newFakeClient(t, - verifiedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), - pendingProxy("web", "e5f6a7b8.datumproxy.net", "www.acme.io"), - serviceWith("web", 3000, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - if err := runList(context.Background(), &out, c, testProject, util.OutputTable, false); err != nil { - t.Fatalf("runList: %v", err) - } - got := out.String() - - if f := row(t, got, "WORKLOAD"); strings.Join(f, " ") != "WORKLOAD DOMAIN STATUS CERTIFICATE" { - t.Errorf("header = %v", f) - } - - // The custom hostname comes before the managed one for the same workload: - // the name the user chose is the one they are looking for. - custom := row(t, got, testWorkload, testCustom) - if strings.Join(custom[2:], " ") != "active valid" { - t.Errorf("custom row = %v, want active valid", custom) - } - if strings.Contains(strings.Join(custom, " "), managedMarker) { - t.Errorf("custom hostname marked as Datum-managed: %v", custom) - } - - managedRow := row(t, got, testWorkload, testCanonical) - if strings.Join(managedRow[2:], " ") != "active valid "+managedMarker { - t.Errorf("managed row = %v, want the %s marker", managedRow, managedMarker) - } - - // The server's own words for a hostname that is not serving yet: its - // messages, never the camelCase reasons behind them. - pending := strings.Join(row(t, got, "web", "www.acme.io")[2:], " ") - if pending != "waiting for DNS ACME challenge in progress" { - t.Errorf("pending row = %q, want the server's messages verbatim", pending) - } - if strings.Contains(got, "ChallengeInProgress") { - t.Errorf("a raw condition reason reached the table:\n%s", got) - } - - // Workloads are ordered, so two runs read the same. - if strings.Index(got, "\n api ") > strings.Index(got, "\n web ") { - t.Errorf("workloads out of order:\n%s", got) - } -} - -func TestListManagedHostnameAlwaysAppears(t *testing.T) { - // A workload with no custom hostname still has a domain: the managed one. - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - if err := runList(context.Background(), &out, c, testProject, util.OutputTable, false); err != nil { - t.Fatalf("runList: %v", err) - } - - f := row(t, out.String(), testWorkload, testCanonical) - if f[len(f)-1] != managedMarker { - t.Errorf("row = %v, want it marked %s", f, managedMarker) - } -} - -func TestListEmpty(t *testing.T) { - c := newFakeClient(t) - - var out bytes.Buffer - if err := runList(context.Background(), &out, c, testProject, util.OutputTable, false); err != nil { - t.Fatalf("runList: %v", err) - } - got := out.String() - - if !strings.Contains(got, "No domains in project "+testProject) { - t.Errorf("missing the empty-state sentence:\n%s", got) - } - if !strings.Contains(got, "--http-port") { - t.Errorf("empty state should say how to get a domain:\n%s", got) - } - if strings.Contains(got, "WORKLOAD") { - t.Errorf("empty state should not print a header row:\n%s", got) - } -} - -func TestListNoHeaders(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - if err := runList(context.Background(), &out, c, testProject, util.OutputTable, true); err != nil { - t.Fatalf("runList: %v", err) - } - if strings.Contains(out.String(), "WORKLOAD") { - t.Errorf("--no-headers still printed a header:\n%s", out.String()) - } -} - -func TestListJSONIsScriptable(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true)), - ) - - var out bytes.Buffer - if err := runList(context.Background(), &out, c, testProject, util.OutputJSON, false); err != nil { - t.Fatalf("runList: %v", err) - } - got := out.String() - - for _, want := range []string{`"domain": "` + testCustom + `"`, `"url": "https://` + testCustom + `"`, `"managed": true`} { - if !strings.Contains(got, want) { - t.Errorf("JSON missing %s:\n%s", want, got) - } - } -} - -func TestListRowsOrderCustomBeforeManaged(t *testing.T) { - rows := listRows(map[string]*url.Info{ - "web": {Hostnames: []url.Hostname{{Hostname: "www.acme.io"}}}, - "api": {Hostnames: []url.Hostname{{Hostname: testCustom}, {Hostname: testCanonical, Managed: true}}}, - }) - - got := make([]string, 0, len(rows)) - for _, r := range rows { - got = append(got, r.Workload+"/"+r.Domain) - } - want := []string{"api/" + testCustom, "api/" + testCanonical, "web/www.acme.io"} - if strings.Join(got, ",") != strings.Join(want, ",") { - t.Errorf("rows = %v, want %v", got, want) - } -} - -// ----------------------------------------------------------------------- -// the detail view -// ----------------------------------------------------------------------- - -func TestDetailDegradedAcrossCities(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true), location("IAD", 2, 0, false)), - ) - - var out bytes.Buffer - if err := runDetail(context.Background(), &out, c, testProject, testWorkload, util.OutputTable); err != nil { - t.Fatalf("runDetail: %v", err) - } - got := out.String() - - if !strings.Contains(got, "https://"+testCustom) || !strings.Contains(got, "https://"+testCanonical) { - t.Errorf("detail view should list both hostnames:\n%s", got) - } - if f := row(t, got, "Backend"); strings.Join(f[1:], " ") != "port 8080/tcp" { - t.Errorf("backend row = %v", f) - } - if !strings.Contains(got, "Degraded — 2 of 4 backends healthy") { - t.Errorf("missing the health summary:\n%s", got) - } - - if f := row(t, got, "CITY"); strings.Join(f, " ") != "CITY BACKENDS HEALTHY SERVING" { - t.Errorf("per-city header = %v", f) - } - if f := row(t, got, "DFW"); strings.Join(f[1:], " ") != "2 2 yes" { - t.Errorf("DFW row = %v", f) - } - if f := row(t, got, "IAD"); strings.Join(f[1:], " ") != "2 0 no" { - t.Errorf("IAD row = %v", f) - } - - if !strings.Contains(got, "IAD: no healthy backends") { - t.Errorf("missing the degraded narrative:\n%s", got) - } - if !strings.Contains(got, "Next steps:") || - !strings.Contains(got, "datumctl compute instances --workload=api --city=IAD") { - t.Errorf("missing the next-steps block:\n%s", got) - } -} - -func TestDetailHealthyHasNoDiagnosis(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical), - serviceWith(testWorkload, 8080, location("DFW", 2, 2, true), location("IAD", 2, 2, true)), - ) - - var out bytes.Buffer - if err := runDetail(context.Background(), &out, c, testProject, testWorkload, util.OutputTable); err != nil { - t.Fatalf("runDetail: %v", err) - } - got := out.String() - - if !strings.Contains(got, "Healthy — 4 of 4 backends healthy") { - t.Errorf("missing the health summary:\n%s", got) - } - if strings.Contains(got, "Next steps:") { - t.Errorf("a healthy URL should not suggest a fix:\n%s", got) - } -} - -func TestDetailJSON(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - if err := runDetail(context.Background(), &out, c, testProject, testWorkload, util.OutputJSON); err != nil { - t.Fatalf("runDetail: %v", err) - } - if !strings.Contains(out.String(), `"url": "https://`+testCanonical+`"`) { - t.Errorf("JSON detail missing the URL:\n%s", out.String()) - } -} - -func TestDetailErrors(t *testing.T) { - tests := []struct { - name string - objects []client.Object - workload string - want string - }{ - { - name: "workload exists but was never published", - objects: []client.Object{workload(testWorkload)}, - workload: testWorkload, - want: wantPublishHint, - }, - { - name: "workload does not exist", - objects: nil, - workload: "ghost", - want: "not found in project " + testProject, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - c := newFakeClient(t, tc.objects...) - var out bytes.Buffer - err := runDetail(context.Background(), &out, c, testProject, tc.workload, util.OutputTable) - if err == nil { - t.Fatalf("expected an error, got output:\n%s", out.String()) - } - if !strings.Contains(err.Error(), tc.want) { - t.Errorf("error = %q, want it to mention %q", err, tc.want) - } - }) - } -} - -// hostnamesOf reads back what a proxy declares, for mutation assertions. -func hostnamesOf(t *testing.T, c client.Client) []string { - t.Helper() - var proxy networkingv1alpha.HTTPProxy - key := types.NamespacedName{Namespace: util.ResourceNamespace, Name: testWorkload} - if err := c.Get(context.Background(), key, &proxy); err != nil { - t.Fatalf("reading the URL for workload %q: %v", testWorkload, err) - } - out := make([]string, 0, len(proxy.Spec.Hostnames)) - for _, h := range proxy.Spec.Hostnames { - out = append(out, string(h)) - } - return out -} - -func hostnameList(hostnames ...string) []gatewayv1.Hostname { - out := make([]gatewayv1.Hostname, 0, len(hostnames)) - for _, h := range hostnames { - out = append(out, gatewayv1.Hostname(h)) - } - return out -} diff --git a/internal/cmd/compute/domains/hostname.go b/internal/cmd/compute/domains/hostname.go deleted file mode 100644 index 4795c220..00000000 --- a/internal/cmd/compute/domains/hostname.go +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "fmt" - "net" - "regexp" - "strings" -) - -// maxHostnameLength is the length the API allows for a domain name. -const maxHostnameLength = 253 - -// hostnamePattern is the shape the platform stores a hostname in: lowercase -// RFC 1123 labels separated by dots. It matches the Domain API's own pattern, -// so a hostname this accepts is one the server will accept too. -var hostnamePattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`) - -// normalizeHostname reduces what a user might type or paste to the bare -// hostname the platform stores. -// -// People copy a domain out of a browser bar, out of a dig output, or out of -// this CLI's own output, so "https://API.Example.com/health", "api.example.com." -// and "api.example.com:443" all reach us meaning the same name. Comparisons — -// above all the one that refuses to remove the Datum-managed hostname — are -// made on this form, never on the raw string. -func normalizeHostname(raw string) string { - h := strings.TrimSpace(raw) - - // A pasted URL: drop the scheme, then everything after the authority. - if i := strings.Index(h, "://"); i >= 0 { - h = h[i+len("://"):] - } - if i := strings.IndexAny(h, "/?#"); i >= 0 { - h = h[:i] - } - if i := strings.LastIndex(h, "@"); i >= 0 { - h = h[i+1:] - } - // A port, but only when there is one colon to be sure it is a port: an - // IPv6 literal has several, and must survive intact to be rejected as an - // address rather than mangled into something that looks like a bad name. - if strings.HasPrefix(h, "[") { - if i := strings.Index(h, "]"); i >= 0 { - h = h[1:i] - } - } else if i := strings.LastIndex(h, ":"); i >= 0 && isDigits(h[i+1:]) && !strings.Contains(h[:i], ":") { - h = h[:i] - } - - // A trailing dot is the same name, spelled absolutely. - h = strings.TrimRight(h, ".") - - return strings.ToLower(h) -} - -// sameHostname reports whether two hostnames name the same host, whatever form -// each was written in. An empty hostname matches nothing, so an unassigned -// managed hostname never swallows a user's argument. -func sameHostname(a, b string) bool { - na, nb := normalizeHostname(a), normalizeHostname(b) - return na != "" && na == nb -} - -// validateHostname normalizes a hostname and reports why it cannot be used, in -// terms of what the user typed rather than the field the API would complain -// about. -func validateHostname(raw string) (string, error) { - h := normalizeHostname(raw) - - switch { - case h == "": - return "", fmt.Errorf("no hostname given — pass the hostname to attach, for example api.example.com") - case strings.Contains(h, "*"): - return "", fmt.Errorf("wildcard hostnames are not supported — attach each hostname you want to serve") - case len(h) > maxHostnameLength: - return "", fmt.Errorf("hostname %q is longer than %d characters", h, maxHostnameLength) - case net.ParseIP(h) != nil: - return "", fmt.Errorf("%q is an IP address — a domain must be a hostname, for example api.example.com", h) - case !strings.Contains(h, "."): - return "", fmt.Errorf("%q is not a fully qualified hostname — use something like api.example.com", h) - case !hostnamePattern.MatchString(h): - return "", fmt.Errorf("%q is not a valid hostname — use letters, digits, dashes and dots", h) - } - - return h, nil -} - -func isDigits(s string) bool { - if s == "" { - return false - } - for _, r := range s { - if r < '0' || r > '9' { - return false - } - } - return true -} diff --git a/internal/cmd/compute/domains/hostname_test.go b/internal/cmd/compute/domains/hostname_test.go deleted file mode 100644 index c714718b..00000000 --- a/internal/cmd/compute/domains/hostname_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "strings" - "testing" -) - -func TestNormalizeHostname(t *testing.T) { - tests := []struct { - name string - in string - want string - }{ - {"already normal", testCustom, testCustom}, - {"upper case", "API.Example.COM", testCustom}, - {"one trailing dot", testCustom + ".", testCustom}, - {"several trailing dots", testCustom + "..", testCustom}, - {"surrounding space", " " + testCustom + " \n", testCustom}, - {"https prefix", "https://" + testCustom, testCustom}, - {"http prefix", "http://" + testCustom, testCustom}, - {"url with path", "https://" + testCustom + "/healthz", testCustom}, - {"url with query", "https://" + testCustom + "?x=1", testCustom}, - {"url with fragment", "https://" + testCustom + "#top", testCustom}, - {"url with port", "https://" + testCustom + ":443/", testCustom}, - {"bare host and port", testCustom + ":8443", testCustom}, - {"credentials", "https://user:pw@" + testCustom + "/", testCustom}, - {"everything at once", " HTTPS://API.Example.com.:443/health?x=1 ", testCustom}, - {"nothing at all", "", ""}, - {"only a dot", ".", ""}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := normalizeHostname(tc.in); got != tc.want { - t.Errorf("normalizeHostname(%q) = %q, want %q", tc.in, got, tc.want) - } - }) - } -} - -func TestSameHostname(t *testing.T) { - tests := []struct { - name string - a, b string - want bool - }{ - {"identical", testCanonical, testCanonical, true}, - {"case differs", "A1B2C3D4.DatumProxy.NET", testCanonical, true}, - {"one trailing dot", testCanonical + ".", testCanonical, true}, - {"scheme pasted", "https://" + testCanonical, testCanonical, true}, - {"scheme and slash pasted", "https://" + testCanonical + "/", testCanonical, true}, - {"different hosts", testCustom, testCanonical, false}, - {"one empty", "", testCanonical, false}, - {"both empty", "", "", false}, - {"suffix is not the same host", "evil-" + testCanonical, testCanonical, false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := sameHostname(tc.a, tc.b); got != tc.want { - t.Errorf("sameHostname(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) - } - }) - } -} - -func TestValidateHostname(t *testing.T) { - tests := []struct { - name string - in string - want string - wantErr string - }{ - {name: "plain", in: testCustom, want: testCustom}, - {name: "normalized on the way through", in: "HTTPS://API.Example.com./", want: testCustom}, - {name: "only whitespace", in: " ", wantErr: wantNoHostname}, - {name: "wildcard", in: "*.example.com", wantErr: "wildcard hostnames are not supported"}, - {name: "ipv4", in: "192.0.2.10", wantErr: "is an IP address"}, - {name: "ipv6", in: "2001:db8::1", wantErr: "is an IP address"}, - {name: "single label", in: "localhost", wantErr: "not a fully qualified hostname"}, - {name: "underscore", in: "api_v2.example.com", wantErr: "not a valid hostname"}, - {name: "leading dash label", in: "-api.example.com", wantErr: "not a valid hostname"}, - {name: "too long", in: strings.Repeat("a", 250) + ".example.com", wantErr: "longer than 253"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got, err := validateHostname(tc.in) - if tc.wantErr != "" { - if err == nil { - t.Fatalf("validateHostname(%q) = %q, want an error", tc.in, got) - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Errorf("error = %q, want it to mention %q", err, tc.wantErr) - } - return - } - if err != nil { - t.Fatalf("validateHostname(%q): %v", tc.in, err) - } - if got != tc.want { - t.Errorf("validateHostname(%q) = %q, want %q", tc.in, got, tc.want) - } - }) - } -} diff --git a/internal/cmd/compute/domains/remove.go b/internal/cmd/compute/domains/remove.go deleted file mode 100644 index 5f9349ac..00000000 --- a/internal/cmd/compute/domains/remove.go +++ /dev/null @@ -1,193 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "bufio" - "context" - "fmt" - "io" - "os" - "strings" - - "github.com/spf13/cobra" - "golang.org/x/term" - "sigs.k8s.io/controller-runtime/pkg/client" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - - "go.datum.net/compute/internal/cmd/compute/url" - "go.datum.net/compute/internal/cmd/compute/util" -) - -func removeCommand() *cobra.Command { - var yes bool - - cmd := &cobra.Command{ - Use: "remove ", - Short: "Detach a custom hostname from a workload", - Long: "Detach a custom hostname from a workload's URL.\n\n" + - "The workload keeps serving on its Datum-managed hostname, which is permanent\n" + - "and cannot be removed.\n\n" + - "The domain itself is left alone: it stays verified in this project and can be\n" + - "attached to this or another workload without verifying it again.", - Example: " # Detach a hostname\n" + - " datumctl compute domains remove api api.example.com\n\n" + - " # Without the confirmation prompt\n" + - " datumctl compute domains remove api api.example.com --yes", - Args: cobra.ExactArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - c, err := util.NewClient(util.ProjectFromCmd(cmd)) - if err != nil { - return err - } - return runRemove(cmd.Context(), cmd.OutOrStdout(), c, util.ProjectFromCmd(cmd), args[0], args[1], yes) - }, - ValidArgsFunction: completeRemoveArgs, - } - - cmd.Flags().BoolVarP(&yes, "yes", "y", false, "Skip the confirmation prompt") - - return cmd -} - -// runRemove detaches a hostname from a workload's URL. -// -// The Datum-managed hostname is refused: it is the workload's permanent -// address and the fallback a user checks when a custom domain is misbehaving. -// The check is made on the normalized form of both names, so pasting -// "https://A1B2C3D4.DatumProxy.net/" back out of a browser is refused just as -// plainly as typing the hostname exactly. -func runRemove(ctx context.Context, out io.Writer, c client.Client, project, workloadName, rawHostname string, yes bool) error { - if ctx == nil { - ctx = context.Background() - } - - hostname := normalizeHostname(rawHostname) - if hostname == "" { - return fmt.Errorf("no hostname given — pass the hostname to detach, for example api.example.com") - } - - info, err := urlFor(ctx, c, project, workloadName) - if err != nil { - return err - } - - if managed(info, hostname) { - return fmt.Errorf( - "%s is the Datum-managed domain for workload %q and cannot be removed — it is permanent, and stops serving only when the workload stops declaring an HTTP port", - hostname, workloadName, - ) - } - - index := indexOf(info.Proxy.Spec.Hostnames, hostname) - if index < 0 { - return notAttached(info, workloadName, hostname) - } - - // The stored spelling, not the user's: the confirmation and the summary - // should show what is actually on the workload. - attachedAs := string(info.Proxy.Spec.Hostnames[index]) - - ok, err := confirm(out, fmt.Sprintf( - "Remove %s from workload %q? It will stop serving on that hostname. (y/N): ", attachedAs, workloadName), yes) - if err != nil { - return err - } - if !ok { - fmt.Fprintln(out, "Aborted.") - return nil - } - - proxy := info.Proxy.DeepCopy() - proxy.Spec.Hostnames = append(proxy.Spec.Hostnames[:index], proxy.Spec.Hostnames[index+1:]...) - if err := c.Update(ctx, proxy); err != nil { - return fmt.Errorf("detaching %s from workload %q: %w", attachedAs, workloadName, err) - } - - fmt.Fprintf(out, " %s removed from workload %q.\n", attachedAs, workloadName) - if info.CanonicalHostname != "" { - fmt.Fprintf(out, " Still serving on https://%s\n", info.CanonicalHostname) - } - fmt.Fprintln(out, "\n The domain stays verified in this project. Attach it again with:") - fmt.Fprintf(out, " datumctl compute domains add %s %s\n", workloadName, attachedAs) - - return nil -} - -// managed reports whether a hostname is the one the platform assigned. Both the -// status field and the parsed hostname list are checked, so the refusal does -// not depend on which of the two the server filled in. -func managed(info *url.Info, hostname string) bool { - if sameHostname(info.CanonicalHostname, hostname) { - return true - } - for _, h := range info.Hostnames { - if h.Managed && sameHostname(h.Hostname, hostname) { - return true - } - } - return false -} - -// indexOf finds a hostname in the proxy's declared list, comparing normalized -// forms so a trailing dot or a different case still matches what is stored. -func indexOf(hostnames []gatewayv1.Hostname, hostname string) int { - for i, h := range hostnames { - if sameHostname(string(h), hostname) { - return i - } - } - return -1 -} - -// notAttached explains that there is nothing to remove, and lists what could -// be removed instead. -func notAttached(info *url.Info, workloadName, hostname string) error { - if len(info.CustomHostnames) == 0 { - return fmt.Errorf("workload %q has no custom domains — it serves on its Datum-managed domain only", workloadName) - } - return fmt.Errorf("%s is not attached to workload %q — it serves on: %s", - hostname, workloadName, strings.Join(info.CustomHostnames, ", ")) -} - -// confirm asks before doing something that stops traffic. Without a terminal -// to ask on it refuses rather than assuming: a script that meant to remove a -// domain can say so with --yes. -func confirm(out io.Writer, prompt string, yes bool) (bool, error) { - if yes { - return true, nil - } - if !term.IsTerminal(int(os.Stdin.Fd())) { - return false, fmt.Errorf("refusing to remove a domain without confirmation — re-run with --yes") - } - - fmt.Fprint(out, prompt) - line, err := bufio.NewReader(os.Stdin).ReadString('\n') - if err != nil { - return false, fmt.Errorf("reading confirmation: %w", err) - } - line = strings.TrimSpace(line) - return line == "y" || line == "Y", nil -} - -// completeRemoveArgs completes the workload first and then the hostnames that -// workload actually serves, minus the Datum-managed one, which cannot be -// removed and so should never be offered. -func completeRemoveArgs(cmd *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { - if len(args) == 0 { - return util.CompleteWorkloadNames(cmd, args, "") - } - if len(args) > 1 { - return nil, cobra.ShellCompDirectiveNoFileComp - } - - c, err := util.NewClient(util.ProjectFromCmd(cmd)) - if err != nil { - return nil, cobra.ShellCompDirectiveNoFileComp - } - info, err := url.ForWorkload(context.Background(), c, args[0]) - if err != nil || info == nil { - return nil, cobra.ShellCompDirectiveNoFileComp - } - return info.CustomHostnames, cobra.ShellCompDirectiveNoFileComp -} diff --git a/internal/cmd/compute/domains/remove_test.go b/internal/cmd/compute/domains/remove_test.go deleted file mode 100644 index 65d6c3a9..00000000 --- a/internal/cmd/compute/domains/remove_test.go +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "bytes" - "context" - "strings" - "testing" - - "sigs.k8s.io/controller-runtime/pkg/client" - - "go.datum.net/compute/internal/cmd/compute/url" -) - -// TestRemoveRefusesTheManagedDomain is the rule the spec states outright: the -// Datum-managed URL always appears in the list and cannot be removed. It has to -// hold however the user spells it, because the spelling they have to hand is -// whatever they copied out of a browser or out of this CLI's own output. -func TestRemoveRefusesTheManagedDomain(t *testing.T) { - variants := []struct { - name string - given string - }{ - {"exactly", testCanonical}, - {"upper case", "A1B2C3D4.DATUMPROXY.NET"}, - {"mixed case", "A1b2C3d4.DatumProxy.Net"}, - {"with a trailing dot", testCanonical + "."}, - {"https prefix", "https://" + testCanonical}, - {"https prefix and trailing slash", "https://" + testCanonical + "/"}, - {"https prefix and a path", "https://" + testCanonical + "/healthz"}, - {"https prefix, upper case and trailing dot", "HTTPS://A1B2C3D4.DatumProxy.NET./"}, - {"surrounding whitespace", " " + testCanonical + " "}, - {"with a port", testCanonical + ":443"}, - } - - for _, tc := range variants { - t.Run(tc.name, func(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - err := runRemove(context.Background(), &out, c, testProject, testWorkload, tc.given, true) - if err == nil { - t.Fatalf("removing the managed domain as %q was allowed", tc.given) - } - if !strings.Contains(err.Error(), "cannot be removed") { - t.Errorf("error = %q, want it to say the domain cannot be removed", err) - } - - // And it is still there. - if got := hostnamesOf(t, c); len(got) != 1 || got[0] != testCustom { - t.Errorf("hostnames = %v, want the custom one untouched", got) - } - }) - } -} - -// The managed hostname is refused even when the platform reported it only in -// the per-hostname list and not in the canonical field. -func TestRemoveRefusesManagedHostnameFromInfo(t *testing.T) { - info := &url.Info{ - Hostnames: []url.Hostname{{Hostname: testCanonical, Managed: true}}, - } - if !managed(info, "https://"+testCanonical+"/") { - t.Error("a hostname marked managed should be refused") - } - if managed(info, testCustom) { - t.Error("a custom hostname should not be treated as managed") - } -} - -func TestRemoveDetachesHostname(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical, testCustom, "www.example.com"), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - if err := runRemove(context.Background(), &out, c, testProject, testWorkload, "HTTPS://API.Example.com./", true); err != nil { - t.Fatalf("runRemove: %v", err) - } - - if got := hostnamesOf(t, c); len(got) != 1 || got[0] != "www.example.com" { - t.Errorf("hostnames = %v, want only www.example.com left", got) - } - - got := out.String() - if !strings.Contains(got, testCustom+" removed from workload") { - t.Errorf("missing the confirmation line:\n%s", got) - } - if !strings.Contains(got, "Still serving on https://"+testCanonical) { - t.Errorf("should say what the workload still serves on:\n%s", got) - } - // Per the spec, a verified domain outlives the workload it was attached to. - if !strings.Contains(got, "stays verified in this project") { - t.Errorf("should say the domain itself is kept:\n%s", got) - } - if !strings.Contains(got, "datumctl compute domains add "+testWorkload+" "+testCustom) { - t.Errorf("should say how to attach it again:\n%s", got) - } -} - -func TestRemoveErrors(t *testing.T) { - tests := []struct { - name string - objects []client.Object - workload string - hostname string - yes bool - want string - }{ - { - name: "hostname is not attached", - objects: []client.Object{programmedProxy(testWorkload, testCanonical, testCustom)}, - workload: testWorkload, - hostname: "other.example.com", - yes: true, - want: "is not attached to workload", - }, - { - name: "workload has no custom domains at all", - objects: []client.Object{programmedProxy(testWorkload, testCanonical)}, - workload: testWorkload, - hostname: testCustom, - yes: true, - want: "has no custom domains", - }, - { - name: "workload has no URL", - objects: []client.Object{workload(testWorkload)}, - workload: testWorkload, - hostname: testCustom, - yes: true, - want: wantPublishHint, - }, - { - name: "hostname argument is blank", - objects: []client.Object{programmedProxy(testWorkload, testCanonical, testCustom)}, - workload: testWorkload, - hostname: " ", - yes: true, - want: wantNoHostname, - }, - { - // go test runs without a terminal, which is the same position a CI - // job is in: say so rather than removing a domain unasked. - name: "no terminal to confirm on and no --yes", - objects: []client.Object{programmedProxy(testWorkload, testCanonical, testCustom)}, - workload: testWorkload, - hostname: testCustom, - yes: false, - want: "--yes", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - c := newFakeClient(t, tc.objects...) - var out bytes.Buffer - err := runRemove(context.Background(), &out, c, testProject, tc.workload, tc.hostname, tc.yes) - if err == nil { - t.Fatalf("expected an error, got output:\n%s", out.String()) - } - if !strings.Contains(err.Error(), tc.want) { - t.Errorf("error = %q, want it to mention %q", err, tc.want) - } - }) - } -} - -func TestRemoveLeavesTheProxyWhenRefusing(t *testing.T) { - c := newFakeClient(t, programmedProxy(testWorkload, testCanonical, testCustom)) - - var out bytes.Buffer - if err := runRemove(context.Background(), &out, c, testProject, testWorkload, "other.example.com", true); err == nil { - t.Fatal("expected an error") - } - if got := hostnamesOf(t, c); len(got) != 1 || got[0] != testCustom { - t.Errorf("hostnames = %v, want them untouched", got) - } -} - -func TestIndexOfNormalizes(t *testing.T) { - hostnames := hostnameList("www.example.com", "API.Example.com.") - - tests := []struct { - name string - given string - want int - }{ - {"exact", "www.example.com", 0}, - {"stored in another spelling", testCustom, 1}, - {"pasted as a URL", "https://api.example.com/", 1}, - {"absent", "nope.example.com", -1}, - {"nothing at all", "", -1}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := indexOf(hostnames, tc.given); got != tc.want { - t.Errorf("indexOf(%q) = %d, want %d", tc.given, got, tc.want) - } - }) - } -} diff --git a/internal/cmd/compute/domains/wait_failure_test.go b/internal/cmd/compute/domains/wait_failure_test.go deleted file mode 100644 index 3759abef..00000000 --- a/internal/cmd/compute/domains/wait_failure_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "bytes" - "context" - "errors" - "strings" - "testing" - "time" - - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" -) - -// TestAddStopsWhenTheHostnameCanNeverBeRead: `domains add` attaches the -// hostname and then waits for the platform to verify it. A control plane the -// wait cannot read is not going to start answering, so swallowing the read -// error leaves the command polling in silence forever — the user is told to -// wait for DNS and never told anything again. It has to give up and say why. -// -// Regression test: before the failure budget, this hung past the guard below. -func TestAddStopsWhenTheHostnameCanNeverBeRead(t *testing.T) { - boom := errors.New("httpproxies.networking.datumapis.com is forbidden") - - // Reads fail only after the hostname is attached, so the command gets far - // enough to start waiting — which is the state that used to hang. - attached := false - c := interceptor.NewClient(newFakeClient(t, - programmedProxy(testWorkload, testCanonical), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ), interceptor.Funcs{ - Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { - attached = true - return cl.Update(ctx, obj, opts...) - }, - List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if attached { - return boom - } - return cl.List(ctx, list, opts...) - }, - }) - - var out bytes.Buffer - done := make(chan error, 1) - go func() { - done <- runAdd(context.Background(), &out, c, testProject, testWorkload, testCustom) - }() - - select { - case err := <-done: - if err == nil { - t.Fatal("a hostname whose state cannot be read must fail the command, not wait forever") - } - if !errors.Is(err, boom) { - t.Errorf("error = %v, want the server's failure to survive wrapping", err) - } - if !strings.Contains(err.Error(), testCustom) { - t.Errorf("error = %q, want it to name the hostname", err) - } - case <-time.After(30 * time.Second): - t.Fatalf("runAdd never returned — the wait is hanging on an unreadable control plane:\n%s", out.String()) - } -} - -// TestAddRidesOutATransientReadFailure: the counterpart. One failed read inside -// the wait is nothing — the platform was just asked to check the hostname — and -// giving up there would break a command that is going fine. -// -// The hostname is already attached and verified, so runAdd reports that and goes -// straight to the wait. The first read is the one runAdd itself makes; the -// second is the wait's, and that is the one that fails. -func TestAddRidesOutATransientReadFailure(t *testing.T) { - reads := 0 - failed := false - c := interceptor.NewClient(newFakeClient(t, - verifiedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ), interceptor.Funcs{ - List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if _, ok := list.(*networkingv1alpha.HTTPProxyList); ok { - reads++ - if reads == 2 { - failed = true - return errors.New("etcdserver: request timed out") - } - } - return cl.List(ctx, list, opts...) - }, - }) - - var out bytes.Buffer - done := make(chan error, 1) - go func() { - done <- runAdd(context.Background(), &out, c, testProject, testWorkload, testCustom) - }() - - select { - case err := <-done: - if err != nil { - t.Fatalf("one transient read failure must not fail the command: %v\n%s", err, out.String()) - } - if !failed { - t.Error("the interceptor never injected its failure, so nothing was ridden out") - } - case <-time.After(30 * time.Second): - t.Fatalf("runAdd never returned:\n%s", out.String()) - } -} diff --git a/internal/cmd/compute/domains/write_test.go b/internal/cmd/compute/domains/write_test.go deleted file mode 100644 index 629222ab..00000000 --- a/internal/cmd/compute/domains/write_test.go +++ /dev/null @@ -1,204 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package domains - -import ( - "bytes" - "context" - "errors" - "strings" - "testing" - - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - - "go.datum.net/compute/internal/cmd/compute/util" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" -) - -// failingClient answers reads normally and refuses one write, which is what a -// conflicting concurrent deploy or a missing update permission looks like. -func failingClient(t *testing.T, boom error, objs ...client.Object) client.WithWatch { - t.Helper() - return fake.NewClientBuilder(). - WithScheme(testScheme(t)). - WithStatusSubresource(&networkingv1alpha.HTTPProxy{}, &networkingv1alpha.NetworkService{}, &networkingv1alpha.Domain{}). - WithObjects(objs...). - WithInterceptorFuncs(interceptor.Funcs{ - Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { - return boom - }, - }). - Build() -} - -// TestAddReportsAWriteFailure: a hostname that could not be attached must fail -// the command and say so. The wait that follows is the part that looks like -// progress, so falling into it after a failed write would show a user a -// verification that is never going to happen. -func TestAddReportsAWriteFailure(t *testing.T) { - boom := errors.New("Operation cannot be fulfilled on httpproxies.networking.datumapis.com \"api\": the object has been modified") - c := failingClient(t, boom, programmedProxy(testWorkload, testCanonical), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true))) - - var out bytes.Buffer - err := runAdd(context.Background(), &out, c, testProject, testWorkload, testCustom) - if err == nil { - t.Fatal("a hostname that could not be attached must fail the command") - } - if !errors.Is(err, boom) { - t.Errorf("error = %v, want the server's message to reach the user", err) - } - for _, want := range []string{testCustom, testWorkload} { - if !strings.Contains(err.Error(), want) { - t.Errorf("error = %q, want it to name %q", err, want) - } - } - if strings.Contains(out.String(), "Verifying") { - t.Errorf("nothing was attached, so nothing may be reported as verifying:\n%s", out.String()) - } - if got := hostnamesOf(t, c); len(got) != 0 { - t.Errorf("hostnames = %v, want none attached", got) - } -} - -// TestRemoveReportsAWriteFailure: the same on the way out. A detach that did -// not happen must not print the "removed" summary, which is what a user reads -// as confirmation that the hostname is free to attach elsewhere. -func TestRemoveReportsAWriteFailure(t *testing.T) { - boom := errors.New("forbidden") - c := failingClient(t, boom, programmedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true))) - - var out bytes.Buffer - err := runRemove(context.Background(), &out, c, testProject, testWorkload, testCustom, true) - if err == nil { - t.Fatal("a hostname that could not be detached must fail the command") - } - if !errors.Is(err, boom) { - t.Errorf("error = %v, want the server's failure", err) - } - if strings.Contains(out.String(), "removed from workload") { - t.Errorf("nothing was detached, so nothing may claim it was:\n%s", out.String()) - } - if got := hostnamesOf(t, c); len(got) != 1 || got[0] != testCustom { - t.Errorf("hostnames = %v, want the hostname still attached", got) - } -} - -// TestAddKeepsTheHostnamesAlreadyThere: attaching a second domain must not -// disturb the first. `domains add` writes the whole spec back, so this is the -// same class of mistake a redeploy makes, one command over. -func TestAddKeepsTheHostnamesAlreadyThere(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - // The wait is not what this test is about, and a hostname the platform has - // not reported on stays pending forever: detach after the first poll. - var out bytes.Buffer - if err := runAdd(cancelled(), &out, c, testProject, testWorkload, "www."+testApex); err != nil { - t.Fatalf("runAdd: %v", err) - } - - if want := testCustom + ",www." + testApex; strings.Join(hostnamesOf(t, c), ",") != want { - t.Errorf("hostnames = %v, want %q — declared order, nothing lost", hostnamesOf(t, c), want) - } -} - -// TestRemoveKeepsTheOrderOfWhatIsLeft: removing the middle hostname of three -// must leave the other two in the order they were declared, since that order -// is what every view renders and what `add` appends to. -func TestRemoveKeepsTheOrderOfWhatIsLeft(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical, "one."+testApex, "two."+testApex, "three."+testApex), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - if err := runRemove(context.Background(), &out, c, testProject, testWorkload, "two."+testApex, true); err != nil { - t.Fatalf("runRemove: %v", err) - } - - if want := "one." + testApex + ",three." + testApex; strings.Join(hostnamesOf(t, c), ",") != want { - t.Errorf("hostnames = %v, want %q", hostnamesOf(t, c), want) - } -} - -// TestRemoveTheLastCustomHostname: the workload falls back to serving on its -// Datum-managed domain alone, and is told so — that sentence is the difference -// between "I have taken my site down" and "I have detached my domain". -func TestRemoveTheLastCustomHostname(t *testing.T) { - c := newFakeClient(t, - programmedProxy(testWorkload, testCanonical, testCustom), - serviceWith(testWorkload, 8080, location("DFW", 1, 1, true)), - ) - - var out bytes.Buffer - if err := runRemove(context.Background(), &out, c, testProject, testWorkload, testCustom, true); err != nil { - t.Fatalf("runRemove: %v", err) - } - - if got := hostnamesOf(t, c); len(got) != 0 { - t.Errorf("hostnames = %v, want none", got) - } - if !strings.Contains(out.String(), "Still serving on https://"+testCanonical) { - t.Errorf("removing the last custom domain must say what still answers:\n%s", out.String()) - } - - // And the list view still has a row for the workload: the managed domain - // is permanent. - var list bytes.Buffer - if err := runList(context.Background(), &list, c, testProject, util.OutputTable, false); err != nil { - t.Fatalf("runList: %v", err) - } - f := row(t, list.String(), testWorkload, testCanonical) - if f[len(f)-1] != managedMarker { - t.Errorf("row = %v, want the managed domain still listed", f) - } -} - -// TestListPropagatesLookupFailures: `domains` exists to answer "is my URL -// working". An unreadable control plane must say so rather than render an -// empty list, which reads as "you have no domains". -func TestListPropagatesLookupFailures(t *testing.T) { - boom := errors.New("connection refused") - c := interceptor.NewClient(newFakeClient(t, - programmedProxy(testWorkload, testCanonical, testCustom), - ), interceptor.Funcs{ - List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { - return boom - }, - }) - - var out bytes.Buffer - err := runList(context.Background(), &out, c, testProject, util.OutputTable, false) - if !errors.Is(err, boom) { - t.Fatalf("error = %v, want the lookup failure", err) - } - if strings.Contains(out.String(), "No domains in project") { - t.Errorf("an unreadable project must not be reported as an empty one:\n%s", out.String()) - } -} - -// TestDetailPropagatesLookupFailures: the same for the detail view, which is -// the one a user reaches for precisely when something is wrong. -func TestDetailPropagatesLookupFailures(t *testing.T) { - boom := errors.New("connection refused") - c := interceptor.NewClient(newFakeClient(t, workload(testWorkload)), interceptor.Funcs{ - List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { - return boom - }, - }) - - var out bytes.Buffer - err := runDetail(context.Background(), &out, c, testProject, testWorkload, util.OutputTable) - if !errors.Is(err, boom) { - t.Fatalf("error = %v, want the lookup failure", err) - } - if strings.Contains(err.Error(), "--http-port") { - t.Errorf("a failed lookup must not be reported as a workload that was never published: %v", err) - } -} diff --git a/internal/cmd/compute/open/browser.go b/internal/cmd/compute/open/browser.go deleted file mode 100644 index 58f82a15..00000000 --- a/internal/cmd/compute/open/browser.go +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package open - -import ( - "context" - "fmt" - neturl "net/url" - "os/exec" - "runtime" - "strings" -) - -// xdgOpen is the URL handler on every platform that is not macOS or Windows. -const xdgOpen = "xdg-open" - -// opener launches a URL in the user's default browser. It is a function type -// so tests can substitute one that records the URL instead of opening it. -type opener func(ctx context.Context, rawURL string) error - -// openBrowser is the opener used in production. Tests pass their own to run -// rather than reassigning this, so no test can start a real browser. -var openBrowser opener = openInDefaultBrowser - -// openInDefaultBrowser hands a URL to the platform's URL handler. -// -// The URL is validated before it gets here, so nothing but an https:// URL the -// server produced is ever passed on. No shell is involved: the argument is -// passed to the handler directly and is never word-split or expanded. -func openInDefaultBrowser(ctx context.Context, rawURL string) error { - if err := validateURL(rawURL); err != nil { - return err - } - - name, args := browserCommand(runtime.GOOS, rawURL) - - cmd := exec.CommandContext(ctx, name, args...) - if err := cmd.Run(); err != nil { - return fmt.Errorf("opening browser with %s: %w", name, err) - } - return nil -} - -// browserCommand returns the URL handler for an operating system and the -// arguments to hand it. Split out from openInDefaultBrowser so it can be -// tested for every platform without launching anything. -func browserCommand(goos, rawURL string) (name string, args []string) { - switch goos { - case "darwin": - return "open", []string{rawURL} - case "windows": - return "rundll32", []string{"url.dll,FileProtocolHandler", rawURL} - default: - return xdgOpen, []string{rawURL} - } -} - -// validateURL rejects anything that is not an https:// URL of the shape the -// platform hands back. The CLI only ever opens a URL the server reported, and -// this is the check that keeps it that way. -func validateURL(rawURL string) error { - if rawURL == "" { - return fmt.Errorf("no URL to open") - } - if strings.TrimSpace(rawURL) != rawURL { - return fmt.Errorf("refusing to open %q: URL has leading or trailing whitespace", rawURL) - } - - u, err := neturl.Parse(rawURL) - if err != nil { - return fmt.Errorf("refusing to open %q: not a valid URL", rawURL) - } - if u.Scheme != "https" { - return fmt.Errorf("refusing to open %q: only https:// URLs are opened", rawURL) - } - if u.Host == "" { - return fmt.Errorf("refusing to open %q: URL has no host", rawURL) - } - if u.User != nil { - return fmt.Errorf("refusing to open %q: URL carries embedded credentials", rawURL) - } - return nil -} diff --git a/internal/cmd/compute/open/browser_test.go b/internal/cmd/compute/open/browser_test.go deleted file mode 100644 index 747055cf..00000000 --- a/internal/cmd/compute/open/browser_test.go +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package open - -import ( - "context" - "strings" - "testing" -) - -// The substrings the refusal tests match on, named once so the same wording is -// asserted here and in open_test.go. -const ( - wantNotHTTPS = "only https:// URLs" - wantCredentials = "embedded credentials" - wantMalformed = "not a valid URL" -) - -func TestValidateURL(t *testing.T) { - tests := []struct { - name string - rawURL string - wantErr string - }{ - {name: "managed hostname", rawURL: "https://a1b2c3d4.datumproxy.net"}, - {name: "custom hostname with a path", rawURL: "https://api.example.com/healthz"}, - {name: "explicit port", rawURL: "https://api.example.com:8443"}, - - {name: "empty", rawURL: "", wantErr: "no URL to open"}, - {name: "plaintext http", rawURL: "http://api.example.com", wantErr: wantNotHTTPS}, - {name: "javascript scheme", rawURL: "javascript:alert(1)", wantErr: wantNotHTTPS}, - {name: "file scheme", rawURL: "file:///etc/passwd", wantErr: wantNotHTTPS}, - {name: "scheme-relative", rawURL: "//api.example.com", wantErr: wantNotHTTPS}, - {name: "bare hostname", rawURL: "api.example.com", wantErr: wantNotHTTPS}, - {name: "looks like a flag", rawURL: "--version", wantErr: wantNotHTTPS}, - {name: "no host", rawURL: "https://", wantErr: "no host"}, - {name: "embedded credentials", rawURL: "https://user:pw@api.example.com", wantErr: wantCredentials}, - {name: "leading whitespace", rawURL: " https://api.example.com", wantErr: "whitespace"}, - {name: "trailing newline", rawURL: "https://api.example.com\n", wantErr: "whitespace"}, - {name: "unparseable", rawURL: "https://not a host", wantErr: wantMalformed}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := validateURL(tc.rawURL) - switch { - case tc.wantErr == "" && err != nil: - t.Fatalf("validateURL(%q) = %v, want nil", tc.rawURL, err) - case tc.wantErr != "" && err == nil: - t.Fatalf("validateURL(%q) = nil, want an error containing %q", tc.rawURL, tc.wantErr) - case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): - t.Fatalf("validateURL(%q) = %q, want it to contain %q", tc.rawURL, err, tc.wantErr) - } - }) - } -} - -// The real opener refuses anything validateURL refuses, before a process is -// ever started. -func TestOpenInDefaultBrowserValidatesFirst(t *testing.T) { - if err := openInDefaultBrowser(context.Background(), "http://api.example.com"); err == nil { - t.Fatal("openInDefaultBrowser opened a plaintext URL, want a refusal") - } -} - -func TestBrowserCommand(t *testing.T) { - const rawURL = "https://a1b2c3d4.datumproxy.net" - - tests := []struct { - goos string - wantName string - wantArgs []string - }{ - {goos: "darwin", wantName: "open", wantArgs: []string{rawURL}}, - {goos: "linux", wantName: xdgOpen, wantArgs: []string{rawURL}}, - {goos: "freebsd", wantName: xdgOpen, wantArgs: []string{rawURL}}, - {goos: "windows", wantName: "rundll32", wantArgs: []string{"url.dll,FileProtocolHandler", rawURL}}, - } - - for _, tc := range tests { - t.Run(tc.goos, func(t *testing.T) { - name, args := browserCommand(tc.goos, rawURL) - if name != tc.wantName { - t.Errorf("name = %q, want %q", name, tc.wantName) - } - if strings.Join(args, " ") != strings.Join(tc.wantArgs, " ") { - t.Errorf("args = %q, want %q", args, tc.wantArgs) - } - }) - } -} diff --git a/internal/cmd/compute/open/custom_test.go b/internal/cmd/compute/open/custom_test.go deleted file mode 100644 index 819ed894..00000000 --- a/internal/cmd/compute/open/custom_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package open - -import ( - "context" - "errors" - "strings" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - - "go.datum.net/compute/internal/cmd/compute/url" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" -) - -const testCustom = "api.example.com" - -// withCustomHostname returns the objects for a workload serving on a custom -// hostname as well as its managed one. active decides whether the custom -// hostname is the one answering yet. -func withCustomHostname(t *testing.T, active bool) []client.Object { - t.Helper() - objs := publishedObjects(t, testCanonical, true) - - proxy := objs[1].(*networkingv1alpha.HTTPProxy) - proxy.Spec.Hostnames = []gatewayv1.Hostname{testCustom} - - status := metav1.ConditionFalse - reason := "DomainNotVerified" - if active { - status = metav1.ConditionTrue - reason = "Verified" - } - proxy.Status.HostnameStatuses = []networkingv1alpha.HostnameStatus{{ - Hostname: testCustom, - Conditions: []metav1.Condition{ - {Type: networkingv1alpha.HostnameConditionVerified, Status: status, Reason: reason, - Message: "waiting for the TXT record"}, - {Type: networkingv1alpha.HostnameConditionCertificateReady, Status: status, Reason: "CertificateIssued"}, - }, - }} - return objs -} - -// TestOpenPrefersTheCustomHostname: once a custom domain is serving, it is the -// address the user thinks of as their site, and it is the one a browser gets. -// The managed hostname still works and is still listed by `domains`; it is -// just not what `open` is for. -func TestOpenPrefersTheCustomHostname(t *testing.T) { - c := newFakeClient(t, withCustomHostname(t, true)...) - opener := &recordingOpener{} - - var out strings.Builder - if err := run(context.Background(), &out, c, testWorkload, false, opener.open); err != nil { - t.Fatalf("run: %v", err) - } - - want := "https://" + testCustom - if len(opener.calls) != 1 || opener.calls[0] != want { - t.Errorf("opened %v, want %q", opener.calls, want) - } - if !strings.Contains(out.String(), want) { - t.Errorf("output = %q, want it to name the custom hostname", out.String()) - } - if strings.Contains(out.String(), testCanonical) { - t.Errorf("output = %q, should not offer the managed hostname once a custom one serves", out.String()) - } -} - -// TestOpenFallsBackWhileACustomHostnameIsPending: a hostname that has not -// verified does not answer, so opening it would show the user a browser error -// for a workload that is running perfectly. The managed hostname is what works. -func TestOpenFallsBackWhileACustomHostnameIsPending(t *testing.T) { - c := newFakeClient(t, withCustomHostname(t, false)...) - opener := &recordingOpener{} - - var out strings.Builder - if err := run(context.Background(), &out, c, testWorkload, false, opener.open); err != nil { - t.Fatalf("run: %v", err) - } - - if len(opener.calls) != 1 || opener.calls[0] != testURL { - t.Errorf("opened %v, want the managed URL %q while the custom hostname is pending", opener.calls, testURL) - } -} - -// TestOpenURLMatchesWhatDomainsShows is the cross-command contract: the URL -// `open --url` prints is exactly the one the domains detail view reports, so a -// user never has two answers to "what is my URL". -func TestOpenURLMatchesWhatDomainsShows(t *testing.T) { - c := newFakeClient(t, withCustomHostname(t, true)...) - - info, err := url.ForWorkload(context.Background(), c, testWorkload) - if err != nil { - t.Fatalf("lookup: %v", err) - } - - var out strings.Builder - if err := run(context.Background(), &out, c, testWorkload, true, (&recordingOpener{}).open); err != nil { - t.Fatalf("run: %v", err) - } - if got := strings.TrimSpace(out.String()); got != info.URL { - t.Errorf("open --url printed %q, but the URL info says %q", got, info.URL) - } -} - -// A workload the CLI cannot read at all is a different problem from one that -// does not exist, and must not be reported as either "not found" or "no URL". -func TestOpenReportsAnUnreadableWorkload(t *testing.T) { - boom := errors.New("workloads.compute.datumapis.com is forbidden") - c := interceptor.NewClient(newFakeClient(t), interceptor.Funcs{ - Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { - return boom - }, - }) - - opener := &recordingOpener{} - var out strings.Builder - err := run(context.Background(), &out, c, testWorkload, false, opener.open) - if !errors.Is(err, boom) { - t.Fatalf("error = %v, want the server's failure", err) - } - if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "--http-port") { - t.Errorf("a permission failure must not read as a missing or unpublished workload: %v", err) - } - if len(opener.calls) != 0 { - t.Errorf("opened %v, want nothing", opener.calls) - } -} diff --git a/internal/cmd/compute/open/open.go b/internal/cmd/compute/open/open.go deleted file mode 100644 index c9dc9917..00000000 --- a/internal/cmd/compute/open/open.go +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -// Package open implements `datumctl compute open`: the one-word path from a -// workload name to the URL in a browser. -package open - -import ( - "context" - "fmt" - "io" - - "github.com/spf13/cobra" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - computev1alpha "go.datum.net/compute/api/v1alpha" - "go.datum.net/compute/internal/cmd/compute/url" - "go.datum.net/compute/internal/cmd/compute/util" -) - -// Command returns the `datumctl compute open` command. -func Command() *cobra.Command { - var urlOnly bool - - cmd := &cobra.Command{ - Use: "open ", - Short: "Open a workload's URL in the default browser", - Long: "Open a workload's URL in the default browser.\n\n" + - "With --url the URL is printed and nothing is opened, so it can be piped\n" + - "into another command.", - Example: " # Open the URL for the \"api\" workload\n" + - " datumctl compute open api\n\n" + - " # Print the URL instead of opening it\n" + - " datumctl compute open api --url\n\n" + - " # Use it in a script\n" + - " curl -sS \"$(datumctl compute open api --url)/healthz\"", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - c, err := util.NewClient(util.ProjectFromCmd(cmd)) - if err != nil { - return err - } - return run(cmd.Context(), cmd.OutOrStdout(), c, args[0], urlOnly, openBrowser) - }, - ValidArgsFunction: util.CompleteWorkloadNames, - } - - cmd.Flags().BoolVar(&urlOnly, "url", false, "Print the URL instead of opening it") - - return cmd -} - -// run resolves the workload's URL and either prints it or opens it. -// -// Two failures a developer hits here are different problems with different -// fixes, so they get different errors: a workload that does not exist, and a -// workload that exists but has never been published. -func run(ctx context.Context, out io.Writer, c client.Client, workloadName string, urlOnly bool, open opener) error { - if ctx == nil { - ctx = context.Background() - } - - var workload computev1alpha.Workload - if err := c.Get(ctx, types.NamespacedName{Namespace: util.ResourceNamespace, Name: workloadName}, &workload); err != nil { - if k8serrors.IsNotFound(err) { - return fmt.Errorf("workload %q not found — run 'datumctl compute workloads' to see what is deployed", workloadName) - } - return fmt.Errorf("getting workload: %w", err) - } - - info, err := url.ForWorkload(ctx, c, workloadName) - if err != nil { - return err - } - if info == nil { - return fmt.Errorf( - "workload %q has no URL — it does not declare an HTTP port.\nTo publish it: datumctl compute deploy %s --http-port ", - workloadName, workloadName, - ) - } - if info.URL == "" { - return fmt.Errorf("workload %q does not have a URL yet — the platform is still assigning one. Try again in a moment", workloadName) - } - - // Validate before anything else touches the URL, so the bare --url form is - // held to the same standard as the one handed to a browser. - if err := validateURL(info.URL); err != nil { - return err - } - - // The pipe-friendly form: the URL, a newline, nothing else. - if urlOnly { - fmt.Fprintln(out, info.URL) - return nil - } - - if !info.Live() { - fmt.Fprintln(out, "The URL is not fully live yet — it may not respond immediately.") - } - fmt.Fprintf(out, "Opening %s\n", info.URL) - - return open(ctx, info.URL) -} diff --git a/internal/cmd/compute/open/open_test.go b/internal/cmd/compute/open/open_test.go deleted file mode 100644 index fdcf9b2f..00000000 --- a/internal/cmd/compute/open/open_test.go +++ /dev/null @@ -1,283 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package open - -import ( - "context" - "errors" - "strings" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - - computev1alpha "go.datum.net/compute/api/v1alpha" - "go.datum.net/compute/internal/cmd/compute/url" - "go.datum.net/compute/internal/cmd/compute/util" - networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" -) - -const ( - testWorkload = "api" - testPortName = "http" - testCanonical = "a1b2c3d4.datumproxy.net" - testURL = "https://" + testCanonical -) - -func testScheme(t *testing.T) *runtime.Scheme { - t.Helper() - s := runtime.NewScheme() - if err := computev1alpha.AddToScheme(s); err != nil { - t.Fatalf("registering compute scheme: %v", err) - } - if err := networkingv1alpha.AddToScheme(s); err != nil { - t.Fatalf("registering networking scheme: %v", err) - } - return s -} - -func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch { - t.Helper() - return fake.NewClientBuilder(). - WithScheme(testScheme(t)). - WithStatusSubresource(&networkingv1alpha.HTTPProxy{}, &networkingv1alpha.NetworkService{}). - WithObjects(objs...). - Build() -} - -func testWorkloadObject(name string) *computev1alpha.Workload { - return &computev1alpha.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: util.ResourceNamespace, - UID: "workload-uid", - }, - } -} - -// publishedObjects returns the two objects the platform reports for a workload -// that is serving on the managed hostname. -func publishedObjects(t *testing.T, canonical string, programmed bool) []client.Object { - t.Helper() - w := testWorkloadObject(testWorkload) - - proxy := url.BuildHTTPProxy(w, testPortName, nil) - proxy.Status.CanonicalHostname = canonical - programmedStatus := metav1.ConditionFalse - if programmed { - programmedStatus = metav1.ConditionTrue - } - proxy.Status.Conditions = []metav1.Condition{ - {Type: networkingv1alpha.HTTPProxyConditionProgrammed, Status: programmedStatus, Reason: "Programmed"}, - {Type: networkingv1alpha.HTTPProxyConditionCertificatesReady, Status: metav1.ConditionTrue, Reason: "AllCertificatesReady"}, - } - - svc := url.BuildNetworkService(w, testPortName, 8080) - svc.Status.Summary = networkingv1alpha.NetworkServiceSummary{Locations: 1, Members: 2, Healthy: 2} - - return []client.Object{w, proxy, svc} -} - -// recordingOpener stands in for the browser: it records what it was handed and -// never launches anything. -type recordingOpener struct { - calls []string - err error -} - -func (r *recordingOpener) open(_ context.Context, rawURL string) error { - r.calls = append(r.calls, rawURL) - return r.err -} - -func TestRun(t *testing.T) { - tests := []struct { - name string - // objects seeded into the control plane - objects func(t *testing.T) []client.Object - // arguments - workload string - urlOnly bool - openErr error - - wantErr string - wantOut string - wantContains []string - wantOpened []string - }{ - { - name: "opens the URL in the browser", - objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, true) }, - workload: testWorkload, - wantOut: "Opening " + testURL + "\n", - wantOpened: []string{testURL}, - }, - { - name: "--url prints the bare URL and opens nothing", - objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, true) }, - workload: testWorkload, - urlOnly: true, - wantOut: testURL + "\n", - wantOpened: nil, - }, - { - name: "--url output has no decoration to break a pipe", - objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, false) }, - workload: testWorkload, - urlOnly: true, - wantOut: testURL + "\n", - wantOpened: nil, - }, - { - name: "a URL that is not live yet still opens, with a note", - objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, false) }, - workload: testWorkload, - wantContains: []string{ - "not fully live yet", - "Opening " + testURL, - }, - wantOpened: []string{testURL}, - }, - { - name: "workload that is not published names the command that publishes it", - objects: func(t *testing.T) []client.Object { - return []client.Object{testWorkloadObject(testWorkload)} - }, - workload: testWorkload, - wantErr: "datumctl compute deploy api --http-port", - wantOpened: nil, - }, - { - name: "workload that does not exist is a different error", - objects: func(t *testing.T) []client.Object { - return []client.Object{testWorkloadObject("other")} - }, - workload: testWorkload, - wantErr: `workload "api" not found`, - wantOpened: nil, - }, - { - name: "published but no hostname assigned yet", - objects: func(t *testing.T) []client.Object { - return publishedObjects(t, "", true) - }, - workload: testWorkload, - wantErr: "does not have a URL yet", - wantOpened: nil, - }, - { - name: "a hostname carrying credentials is never opened", - objects: func(t *testing.T) []client.Object { - return publishedObjects(t, "user:password@evil.example.com", true) - }, - workload: testWorkload, - wantErr: wantCredentials, - wantOpened: nil, - }, - { - name: "a malformed hostname is never opened", - objects: func(t *testing.T) []client.Object { - return publishedObjects(t, "not a host", true) - }, - workload: testWorkload, - wantErr: wantMalformed, - wantOpened: nil, - }, - { - name: "a malformed hostname is not printed by --url either", - objects: func(t *testing.T) []client.Object { - return publishedObjects(t, "not a host", true) - }, - workload: testWorkload, - urlOnly: true, - wantErr: wantMalformed, - wantOpened: nil, - }, - { - name: "a browser failure is reported", - objects: func(t *testing.T) []client.Object { return publishedObjects(t, testCanonical, true) }, - workload: testWorkload, - openErr: errors.New("no browser"), - wantErr: "no browser", - wantOpened: []string{testURL}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - c := newFakeClient(t, tc.objects(t)...) - opener := &recordingOpener{err: tc.openErr} - var out strings.Builder - - err := run(context.Background(), &out, c, tc.workload, tc.urlOnly, opener.open) - - switch { - case tc.wantErr == "" && err != nil: - t.Fatalf("run() error = %v, want nil", err) - case tc.wantErr != "" && err == nil: - t.Fatalf("run() error = nil, want one containing %q", tc.wantErr) - case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): - t.Fatalf("run() error = %q, want it to contain %q", err, tc.wantErr) - } - - if tc.wantOut != "" && out.String() != tc.wantOut { - t.Errorf("output = %q, want %q", out.String(), tc.wantOut) - } - for _, want := range tc.wantContains { - if !strings.Contains(out.String(), want) { - t.Errorf("output = %q, want it to contain %q", out.String(), want) - } - } - if got, want := strings.Join(opener.calls, ","), strings.Join(tc.wantOpened, ","); got != want { - t.Errorf("opened %q, want %q", got, want) - } - }) - } -} - -// A failing lookup is the platform's problem to report, not something to -// translate into "this workload has no URL". -func TestRunPropagatesLookupErrors(t *testing.T) { - boom := errors.New("connection refused") - c := interceptor.NewClient(newFakeClient(t, testWorkloadObject(testWorkload)), interceptor.Funcs{ - List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { - return boom - }, - }) - - opener := &recordingOpener{} - var out strings.Builder - err := run(context.Background(), &out, c, testWorkload, false, opener.open) - if !errors.Is(err, boom) { - t.Fatalf("run() error = %v, want the transport error to propagate", err) - } - if len(opener.calls) != 0 { - t.Fatalf("opened %v, want nothing", opener.calls) - } - if out.String() != "" { - t.Errorf("output = %q, want nothing", out.String()) - } -} - -func TestCommand(t *testing.T) { - cmd := Command() - - if cmd.Use != "open " { - t.Errorf("Use = %q", cmd.Use) - } - if cmd.ValidArgsFunction == nil { - t.Error("ValidArgsFunction is nil, want util.CompleteWorkloadNames") - } - if cmd.Flags().Lookup("url") == nil { - t.Error("--url flag is not registered") - } - if err := cmd.Args(cmd, []string{}); err == nil { - t.Error("Args accepted no workload name, want exactly one") - } - if err := cmd.Args(cmd, []string{"a", "b"}); err == nil { - t.Error("Args accepted two workload names, want exactly one") - } -} diff --git a/internal/cmd/compute/root.go b/internal/cmd/compute/root.go index 3f703d3b..d9c25a8c 100644 --- a/internal/cmd/compute/root.go +++ b/internal/cmd/compute/root.go @@ -8,9 +8,7 @@ import ( "go.datum.net/compute/internal/cmd/compute/build" "go.datum.net/compute/internal/cmd/compute/deploy" "go.datum.net/compute/internal/cmd/compute/destroy" - "go.datum.net/compute/internal/cmd/compute/domains" "go.datum.net/compute/internal/cmd/compute/instances" - "go.datum.net/compute/internal/cmd/compute/open" "go.datum.net/compute/internal/cmd/compute/quota" "go.datum.net/compute/internal/cmd/compute/restart" "go.datum.net/compute/internal/cmd/compute/rollout" @@ -39,9 +37,7 @@ func Command() *cobra.Command { access.Command(), deploy.Command(), destroy.Command(), - domains.Command(), instances.Command(), - open.Command(), quota.Command(), restart.Command(), rollout.Command(), diff --git a/internal/cmd/compute/url/publish.go b/internal/cmd/compute/url/publish.go index 28018d3c..0c82f1b8 100644 --- a/internal/cmd/compute/url/publish.go +++ b/internal/cmd/compute/url/publish.go @@ -125,7 +125,7 @@ func Declare( // to pick it back up. func detached(out io.Writer, workloadName string) { fmt.Fprintf(out, "\nDetached. Publishing continues in the background.\n") - fmt.Fprintf(out, " Check it with: datumctl compute open %s\n", workloadName) + fmt.Fprintf(out, " Check it with: datumctl compute workloads describe %s\n", workloadName) } // Wait polls until the workload's URL is live, printing each stage as it @@ -179,7 +179,7 @@ func Wait(ctx context.Context, out io.Writer, c client.Client, workloadName stri case <-deadline.C: return nil, fmt.Errorf( - "the URL for %q was still not answering after %s — it may yet come up; check it with: datumctl compute open %s", + "the URL for %q was still not answering after %s — it may yet come up; check it with: datumctl compute workloads describe %s", workloadName, maxWait, workloadName) case <-ticker.C: diff --git a/internal/cmd/compute/url/publish_failure_test.go b/internal/cmd/compute/url/publish_failure_test.go index 413f0b74..8cb6e3e4 100644 --- a/internal/cmd/compute/url/publish_failure_test.go +++ b/internal/cmd/compute/url/publish_failure_test.go @@ -359,7 +359,7 @@ func TestPublishDetachesWhenInterruptedMidWrite(t *testing.T) { if !strings.Contains(out.String(), "Detached") { t.Errorf("output = %q, want the same detach note the wait prints", out.String()) } - if !strings.Contains(out.String(), "datumctl compute open "+testWorkloadName) { + if !strings.Contains(out.String(), "datumctl compute workloads describe "+testWorkloadName) { t.Errorf("output = %q, want a pointer to how to pick it up again", out.String()) } }) diff --git a/internal/cmd/compute/url/publish_test.go b/internal/cmd/compute/url/publish_test.go index fb7f56ce..987113d9 100644 --- a/internal/cmd/compute/url/publish_test.go +++ b/internal/cmd/compute/url/publish_test.go @@ -79,8 +79,8 @@ func TestPublishCreatesBackendsBeforeTheProxy(t *testing.T) { if !strings.Contains(out.String(), "Detached") { t.Errorf("output = %q, want a detach note", out.String()) } - if !strings.Contains(out.String(), "datumctl compute open api") { - t.Errorf("output = %q, want a pointer to `datumctl compute open`", out.String()) + if !strings.Contains(out.String(), "datumctl compute workloads describe api") { + t.Errorf("output = %q, want a pointer to `datumctl compute workloads describe`", out.String()) } var svc networkingv1alpha.NetworkService diff --git a/internal/cmd/compute/url/render.go b/internal/cmd/compute/url/render.go index deaa9ac9..f50ca15a 100644 --- a/internal/cmd/compute/url/render.go +++ b/internal/cmd/compute/url/render.go @@ -29,7 +29,11 @@ func RenderDetail(out io.Writer, info *Info) { renderURLs(out, info) renderBackendLine(out, info) fmt.Fprintln(out) - fmt.Fprintf(out, "%-*s %s\n", labelWidth, "Health", health(info)) + // "Serving", not "Health": this block is embedded under a workload's own + // Health line, and two identically labelled rows at the same indent read as + // a contradiction rather than two different facts. Backend health is about + // whether the URL is taking traffic. + fmt.Fprintf(out, "%-*s %s\n", labelWidth, "Serving", health(info)) if len(info.Locations) > 0 { fmt.Fprintln(out) diff --git a/internal/cmd/compute/url/render_test.go b/internal/cmd/compute/url/render_test.go index 7b9e9c56..14e2fcc6 100644 --- a/internal/cmd/compute/url/render_test.go +++ b/internal/cmd/compute/url/render_test.go @@ -42,7 +42,9 @@ func TestRenderDetailDegraded(t *testing.T) { if f := row(t, got, "Backend"); f[1] != "port" || f[2] != "8080/tcp" { t.Errorf("Backend row = %v, want port 8080/tcp", f) } - if !strings.Contains(got, "Health Degraded — 2 of 4 backends healthy") { + // "Serving", not "Health": this block renders under a workload's own Health + // line in `workloads describe`, so the labels have to stay distinguishable. + if !strings.Contains(got, "Serving Degraded — 2 of 4 backends healthy") { t.Errorf("missing the health summary:\n%s", got) } diff --git a/internal/cmd/compute/workloads/workloads.go b/internal/cmd/compute/workloads/workloads.go index e2c21d6a..a813624f 100644 --- a/internal/cmd/compute/workloads/workloads.go +++ b/internal/cmd/compute/workloads/workloads.go @@ -566,6 +566,16 @@ func runDescribe(cmd *cobra.Command, args []string) error { fmt.Fprintf(out, "%-12s %s\n", "Health", health) fmt.Fprintf(out, "\n") + // URL block. A workload that was never published simply has no URL, so a + // lookup failure is reported and skipped rather than failing the whole + // describe — the config and health above are still what the user asked for. + if info, err := url.ForWorkload(ctx, c, workloadName); err != nil { + fmt.Fprintf(out, "URL\n (could not be read: %v)\n\n", err) + } else if info != nil { + url.RenderDetail(out, info) + fmt.Fprintf(out, "\n") + } + // Placements block. fmt.Fprintf(out, "Placements\n") if len(wl.Spec.Placements) == 0 { From 47a318bd52cbbae12bcd33d1cb482ae19eed4072 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 19:36:58 -0500 Subject: [PATCH 10/12] fix(cli): follow the locations API in the URL views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stacking on the locations work leaves the URL views speaking the old vocabulary. The per-location backend table said CITY, and its next-steps line told users to run 'compute instances --city=', a flag that no longer exists — the suggested command would have failed. Renames url.Location.City to Location, the column to LOCATION, and the suggested flag to --location, so every place a URL reports where it is serving from uses one word for it. Co-Authored-By: Claude Opus 5 --- docs/enhancements/datumctl-compute-urls.md | 36 ++-- docs/scoping/alb-workload-exposure.md | 199 +++++++++++++++++++++ internal/cmd/compute/url/lookup.go | 10 +- internal/cmd/compute/url/lookup_test.go | 2 +- internal/cmd/compute/url/publish.go | 4 +- internal/cmd/compute/url/render.go | 20 +-- internal/cmd/compute/url/render_test.go | 6 +- 7 files changed, 238 insertions(+), 39 deletions(-) create mode 100644 docs/scoping/alb-workload-exposure.md diff --git a/docs/enhancements/datumctl-compute-urls.md b/docs/enhancements/datumctl-compute-urls.md index 29ca8abf..687884d7 100644 --- a/docs/enhancements/datumctl-compute-urls.md +++ b/docs/enhancements/datumctl-compute-urls.md @@ -42,10 +42,10 @@ That boundary is why there is no `compute domains` command group and no `compute ### Deploy and get a URL ``` -$ datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --city=DFW,IAD --min=2 --http-port=8080 +$ datumctl compute deploy api --image=ghcr.io/acme/api:1.4.2 --location=us-east-1,eu-west-1 --min=2 --http-port=8080 Resolving workload "api" in project acme-prod... - Placement "default": cities=[DFW, IAD], min=2 + Placement "default": locations=[us-east-1, eu-west-1], min=2 HTTP service: port 8080 → Datum-managed URL Apply? (Y/n): y @@ -53,14 +53,14 @@ Apply? (Y/n): y Waiting for rollout. Ctrl-C to detach (rollout continues in background). - PLACEMENT CITY UPDATED READY OLD PHASE - default DFW 2 2 0 Done - default IAD 2 2 0 Done + PLACEMENT LOCATION UPDATED READY OLD PHASE + default us-east-1 2 2 0 Done + default eu-west-1 2 2 0 Done Rollout complete in 47s. Publishing... - Backends 4 healthy across DFW, IAD + Backends 4 healthy across us-east-1, eu-west-1 Edge programmed Certificate issued @@ -85,14 +85,14 @@ There is no `status` command in this CLI (the DX doc proposes one that was never ``` $ datumctl compute workloads - NAME CITIES READY IMAGE URL - api DFW, IAD 4/4 ghcr.io/acme/api:1.4.2 https://api.example.com - worker DFW 1/1 ghcr.io/acme/worker:2.0 — + NAME LOCATIONS READY IMAGE URL + api us-east-1, eu-west-1 4/4 ghcr.io/acme/api:1.4.2 https://api.example.com + worker us-east-1 1/1 ghcr.io/acme/worker:2.0 — ``` As a field, so `| jq -r .url` works: `datumctl compute workloads -o json`. -And in `describe`, with per-city backend health — the view that makes multi-city routing visible, which it previously was not: +And in `describe`, with per-location backend health — the view that makes multi-location serving visible, which it previously was not: ``` $ datumctl compute workloads describe api @@ -108,15 +108,15 @@ Backend port 8080/tcp Serving Degraded — 2 of 4 backends healthy - CITY BACKENDS HEALTHY SERVING - DFW 2 2 yes - IAD 2 0 no + LOCATION BACKENDS HEALTHY SERVING + us-east-1 2 2 yes + eu-west-1 2 0 no - IAD: no healthy backends — instances are running but not passing health checks. - Traffic is being served from DFW only. + eu-west-1: no healthy backends — instances are running but not passing health checks. + Traffic is being served from us-east-1 only. Next steps: - Check instances: datumctl compute instances --workload=api --city=IAD + Check instances: datumctl compute instances --workload=api --location=eu-west-1 ``` ### Stop serving @@ -127,7 +127,7 @@ Serving Degraded — 2 of 4 backends healthy $ datumctl compute destroy api Workload: api -Placements: 1 Cities: DFW, IAD +Placements: 1 Locations: us-east-1, eu-west-1 Min replicas: 2 URLs: https://api.example.com, https://a1b2c3d4.datumproxy.net @@ -145,7 +145,7 @@ Changed, and nothing added: ``` datumctl compute deploy --port → --http-port (breaking); --no-http removes it datumctl compute workloads URL column, and a `url` field in -o json/yaml -datumctl compute workloads describe URL, backend port, and per-city backend health +datumctl compute workloads describe URL, backend port, and per-location backend health datumctl compute destroy lists URLs in its summary and deletes them ``` diff --git a/docs/scoping/alb-workload-exposure.md b/docs/scoping/alb-workload-exposure.md new file mode 100644 index 00000000..55406e9a --- /dev/null +++ b/docs/scoping/alb-workload-exposure.md @@ -0,0 +1,199 @@ +# Scoping: Auto-creating an ALB / HTTP proxy to expose a Workload + +## 1. The NetworkService API (PR 411) + +**Status: `datum-cloud/network-services-operator#411` is OPEN and a DRAFT.** Branch `proto/network-service` → `main`. The body says verbatim: *"Draft: this is a working prototype to prove the design, not a merge candidate."* Design doc is `datum-cloud/enhancements#870`. Everything below can change. + +**GVK:** `networking.datumapis.com/v1alpha`, `Kind: NetworkService`, **namespaced**. Defined in `api/v1alpha/networkservice_types.go` (new, 305 lines); CRD at `config/crd/bases/networking.datumapis.com_networkservices.yaml`; registered as an IAM `ProtectedResource` parented to `resourcemanager.miloapis.com/Project` in `config/iam/protected-resources/networkservices.yaml`, so it is a **user-facing, project-scoped** resource. + +```yaml +apiVersion: networking.datumapis.com/v1alpha +kind: NetworkService +metadata: {name: storefront, namespace: default} +spec: + networkInterfaces: # REQUIRED + selector: # REQUIRED metav1.LabelSelector, CEL-validated non-empty + matchLabels: {compute.datumapis.com/workload-name: storefront} + ports: # REQUIRED, 1..16, unique name + unique number (CEL) + - name: http # DNS label, <=63, ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + port: 8080 # 1..65535 + protocol: TCP # optional, default TCP, enum{TCP} only + trafficDistribution: # optional, defaulted + strategy: Nearest # enum{Nearest} only +status: + summary: {locations: 2, members: 6, healthy: 5} + locations: # listType=map on name, MaxItems=64 + - {name: us-central-1, members: 3, healthy: 2, serving: true} + conditions: [MembersResolved, Ready] # both default-seeded Unknown/Pending +``` + +Key semantics: +- **The user gets no hostname or IP from a NetworkService.** Its status is membership/health only. The URL comes from the HTTPProxy in front of it. +- **Backends are selected by label, not referenced.** Nothing in the type names a Workload. Membership = `NetworkInterface` objects **in the same namespace** matching the selector (`internal/controller/networkservice_controller.go`, `matchingInterfaces(ctx, cl, service.Namespace, selector)`). +- Member health is read from the **new `HolderAvailable` condition on `NetworkInterface`** (`api/v1alpha/networkinterface_types.go`, +38 lines). NSO only ever writes it `Unknown`; True/False is the holder's (compute's) to write. +- Conditions: `MembersResolved` (reasons `NoMatchingInterfaces`, `MultipleNetworks`, `InvalidSelector`) and **`Ready` — "wait on this one rather than on what it summarizes."** + +**Label plumbing (already working in this repo):** compute stamps `compute.datumapis.com/{workload-name,placement-name,city-code,instance-index}` on the `NetworkInterfaceClaim` it creates — `internal/controller/networkinterfaceclaim.go:220-238` (`desiredNetworkInterfaceClaimLabels`). PR 411's new `internal/controller/networkinterface_labels.go` allow-lists the `compute.datumapis.com/` prefix and copies those keys claim→interface, and stamps `networking.datumapis.com/location`. So **`matchLabels: {compute.datumapis.com/workload-name: }` is the intended selector and needs no new labelling work in compute.** + +**HTTPProxy gains a fourth backend form** (`api/v1alpha/httpproxy_types.go`, +48/-1): + +```go +type NetworkServiceBackendRef struct { + Name string `json:"name"` // NetworkService in the same namespace + Port string `json:"port"` // names a spec.ports[].name, not a number +} +``` +```yaml +rules: + - backends: + - networkService: {name: storefront, port: http} +``` +Mutually exclusive with `endpoint`/`connector`/`instance` (CEL). **Backend TLS is rejected for `networkService` backends** — the edge always reaches members over plaintext HTTP. New reasons: `NetworkServiceBackendNotFound`, `NetworkServiceMembersUnreferenced` (>100 members: the proxy serves the first shard and says so). + +**Gateway API relation:** indirect. HTTPProxy reuses `gatewayv1` types for `Hostname`, `HTTPRouteMatch`, `HTTPRouteFilter`, `GatewayStatusAddress`, and NSO translates it into downstream Gateway/EnvoyProxy resources. The CLI never touches Gateway API objects. + +**Where the URL comes from** (already in the pinned dep, `api/v1alpha/httpproxy_types.go:240-280`): +- `status.canonicalHostname` — platform-managed stable `.datumproxy.net`. **This is the zero-config URL.** +- `status.addresses`, `status.hostnameStatuses[]` (per-hostname `Verified`, `DNSRecordProgrammed`, `Available`, `CertificateReady`). +- Conditions: `Accepted`, `Programmed`, `HostnamesVerified`, `CertificatesReady`, `DNSRecordsProgrammed`. +- `spec.hostnames` is **optional**; a custom hostname needs a verified `Domain` in the same namespace (auto-created if absent, but still requires user verification). + +## 2. The CLI today + +Entry point `cmd/datumctl-compute/main.go` → `internal/cmd/compute/root.go` (50 lines): `plugin.NewRootCmd("compute", …)` from `go.datum.net/datumctl/plugin`, which supplies persistent `--org`, `--project`, `-o/--output`. A `PersistentPreRunE` runs `util.RunActivationGate`. Subcommands are plain cobra `Command()` constructors registered at `root.go:35-46`. + +- **Client:** `internal/cmd/compute/util/client.go:34-69`. `client.New` (controller-runtime), bearer token from `plugin.Token()`, host = `https:///apis/resourcemanager.miloapis.com/v1alpha1/projects//control-plane`. Scheme registers `computev1alpha`, **`networkingv1alpha` (already!)**, `locationsv1alpha1`, `quotav1alpha1`. Everything lives in namespace `"default"` (`util.ResourceNamespace`, `client.go:24`). +- **Workload creation:** `internal/cmd/compute/deploy/deploy.go:131-263`. Typed structs, `c.Get` → `c.Create`/`c.Update`. `--port` produces exactly one `NamedPort{Name: "http", Port: n, Protocol: TCP}` (`deploy.go:183-187`). One interface on network `"default"` (`deploy.go:210-215`). +- **Precedent for auto-creating a dependent networking resource:** `ensureNetwork` (`deploy.go:384-428`) checks for the `Network`, prompts `"Create it now? (Y/n)"`, creates a minimal auto-IPAM `Network`, refuses in non-interactive mode without `--yes`. **The exposure flow should mirror this exactly.** +- **Readiness waiting:** `internal/cmd/compute/watch/watch.go:39-92` — 2s `time.Ticker` poll of `WorkloadDeploymentList` selected by `compute.datumapis.com/workload-uid`, tabwriter rows, `signal.NotifyContext` for Ctrl-C detach. +- **Status/conditions helpers:** `internal/cmd/compute/util/conditions.go` — `FindCondition`, `ReadinessBlock` (with an explicit rule: *"Callers must not branch on specific reason values — display whatever the server emits"*), `InstanceStatus`/`InstanceStatusDetail`. +- **Output:** `util/printer.go` (`PrintJSON`/`PrintYAML`), `util/table.go` (`NewTabWriter`). `-o yaml/json` exists **only on read commands** (`workloads`, `instances`, `quota`, `access`). +- **No `--dry-run` anywhere in the plugin.** A grep over `internal/cmd/` and `cmd/` returns nothing. +- **Delete:** `internal/cmd/compute/destroy/destroy.go:82-84` deletes only the `Workload`. +- **Completions:** `util/completion.go` — `CompleteWorkloadNames`, `CompleteCityCodes`, `CompleteOutputFormats`. + +**Dependency status (the key finding):** `go.mod:14-17` already pins `go.datum.net/network-services-operator v0.26.1-0.20260821014231-aceb24b1b569` as a **direct** dependency, with a comment saying it is pinned to main for the `Prepared` condition. That pin **has `httpproxy_types.go` but NOT `networkservice_types.go`**, and its `HTTPProxyRuleBackend` has no `NetworkService` field. Re-pinning is required and is the hard blocker. Once re-pinned, **no scheme change is needed** — PR 411 adds `NetworkService`/`NetworkServiceList` to the existing `SchemeBuilder` in `api/v1alpha/groupversion_info.go` (+4), which `util/client.go:53` already calls. + +Compute also already writes the condition NSO reads: `internal/controller/networkinterface_holder.go:27` declares `networkInterfaceHolderAvailable = "HolderAvailable"` (as a local literal; `datum-cloud/compute#254`, still open, swaps it for the NSO constant). + +## 3. Proposed UX + +### Alternative A — flag on `deploy` +``` +datumctl compute deploy api --image=… --city=DFW --port=8080 --expose-http +datumctl compute deploy api … --expose-http --hostname=api.example.com +``` +Pros: single command from source to URL; matches the DX arc in `docs/enhancements/datumctl-compute-dx.md` (whose interactive mock at line 91 already prompts `? Expose port (optional): 8080`). Cons: no way to expose an existing workload; no way to unexpose without editing a manifest; couples exposure lifecycle to deploy lifecycle; `deploy -f workload.yaml` has nowhere sensible to put the flag. + +### Alternative B — dedicated `expose` verb group +``` +datumctl compute expose [--port=8080|--port-name=http] [--hostname=…] [--wait] [-o yaml] +datumctl compute expose status +datumctl compute unexpose +``` +Pros: exposure has its own lifecycle (hostnames, DNS, certs, delete semantics) and deserves its own verbs; works on existing workloads; `unexpose` is discoverable; `expose -o yaml` gives a manifest-first path without inventing `--dry-run`. Cons: two commands to a URL. + +### Recommendation: **B as the foundation, A as a thin caller.** + +Build `internal/cmd/compute/expose/` with an exported `Ensure(ctx, c, out, opts)` that owns all resource construction and readiness. Then `deploy --expose-http` is ~10 lines calling `Ensure` after `watch.Rollout` returns. Rationale: + +1. Exposure state must be inspectable and removable independent of the workload — `expose status` / `unexpose` are not optional, so B's surface has to exist regardless. A alone cannot get there. +2. The custom-hostname path (Domain verification, TXT records, cert issuance) is inherently multi-step and interactive; it needs a command that can be re-run to poll. Bolting that onto `deploy` makes `deploy` unpredictable. +3. `deploy` already sets the precedent for auto-creating a dependency with a prompt (`ensureNetwork`), so `--expose-http` fits naturally as sugar without owning the logic. +4. `expose -o yaml` printing the two objects gives the manifest-driven users (`deploy -f`) what they need, and is cheaper than retrofitting `--dry-run` across the plugin. + +**Recommended phase-1 scope: no `--hostname`.** Ship the zero-config path only — `status.canonicalHostname` gives a working `https://.datumproxy.net` with a platform-managed cert and no user DNS. Defer custom hostnames to phase 2. + +Proposed output: +``` +$ datumctl compute expose api --port=8080 + networkservice/api created (selector: compute.datumapis.com/workload-name=api) + httpproxy/api created +Waiting for endpoints and edge programming. Ctrl-C to detach. + + RESOURCE STATE + networkservice/api Ready (2 locations, 4 members, 4 healthy) + httpproxy/api Programmed + + https://a1b2c3d4.datumproxy.net +``` + +## 4. Resources created, order, ownership, deletion + +Order (each `Get` → `Create`-or-`Update`, matching `deploy.go:238-249`): + +1. **Preflight.** Workload exists; resolve the port. Prefer an existing `NamedPort` from `workload.Spec.Template.Spec.Runtime.Sandbox.Containers[*].Ports` (or `VirtualMachine.Ports`); require `--port`/`--port-name` only when ambiguous. Fail early with a clear message if the workload declares no ports. +2. **`NetworkService/`** in `default`: + - `spec.networkInterfaces.selector.matchLabels = {compute.datumapis.com/workload-name: }` (constant `computev1alpha.WorkloadNameLabel`, `api/v1alpha/labels.go:23`). + - `spec.ports = [{name: , port: , protocol: TCP}]`. + - Leave `trafficDistribution` unset (the type comment explicitly says *"Leave it unset"*). +3. **`HTTPProxy/`** in `default`: `spec.rules[0].backends[0].networkService = {name: , port: }`. Omit `spec.hostnames` in phase 1. Omit `matches` (CRD defaults to `PathPrefix: /`). Never set backend `tls`. +4. **Wait** (opt-out with `--no-wait`): poll `NetworkService` for `Ready=True`, then `HTTPProxy` for `Programmed=True` and non-empty `status.canonicalHostname`. Surface blocking reason+message verbatim via `util.ReadinessBlock`, per the existing house rule. + +**Ownership and labels** — set on both objects: +- `ownerReferences: [{apiVersion: compute.datumapis.com/v1alpha, kind: Workload, name, uid, controller: false, blockOwnerDeletion: false}]`. Same namespace, so this is legal; cross-group is fine. +- `labels: {compute.datumapis.com/workload-name: , compute.datumapis.com/workload-uid: }` — existing constants `WorkloadNameLabel` / `WorkloadUIDLabel` (`api/v1alpha/labels.go:19-23`). The labels, not the ownerRef, are what `expose status` and `unexpose` list on; they also survive a project control plane whose GC behaviour is unverified. + +**On delete:** +- `unexpose `: delete `HTTPProxy` first, then `NetworkService` (proxy-first avoids a window where the proxy reports `NetworkServiceBackendNotFound`), selected by the UID label. Warn that any `Domain` created for a custom hostname is deliberately left behind (it is a project-level ownership record, not per-workload). +- `destroy ` (`destroy.go`): list the labelled `HTTPProxy`/`NetworkService`, include them in the confirmation summary, and delete them explicitly rather than trusting owner-reference GC. Do not silently rely on GC until it is confirmed to run in project virtual control planes. +- The `Network` is never deleted (consistent with `ensureNetwork` never cleaning up). + +## 5. Implementation plan + +**Dependencies** +- `go.mod:14-17` — re-pin `go.datum.net/network-services-operator` to a commit carrying `NetworkService` + the `networkService` backend field. **Blocked on PR 411 merging** (it is an explicit non-merge-candidate today). Update the existing pin comment, which currently explains the `Prepared` condition rationale. +- No new modules. `sigs.k8s.io/gateway-api v1.5.1` is already required (`go.mod:29`) for the `gatewayv1.Hostname` types. + +**Scheme registration:** none. `networkingv1alpha.AddToScheme` at `internal/cmd/compute/util/client.go:53` picks up the new kinds automatically. Add a defensive `meta.IsNoMatchError` check so an older control plane yields *"HTTP exposure is not available in this project"* rather than a raw REST mapper error. + +**Files to touch** + +| File | Change | +|---|---| +| `go.mod` / `go.sum` | Re-pin NSO | +| `internal/cmd/compute/expose/expose.go` *(new)* | `Command()`, `Ensure()`, `Remove()`, `Status()` | +| `internal/cmd/compute/expose/resources.go` *(new)* | Pure builders: workload → `NetworkService` + `HTTPProxy`. Unit-testable, no client. | +| `internal/cmd/compute/expose/wait.go` *(new)* | Ticker-based readiness, modelled on `watch/watch.go:39-92` | +| `internal/cmd/compute/expose/*_test.go` *(new)* | Builder table tests + fake-client flow tests | +| `internal/cmd/compute/root.go:35-46` | Register `expose.Command()`, `unexpose.Command()` | +| `internal/cmd/compute/deploy/deploy.go` | `--expose-http` flag (`~line 85`), validation in `runDeploy`, call `expose.Ensure` after `watch.Rollout` (`:262`) | +| `internal/cmd/compute/destroy/destroy.go:55-84` | List + summarize + delete exposure resources | +| `internal/cmd/compute/util/conditions.go` | `NetworkServiceStatus()` / `HTTPProxyStatus()` summarizers, same shape as `InstanceStatus` | +| `internal/cmd/compute/util/completion.go` | `CompleteExposedWorkloads` for `unexpose` | +| `docs/enhancements/datumctl-compute-dx.md` | Update the interactive mock (line 91) and add the exposure flow | + +**RBAC:** the CLI acts as the end user via `plugin.Token()`; there is no service account to grant. The user needs `networking.datumapis.com/networkservices.{create,get,list,watch,update,patch,delete}` and the equivalent `httpproxies` permissions. PR 411 adds `networkservices.{create,update,patch,delete}` **only to `config/iam/roles/networking-admin.yaml`** and read verbs to `networking-viewer.yaml`. **A project member holding only compute roles will get a 403.** This is a cross-repo prerequisite: either the compute roles need these permissions, or the docs must state that `networking-admin` is required to expose a workload. Worth raising with the networking team before build starts. + +## 6. Open questions and risks + +1. **PR 411 is a draft prototype, not a merge candidate.** Everything is blocked on it. Treat all field names as provisional. +2. **The PR body contradicts the code.** The body's YAML example uses `spec.networkInterfaceClaims:`, but the Go type is `NetworkInterfaces NetworkServiceInterfaceSelector` with json tag `networkInterfaces`, and the CRD/chainsaw test both use `networkInterfaces`. Confirm which survives before writing builders. +3. **Biggest functional risk — membership may not resolve at all.** The controller lists `NetworkInterface` objects **in the NetworkService's own namespace**. Compute creates claims in the cell control plane: *"the claim is served by the control plane the instance runs in"* (`internal/controller/networkinterfaceclaim.go:69-72`). PR 411 states plainly: *"claims are not published to the consumer's project, so membership currently resolves where the claims already are."* Until interfaces are projected into the project control plane, a CLI-written NetworkService in `default` will sit at `MembersResolved=False/NoMatchingInterfaces` forever. **Verify this against a real staging project before committing to the design.** +4. **Multi-city is the default and is currently broken.** PR 411: *"a service with members in two locations binds no VRF and fails every request"* until `datum-cloud/cloud#16` lands. `deploy --city=DFW,IAD` is the documented happy path (`deploy.go:60`). The CLI must detect >1 city and warn loudly, or refuse, rather than producing a silently non-serving URL. +5. **No location coordinates exist yet**, so `Nearest` ranking is not actually computable — cross-location behaviour is untested end to end (single-cell environment). +6. **Single network per service.** A workload with interfaces on two networks yields `MultipleNetworks`. Today `deploy` hardcodes one interface on `"default"` (`deploy.go:210-215`), so this is safe now but fragile; consider adding the network to the selector. +7. **100-member cap.** Past it the proxy serves the first shard and reports `NetworkServiceMembersUnreferenced`. `expose status` must surface it. +8. **Port-name mismatch.** `computev1alpha.NamedPort.Name` (`api/v1alpha/instance_types.go:254-258`) has no pattern constraint; `NetworkServicePort.Name` requires `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`, <=63. Needs sanitization with a clear error. `deploy` currently hardcodes `"http"` (`deploy.go:185`), which is safe. +9. **Plaintext to the backend, always.** `networkService` backends reject `tls`. Users terminating TLS in their container will break. Must be documented. +10. **DNS/certs.** Phase 1 is free — `.datumproxy.net` with platform-managed A/AAAA and certs. Phase 2 (`--hostname`) drags in `Domain` creation, TXT/HTTP verification, `HostnamesVerified`/`CertificatesReady`/`DNSRecordsProgrammed`, and hostname-uniqueness conflicts across the whole platform. Substantially more work than phase 1. +11. **`HolderAvailable` string coupling.** Compute writes the literal `"HolderAvailable"` (`internal/controller/networkinterface_holder.go:27`). If NSO renames it, compute keeps compiling and every member silently reads unhealthy. `datum-cloud/compute#254` fixes this and should land first. +12. **Multi-tenancy.** All resources go to namespace `default` in the project's virtual control plane, so naming collides on the workload name — acceptable, and it makes exposure idempotent per workload, but it means one workload cannot have two proxies. +13. **Alpha API, no conversion guarantees.** Handle `NoKindMatch` gracefully. + +## 7. Effort estimate + +| Chunk | Est. | +|---|---| +| Re-pin NSO, verify build + scheme, `NoKindMatch` guard | 0.5 d *(gated on PR 411)* | +| Resource builders + table tests (`resources.go`) | 1.5 d | +| `expose` / `unexpose` / `expose status` commands, fake-client tests | 2 d | +| Readiness watcher + status summarizers + condition messaging | 2 d | +| `deploy --expose-http` wiring + interactive prompt | 1 d | +| `destroy` cascade, ownership/labels, completions | 1 d | +| Docs (`datumctl-compute-dx.md`) | 0.5 d | +| **Phase 1 total** | **~8.5 dev-days** | +| Phase 2: `--hostname`, Domain creation + verification UX, cert/DNS status | +4 d | +| Cross-repo prerequisites (IAM roles, interface projection, `#254`) | not estimated — external | + +Add ~1 d of slack for API churn while PR 411 is a draft. diff --git a/internal/cmd/compute/url/lookup.go b/internal/cmd/compute/url/lookup.go index 932f64c4..4577b0cb 100644 --- a/internal/cmd/compute/url/lookup.go +++ b/internal/cmd/compute/url/lookup.go @@ -43,13 +43,13 @@ type Backends struct { // Location reports the backends a URL has in one city, and whether that city // is taking traffic. type Location struct { - // City is the location as the platform reports it. - City string `json:"city"` - // Backends is how many instances in this city back the URL. + // Location is the location as the platform reports it. + Location string `json:"location"` + // Backends is how many instances in this location back the URL. Backends int32 `json:"backends"` // Healthy is how many of them are taking traffic. Healthy int32 `json:"healthy"` - // Serving reports whether this city is in rotation. + // Serving reports whether this location is in rotation. Serving bool `json:"serving"` } @@ -315,7 +315,7 @@ func newInfo(workloadName string, proxy *networkingv1alpha.HTTPProxy, service *n } for _, l := range service.Status.Locations { info.Locations = append(info.Locations, Location{ - City: l.Name, + Location: l.Name, Backends: l.Members, Healthy: l.Healthy, Serving: l.Serving, diff --git a/internal/cmd/compute/url/lookup_test.go b/internal/cmd/compute/url/lookup_test.go index 0927f0a3..e07b4caa 100644 --- a/internal/cmd/compute/url/lookup_test.go +++ b/internal/cmd/compute/url/lookup_test.go @@ -136,7 +136,7 @@ func TestForWorkloadPublished(t *testing.T) { if info.Backends != want { t.Errorf("Backends = %+v, want %+v", info.Backends, want) } - if len(info.Locations) != 2 || info.Locations[0] != (Location{City: "DFW", Backends: 2, Healthy: 2, Serving: true}) { + if len(info.Locations) != 2 || info.Locations[0] != (Location{Location: "DFW", Backends: 2, Healthy: 2, Serving: true}) { t.Errorf("Locations = %+v", info.Locations) } if info.Proxy == nil || info.Service == nil { diff --git a/internal/cmd/compute/url/publish.go b/internal/cmd/compute/url/publish.go index 0c82f1b8..4ebf148f 100644 --- a/internal/cmd/compute/url/publish.go +++ b/internal/cmd/compute/url/publish.go @@ -279,9 +279,9 @@ func servingCities(info *Info) []string { serving := make([]string, 0, len(info.Locations)) all := make([]string, 0, len(info.Locations)) for _, l := range info.Locations { - all = append(all, l.City) + all = append(all, l.Location) if l.Serving { - serving = append(serving, l.City) + serving = append(serving, l.Location) } } if len(serving) > 0 { diff --git a/internal/cmd/compute/url/render.go b/internal/cmd/compute/url/render.go index f50ca15a..3e156c73 100644 --- a/internal/cmd/compute/url/render.go +++ b/internal/cmd/compute/url/render.go @@ -12,7 +12,7 @@ import ( ) // RenderDetail writes the per-URL detail view: where the URL is, what backs -// it, and — the part a multi-city platform owes its users — which city is +// it, and — the part a multi-location platform owes its users — which one is // actually serving. // // Nothing here names the machinery. When something is wrong, the server's own @@ -74,14 +74,14 @@ func renderBackendLine(out io.Writer, info *Info) { fmt.Fprintf(out, "%-*s port %d/%s\n", labelWidth, "Backend", info.Port, protocol) } -// renderLocations prints the per-city breakdown, indented under the label +// renderLocations prints the per-location breakdown, indented under the label // column so it reads as part of the health block. func renderLocations(out io.Writer, info *Info) { indent := strings.Repeat(" ", labelWidth+1) tw := util.NewTabWriter(out) - fmt.Fprintf(tw, "%sCITY\tBACKENDS\tHEALTHY\tSERVING\n", indent) + fmt.Fprintf(tw, "%sLOCATION\tBACKENDS\tHEALTHY\tSERVING\n", indent) for _, l := range info.Locations { - fmt.Fprintf(tw, "%s%s\t%d\t%d\t%s\n", indent, l.City, l.Backends, l.Healthy, yesNo(l.Serving)) + fmt.Fprintf(tw, "%s%s\t%d\t%d\t%s\n", indent, l.Location, l.Backends, l.Healthy, yesNo(l.Serving)) } _ = tw.Flush() } @@ -108,7 +108,7 @@ func renderDiagnosis(out io.Writer, info *Info) { var unhealthy []string for _, l := range info.Locations { if l.Backends > 0 && l.Healthy == 0 { - unhealthy = append(unhealthy, l.City) + unhealthy = append(unhealthy, l.Location) } } @@ -120,12 +120,12 @@ func renderDiagnosis(out io.Writer, info *Info) { fmt.Fprintln(out) indent := strings.Repeat(" ", 7) - for _, city := range unhealthy { - fmt.Fprintf(out, " %s: no healthy backends — instances are running but not passing health checks.\n", city) + for _, location := range unhealthy { + fmt.Fprintf(out, " %s: no healthy backends — instances are running but not passing health checks.\n", location) serving := servingCities(info) switch { case len(serving) == 0: - fmt.Fprintf(out, "%sNo city is taking traffic, so the URL is not answering.\n", indent) + fmt.Fprintf(out, "%sNo location is taking traffic, so the URL is not answering.\n", indent) case len(serving) == 1: fmt.Fprintf(out, "%sTraffic is being served from %s only.\n", indent, serving[0]) default: @@ -143,8 +143,8 @@ func renderDiagnosis(out io.Writer, info *Info) { fmt.Fprintf(out, " Check instances: datumctl compute instances --workload=%s\n", info.WorkloadName) return } - for _, city := range unhealthy { - fmt.Fprintf(out, " Check instances: datumctl compute instances --workload=%s --city=%s\n", info.WorkloadName, city) + for _, location := range unhealthy { + fmt.Fprintf(out, " Check instances: datumctl compute instances --workload=%s --location=%s\n", info.WorkloadName, location) } } diff --git a/internal/cmd/compute/url/render_test.go b/internal/cmd/compute/url/render_test.go index 14e2fcc6..40af88fa 100644 --- a/internal/cmd/compute/url/render_test.go +++ b/internal/cmd/compute/url/render_test.go @@ -48,14 +48,14 @@ func TestRenderDetailDegraded(t *testing.T) { t.Errorf("missing the health summary:\n%s", got) } - if f := row(t, got, "CITY"); strings.Join(f, " ") != "CITY BACKENDS HEALTHY SERVING" { + if f := row(t, got, "LOCATION"); strings.Join(f, " ") != "LOCATION BACKENDS HEALTHY SERVING" { t.Errorf("table header = %v", f) } if f := row(t, got, "DFW"); strings.Join(f[1:], " ") != "2 2 yes" { t.Errorf("DFW row = %v, want 2 2 yes", f) } if f := row(t, got, "IAD:"); len(f) == 0 { - t.Error("expected a narrative line for the unhealthy city") + t.Error("expected a narrative line for the unhealthy location") } if f := row(t, got, "IAD"); strings.Join(f[1:], " ") != "2 0 no" { t.Errorf("IAD row = %v, want 2 0 no", f) @@ -65,7 +65,7 @@ func TestRenderDetailDegraded(t *testing.T) { "IAD: no healthy backends — instances are running but not passing health checks.", "Traffic is being served from DFW only.", "Next steps:", - "datumctl compute instances --workload=api --city=IAD", + "datumctl compute instances --workload=api --location=IAD", } { if !strings.Contains(got, want) { t.Errorf("output missing %q:\n%s", want, got) From accb6491b2b0c6f8256c752786c3a52ae2a87fec Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 19:45:20 -0500 Subject: [PATCH 11/12] fix(cli): satisfy the linter after the locations rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's golangci-lint run fails on this branch: deployFromFlags crossed the gocyclo limit at 32 once the locations validation and the HTTP service handling both landed in it, and two test fixtures wrote "http" where the httpPortName constant already exists. Lifts the three mutually exclusive placement flags into resolveLocationSelector, which is pure and now has the coverage the inline version never could — everything else in deployFromFlags needs a control plane behind the activation gate. Co-Authored-By: Claude Opus 5 --- internal/cmd/compute/deploy/deploy.go | 54 +++++++----- internal/cmd/compute/deploy/deploy_test.go | 2 +- .../compute/deploy/location_selector_test.go | 87 +++++++++++++++++++ internal/cmd/compute/deploy/nohttp_test.go | 2 +- 4 files changed, 122 insertions(+), 23 deletions(-) create mode 100644 internal/cmd/compute/deploy/location_selector_test.go diff --git a/internal/cmd/compute/deploy/deploy.go b/internal/cmd/compute/deploy/deploy.go index bac21477..6d08af3e 100644 --- a/internal/cmd/compute/deploy/deploy.go +++ b/internal/cmd/compute/deploy/deploy.go @@ -229,36 +229,48 @@ func runDeploy(cmd *cobra.Command, args []string, opts *options) error { } // deployFromFlags implements Path A: deploy a workload using CLI flags. -func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) error { - project := util.ProjectFromCmd(cmd) - if project == "" { - return fmt.Errorf("no project set — pass --project or run 'datumctl config set project '") - } - if opts.image == "" { - return fmt.Errorf("--image is required") - } - placementFlags := 0 - for _, set := range []bool{len(opts.locations) > 0, len(opts.cities) > 0, opts.locationSelector != ""} { - if set { - placementFlags++ +// resolveLocationSelector validates the three mutually exclusive ways a deploy +// can say where to run, and returns the selector they resolve to. --location +// names its locations outright and needs no selector, so a nil return with a +// nil error means "the locations were named". +func resolveLocationSelector(opts *options) (*metav1.LabelSelector, error) { + set := 0 + for _, given := range []bool{len(opts.locations) > 0, len(opts.cities) > 0, opts.locationSelector != ""} { + if given { + set++ } } - if placementFlags == 0 { - return fmt.Errorf("--location is required (e.g. --location=us-east-1,eu-west-1); or use --city to deploy to every location in a city, or --location-selector to select locations by topology") - } - if placementFlags > 1 { - return fmt.Errorf("--location, --city, and --location-selector are mutually exclusive") + switch { + case set == 0: + return nil, fmt.Errorf("--location is required (e.g. --location=us-east-1,eu-west-1); or use --city to deploy to every location in a city, or --location-selector to select locations by topology") + case set > 1: + return nil, fmt.Errorf("--location, --city, and --location-selector are mutually exclusive") } - var locationSelector *metav1.LabelSelector + if opts.locationSelector != "" { parsed, err := metav1.ParseToLabelSelector(opts.locationSelector) if err != nil { - return fmt.Errorf("invalid --location-selector %q: %w", opts.locationSelector, err) + return nil, fmt.Errorf("invalid --location-selector %q: %w", opts.locationSelector, err) } - locationSelector = parsed + return parsed, nil } if len(opts.cities) > 0 { - locationSelector = computev1alpha.CityCodeSelector(opts.cities) + return computev1alpha.CityCodeSelector(opts.cities), nil + } + return nil, nil +} + +func deployFromFlags(cmd *cobra.Command, workloadName string, opts *options) error { + project := util.ProjectFromCmd(cmd) + if project == "" { + return fmt.Errorf("no project set — pass --project or run 'datumctl config set project '") + } + if opts.image == "" { + return fmt.Errorf("--image is required") + } + locationSelector, err := resolveLocationSelector(opts) + if err != nil { + return err } instanceType := opts.instanceType if instanceType == "" { diff --git a/internal/cmd/compute/deploy/deploy_test.go b/internal/cmd/compute/deploy/deploy_test.go index 90f69a16..d0f20a4b 100644 --- a/internal/cmd/compute/deploy/deploy_test.go +++ b/internal/cmd/compute/deploy/deploy_test.go @@ -187,7 +187,7 @@ func TestDeclaredHTTPPort(t *testing.T) { }, { name: "the http port", - workload: workloadWithPorts(computev1alpha.NamedPort{Name: "http", Port: 8080, Protocol: &tcp}), + workload: workloadWithPorts(computev1alpha.NamedPort{Name: httpPortName, Port: 8080, Protocol: &tcp}), want: 8080, }, { diff --git a/internal/cmd/compute/deploy/location_selector_test.go b/internal/cmd/compute/deploy/location_selector_test.go new file mode 100644 index 00000000..8fdc48fb --- /dev/null +++ b/internal/cmd/compute/deploy/location_selector_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "strings" + "testing" +) + +// TestResolveLocationSelector pins the three mutually exclusive ways a deploy +// says where to run. This validation was lifted out of deployFromFlags to keep +// it under the complexity limit, so it needs its own coverage: nothing else +// exercises it without a live control plane behind the activation gate. +func TestResolveLocationSelector(t *testing.T) { + for _, tc := range []struct { + name string + opts options + wantErr string + wantNil bool + wantMatches map[string]string + }{{ + name: "no placement flag at all", + opts: options{}, + wantErr: "--location is required", + }, { + name: "location and city together", + opts: options{locations: []string{"us-east-1"}, cities: []string{"DFW"}}, + wantErr: "mutually exclusive", + }, { + name: "location and selector together", + opts: options{locations: []string{"us-east-1"}, locationSelector: "a=b"}, + wantErr: "mutually exclusive", + }, { + name: "all three together", + opts: options{locations: []string{"us-east-1"}, cities: []string{"DFW"}, locationSelector: "a=b"}, + wantErr: "mutually exclusive", + }, { + name: "named locations need no selector", + opts: options{locations: []string{"us-east-1", "eu-west-1"}}, + wantNil: true, + }, { + name: "cities become a city-code selector", + opts: options{cities: []string{"DFW"}}, + wantMatches: map[string]string{"topology.datum.net/city-code": "DFW"}, + }, { + name: "an explicit selector is parsed", + opts: options{locationSelector: "topology.datum.net/region=us-east-1"}, + wantMatches: map[string]string{"topology.datum.net/region": "us-east-1"}, + }, { + name: "an unparseable selector is reported, not ignored", + opts: options{locationSelector: "=="}, + wantErr: "invalid --location-selector", + }} { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveLocationSelector(&tc.opts) + + if tc.wantErr != "" { + if err == nil { + t.Fatalf("want an error containing %q, got selector %v", tc.wantErr, got) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error = %q, want it to contain %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if tc.wantNil { + if got != nil { + t.Errorf("selector = %v, want nil — named locations select nothing", got) + } + return + } + + if got == nil { + t.Fatal("selector is nil, want one") + } + for k, v := range tc.wantMatches { + if got.MatchLabels[k] != v { + t.Errorf("matchLabels[%q] = %q, want %q (got %v)", k, got.MatchLabels[k], v, got.MatchLabels) + } + } + }) + } +} diff --git a/internal/cmd/compute/deploy/nohttp_test.go b/internal/cmd/compute/deploy/nohttp_test.go index dfe0079b..04cc271e 100644 --- a/internal/cmd/compute/deploy/nohttp_test.go +++ b/internal/cmd/compute/deploy/nohttp_test.go @@ -396,7 +396,7 @@ func TestManifestDeployReportsTheDeadEnd(t *testing.T) { }) t.Run("a declared http port is not a dead end", func(t *testing.T) { - w := workloadWithPorts(computev1alpha.NamedPort{Name: "http", Port: 8080}) + w := workloadWithPorts(computev1alpha.NamedPort{Name: httpPortName, Port: 8080}) var out bytes.Buffer reportManifestReachability(&out, &w) From 97ec4d3330de57e7ac4dbfc7cd9dc49b6ca03858 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 9 Sep 2026 19:47:08 -0500 Subject: [PATCH 12/12] fix(cli): extract test constants flagged by goconst Co-Authored-By: Claude Opus 5 --- .../compute/deploy/location_selector_test.go | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/internal/cmd/compute/deploy/location_selector_test.go b/internal/cmd/compute/deploy/location_selector_test.go index 8fdc48fb..21cb652c 100644 --- a/internal/cmd/compute/deploy/location_selector_test.go +++ b/internal/cmd/compute/deploy/location_selector_test.go @@ -7,6 +7,12 @@ import ( "testing" ) +const ( + testCity = "DFW" + testLocation = "us-east-1" + errMutuallyExclusive = "mutually exclusive" +) + // TestResolveLocationSelector pins the three mutually exclusive ways a deploy // says where to run. This validation was lifted out of deployFromFlags to keep // it under the complexity limit, so it needs its own coverage: nothing else @@ -24,28 +30,28 @@ func TestResolveLocationSelector(t *testing.T) { wantErr: "--location is required", }, { name: "location and city together", - opts: options{locations: []string{"us-east-1"}, cities: []string{"DFW"}}, - wantErr: "mutually exclusive", + opts: options{locations: []string{testLocation}, cities: []string{testCity}}, + wantErr: errMutuallyExclusive, }, { name: "location and selector together", - opts: options{locations: []string{"us-east-1"}, locationSelector: "a=b"}, - wantErr: "mutually exclusive", + opts: options{locations: []string{testLocation}, locationSelector: "a=b"}, + wantErr: errMutuallyExclusive, }, { name: "all three together", - opts: options{locations: []string{"us-east-1"}, cities: []string{"DFW"}, locationSelector: "a=b"}, - wantErr: "mutually exclusive", + opts: options{locations: []string{testLocation}, cities: []string{testCity}, locationSelector: "a=b"}, + wantErr: errMutuallyExclusive, }, { name: "named locations need no selector", - opts: options{locations: []string{"us-east-1", "eu-west-1"}}, + opts: options{locations: []string{testLocation, "eu-west-1"}}, wantNil: true, }, { name: "cities become a city-code selector", - opts: options{cities: []string{"DFW"}}, - wantMatches: map[string]string{"topology.datum.net/city-code": "DFW"}, + opts: options{cities: []string{testCity}}, + wantMatches: map[string]string{"topology.datum.net/city-code": testCity}, }, { name: "an explicit selector is parsed", opts: options{locationSelector: "topology.datum.net/region=us-east-1"}, - wantMatches: map[string]string{"topology.datum.net/region": "us-east-1"}, + wantMatches: map[string]string{"topology.datum.net/region": testLocation}, }, { name: "an unparseable selector is reported, not ignored", opts: options{locationSelector: "=="},