Use canonical locations for compute placement

Taskfile.yaml

@@ -431,7 +431,7 @@ tasks:
   # ════════════════════════════════════════════════════════════════════════
 
   e2e:karmada:join-clusters:
-    desc: "Register POP cell clusters with Karmada and apply city-code labels"
+    desc: "Register POP cell clusters with Karmada and apply canonical location labels"
     cmds:
       # cluster.karmada.io is served by the karmada-aggregated-apiserver through
       # API aggregation, and can briefly return ServiceUnavailable after the pod
@@ -507,7 +507,7 @@ tasks:
           {{.CLUSTER_NAME}} \
           {{.INTERNAL_KUBECONFIG}}
       # ── Apply cluster labels ───────────────────────────────────────────
-      # city-code is what compute's federator places deployments by.
+      # location is the canonical identity compute's federator places deployments by.
       #
       # infra.datum.net/gateways=enabled is what NSO's propagation policy selects
       # on, and every cell in infra carries it. Without it that policy matches no
@@ -516,10 +516,10 @@ tasks:
       - |
         kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \
           label cluster {{.CLUSTER_NAME}} \
-          topology.datum.net/city-code={{.CITY_CODE}} \
+          topology.datum.net/location={{.CITY_CODE}} \
           infra.datum.net/gateways=enabled \
           --overwrite
-        echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}} and infra.datum.net/gateways=enabled"
+        echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/location={{.CITY_CODE}} and infra.datum.net/gateways=enabled"
 
   # ════════════════════════════════════════════════════════════════════════
   #  CRD installation

api/v1alpha/instance_types.go

@@ -5,6 +5,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha"
+	locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1"
 )
 
 // InstanceSpec defines the desired state of Instance
@@ -40,7 +41,7 @@ type InstanceSpec struct {
 	// The location which the instance has been scheduled to
 	//
 	// +kubebuilder:validation:Optional
-	Location *networkingv1alpha.LocationReference `json:"location,omitempty"`
+	Location *locationsv1alpha1.LocationReference `json:"location,omitempty"`
 
 	// Controller contains settings driven by the controller managing the instance.
 	//
@@ -887,11 +888,11 @@ const (
 	// waits until the platform resolves the conflict.
 	WorkloadDeploymentReasonAmbiguousServingLocation = "AmbiguousServingLocation"
 
-	// WorkloadDeploymentReasonCityCodeMismatch is set on
-	// WorkloadDeployment.Available when the deployment asks for one city and the
-	// cell serves another. It means the deployment was placed on the wrong cell,
+	// WorkloadDeploymentReasonLocationMismatch is set on
+	// WorkloadDeployment.Available when the deployment asks for one location and
+	// the cell serves another. It means the deployment was placed on the wrong cell,
 	// which is a platform fault rather than anything the user can correct.
-	WorkloadDeploymentReasonCityCodeMismatch = "CityCodeMismatch"
+	WorkloadDeploymentReasonLocationMismatch = "LocationMismatch"
 
 	// WorkloadDeploymentReasonNetworkProvisioning is set on WorkloadDeployment.Available
 	// while the network binding or subnet is still being provisioned.
@@ -923,6 +924,11 @@ const (
 	// WorkloadReasonNoAvailableDeployments is set on a placement's Available
 	// condition when no deployment in that placement is available.
 	WorkloadReasonNoAvailableDeployments = "NoAvailableDeployments"
+
+	// WorkloadReasonNoMatchingLocations is set on a placement's Available
+	// condition when none of the locations it names is Ready, or its selector
+	// matches no Ready location, so the placement has nowhere to run.
+	WorkloadReasonNoMatchingLocations = "NoMatchingLocations"
 )
 
 type InstanceTemplateSpec struct {

api/v1alpha/labels.go

@@ -14,9 +14,9 @@ const (
 	// that owns an Instance. Stamped at creation and kept current on updates.
 	WorkloadDeploymentNameLabel = LabelNamespace + "/workload-deployment-name"
 
-	// CityCodeLabel carries the city code of the WorkloadDeployment that owns
-	// an Instance, matching WorkloadDeploymentSpec.CityCode.
-	CityCodeLabel = LabelNamespace + "/city-code"
+	// LocationLabel carries the canonical location name of the
+	// WorkloadDeployment that owns an Instance.
+	LocationLabel = LabelNamespace + "/location"
 
 	// WorkloadNameLabel carries the name of the Workload that an Instance
 	// ultimately belongs to, sourced from WorkloadDeploymentSpec.WorkloadRef.Name.

api/v1alpha/workload_placement.go

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package v1alpha
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1"
+)
+
+// CityCodeSelector returns the location selector that places at every
+// location in the given cities: an equality for one city, an In expression
+// for several. It is what the deprecated cityCodes field and the CLI's --city
+// flag both stand for, so a placement written either way resolves the same.
+func CityCodeSelector(cityCodes []string) *metav1.LabelSelector {
+	if len(cityCodes) == 1 {
+		return &metav1.LabelSelector{
+			MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: cityCodes[0]},
+		}
+	}
+	return &metav1.LabelSelector{
+		MatchExpressions: []metav1.LabelSelectorRequirement{{
+			Key:      locationsv1alpha1.TopologyCityCodeKey,
+			Operator: metav1.LabelSelectorOpIn,
+			Values:   append([]string(nil), cityCodes...),
+		}},
+	}
+}
+
+// MigrateCityCodes rewrites a placement that still names city codes into the
+// equivalent locationSelector and clears the deprecated field. It reports
+// whether anything changed. A placement that already names locations or a
+// selector is left alone, including one that also carries city codes, which
+// validation rejects rather than guessing which the author meant.
+func (p *WorkloadPlacement) MigrateCityCodes() bool {
+	if len(p.CityCodes) == 0 || len(p.Locations) > 0 || p.LocationSelector != nil {
+		return false
+	}
+	p.LocationSelector = CityCodeSelector(p.CityCodes)
+	p.CityCodes = nil
+	return true
+}
+
+// MigrateCityCodes rewrites every placement that still names city codes. It
+// reports whether the spec changed, so a caller that persisted the workload
+// knows to write it back.
+func (w *Workload) MigrateCityCodes() bool {
+	migrated := false
+	for i := range w.Spec.Placements {
+		if w.Spec.Placements[i].MigrateCityCodes() {
+			migrated = true
+		}
+	}
+	return migrated
+}

api/v1alpha/workload_placement_test.go

@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package v1alpha
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1"
+)
+
+const (
+	placementTestCityDFW = "DFW"
+	placementTestCityIAD = "IAD"
+)
+
+func TestCityCodeSelector(t *testing.T) {
+	t.Parallel()
+
+	assert.Equal(t, &metav1.LabelSelector{
+		MatchLabels: map[string]string{locationsv1alpha1.TopologyCityCodeKey: placementTestCityDFW},
+	}, CityCodeSelector([]string{placementTestCityDFW}), "one city is a plain equality")
+
+	assert.Equal(t, &metav1.LabelSelector{
+		MatchExpressions: []metav1.LabelSelectorRequirement{{
+			Key:      locationsv1alpha1.TopologyCityCodeKey,
+			Operator: metav1.LabelSelectorOpIn,
+			Values:   []string{placementTestCityDFW, placementTestCityIAD},
+		}},
+	}, CityCodeSelector([]string{placementTestCityDFW, placementTestCityIAD}), "several cities are an In expression")
+}
+
+// TestMigrateCityCodes covers the shim for workloads stored before placement
+// moved to locations: a placement that only names city codes becomes the
+// equivalent selector, and anything else is left for validation to judge.
+func TestMigrateCityCodes(t *testing.T) {
+	t.Parallel()
+
+	t.Run("city codes alone are rewritten", func(t *testing.T) {
+		t.Parallel()
+		w := &Workload{Spec: WorkloadSpec{Placements: []WorkloadPlacement{
+			{Name: "a", CityCodes: []string{placementTestCityDFW, placementTestCityIAD}},
+			{Name: "b", Locations: []locationsv1alpha1.LocationReference{{Name: "us-east-1"}}},
+		}}}
+
+		require.True(t, w.MigrateCityCodes())
+		assert.Nil(t, w.Spec.Placements[0].CityCodes, "the deprecated field is cleared")
+		assert.Equal(t, CityCodeSelector([]string{placementTestCityDFW, placementTestCityIAD}), w.Spec.Placements[0].LocationSelector)
+		assert.Empty(t, w.Spec.Placements[1].LocationSelector, "a placement naming locations is untouched")
+		assert.False(t, w.MigrateCityCodes(), "a second pass finds nothing to do")
+	})
+
+	t.Run("city codes beside locations or a selector are left for validation", func(t *testing.T) {
+		t.Parallel()
+		w := &Workload{Spec: WorkloadSpec{Placements: []WorkloadPlacement{
+			{Name: "a", CityCodes: []string{placementTestCityDFW}, Locations: []locationsv1alpha1.LocationReference{{Name: "us-east-1"}}},
+			{Name: "b", CityCodes: []string{placementTestCityDFW}, LocationSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"k": "v"}}},
+		}}}
+
+		assert.False(t, w.MigrateCityCodes())
+		assert.Equal(t, []string{placementTestCityDFW}, w.Spec.Placements[0].CityCodes)
+		assert.Equal(t, []string{placementTestCityDFW}, w.Spec.Placements[1].CityCodes)
+	})
+}

api/v1alpha/workload_types.go

@@ -1,6 +1,7 @@
 package v1alpha
 
 import (
+	locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1"
 	k8scorev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -130,15 +131,41 @@ type WorkloadList struct {
 	Items           []Workload `json:"items"`
 }
 
+// +kubebuilder:validation:XValidation:message="exactly one of locations, locationSelector, or cityCodes must be set",rule="(has(self.locations) ? 1 : 0) + (has(self.locationSelector) ? 1 : 0) + (has(self.cityCodes) ? 1 : 0) == 1"
 type WorkloadPlacement struct {
 	// The name of the placement
 	//
 	// +kubebuilder:validation:Required
 	Name string `json:"name"`
 
-	// A list of city codes that define where the instances should be deployed.
+	// The locations where the instances should be deployed, by name. Use this
+	// to pin a placement to specific locations. Exactly one of locations or
+	// locationSelector must be set.
 	//
-	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:MinItems=1
+	Locations []locationsv1alpha1.LocationReference `json:"locations,omitempty"`
+
+	// A selector over the topology of the locations available to the project,
+	// such as topology.datum.net/city-code or topology.datum.net/region. Every
+	// Ready location whose topology matches receives a deployment, and the set
+	// is re-evaluated as locations are added, removed, or change readiness. An
+	// empty selector is rejected rather than treated as matching every
+	// location. Exactly one of locations or locationSelector must be set.
+	//
+	// +kubebuilder:validation:Optional
+	LocationSelector *metav1.LabelSelector `json:"locationSelector,omitempty"`
+
+	// The city codes this placement was written against before placement
+	// moved to locations. This field is deprecated and kept only so workloads
+	// stored before that change keep running: admission and the workload
+	// controller rewrite it into a locationSelector on
+	// topology.datum.net/city-code, which places at every location in those
+	// cities, and clear it. New workloads set locations or locationSelector
+	// instead.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:MinItems=1
 	CityCodes []string `json:"cityCodes,omitempty"`
 
 	// Scale settings such as minimum and maximum replica counts.
@@ -151,6 +178,10 @@ type WorkloadPlacementStatus struct {
 	// The name of the placement
 	Name string `json:"name"`
 
+	// The locations the placement currently resolves to: the Ready locations
+	// it names, or every Ready location its selector matches.
+	Locations []locationsv1alpha1.LocationReference `json:"locations,omitempty"`
+
 	// Represents the observations of a placement's current state.
 	// Known condition types are: "Available", "Progressing"
 	Conditions []metav1.Condition `json:"conditions,omitempty"`

api/v1alpha/workloaddeployment_types.go

@@ -1,9 +1,8 @@
 package v1alpha
 
 import (
+	locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha"
 )
 
 // WorkloadDeploymentSpec defines the desired state of WorkloadDeployment
@@ -18,11 +17,10 @@ type WorkloadDeploymentSpec struct {
 	// +kubebuilder:validation:Required
 	PlacementName string `json:"placementName"`
 
-	// TODO(jreese) think through how to structure this a bit better for when
-	// deployments can be scheduled in ways other than just a city code.
+	// The location where this deployment runs.
 	//
 	// +kubebuilder:validation:Required
-	CityCode string `json:"cityCode"`
+	LocationRef locationsv1alpha1.LocationReference `json:"locationRef"`
 
 	// Defines settings for each instance.
 	//
@@ -43,11 +41,6 @@ type WorkloadDeploymentSpec struct {
 
 // WorkloadDeploymentStatus defines the observed state of WorkloadDeployment
 type WorkloadDeploymentStatus struct {
-	// The location which the deployment has been scheduled to
-	//
-	// +kubebuilder:validation:Optional
-	Location *networkingv1alpha.LocationReference `json:"location,omitempty"`
-
 	// Represents the observations of a deployment's current state.
 	// Known condition types are: "Available", "Progressing"
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
@@ -112,8 +105,7 @@ const (
 // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas`
 // +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas`
 // +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.updatedReplicas`
-// +kubebuilder:printcolumn:name="Location Namespace",type=string,JSONPath=`.status.location.namespace`,priority=1
-// +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name`,priority=1
+// +kubebuilder:printcolumn:name="Location",type=string,JSONPath=`.spec.locationRef.name`
 type WorkloadDeployment struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

cmd/main.go

@@ -89,6 +89,7 @@ func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 
 	utilruntime.Must(config.AddToScheme(scheme))
+	utilruntime.Must(locationsv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(config.RegisterDefaults(scheme))
 	utilruntime.Must(computev1alpha.AddToScheme(scheme))
 	utilruntime.Must(networkingv1alpha.AddToScheme(scheme))