diff --git a/internal/controller/referenceddata_controller.go b/internal/controller/referenceddata_controller.go index 9593bc8a..a113b929 100644 --- a/internal/controller/referenceddata_controller.go +++ b/internal/controller/referenceddata_controller.go @@ -639,7 +639,20 @@ func (r *ReferencedDataController) resolveAndValidateSources( // isOptionalRef returns true when the given ObjectRef corresponds to a source // that was marked optional=true anywhere in the instance template spec. // It checks all volume mounts and env/envFrom sources that match (kind, name). +// +// Image pull credentials are never optional: without them the image cannot be +// pulled at all, so silently skipping a missing or oversized one would trade a +// clear condition on the WorkloadDeployment for an opaque image-pull failure at +// the cell. A Secret used as a pull credential therefore stays required even if +// the same Secret is also referenced optionally somewhere else in the template. func isOptionalRef(ref referenceddata.ObjectRef, tmpl computev1alpha.InstanceTemplateSpec) bool { + if sb := tmpl.Spec.Runtime.Sandbox; sb != nil && ref.Kind == kindSecret { + for _, ps := range sb.ImagePullSecrets { + if ps.Name == ref.Name { + return false + } + } + } if isOptionalInVolumes(ref, tmpl.Spec.Volumes) { return true } diff --git a/internal/controller/referenceddata_controller_test.go b/internal/controller/referenceddata_controller_test.go index c24a46e9..f8594369 100644 --- a/internal/controller/referenceddata_controller_test.go +++ b/internal/controller/referenceddata_controller_test.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -783,7 +784,6 @@ func newRDControllerFederated( t *testing.T, projectCl client.Client, hubCl client.Client, - reader referenceddata.ProjectConfigSecretReader, ) (*ReferencedDataController, string) { t.Helper() clusterName := rdTestClusterName @@ -797,7 +797,6 @@ func newRDControllerFederated( c := &ReferencedDataController{ mgr: mgr, opts: ReferencedDataControllerOptions{ - Reader: reader, FederationClient: hubCl, }, } @@ -841,7 +840,7 @@ func TestReferencedData_Federated_CompanionWrittenToHub(t *testing.T) { require.NoError(t, computev1alpha.AddToScheme(hubScheme)) hubCl := fake.NewClientBuilder().WithScheme(hubScheme).Build() - c, clusterName := newRDControllerFederated(t, projectCl, hubCl, nil) + c, clusterName := newRDControllerFederated(t, projectCl, hubCl) // First reconcile: stamps finalizer. reconcileWD(t, c, clusterName, projNS, "wd-fed-1") @@ -1565,3 +1564,250 @@ func (s *stubReader) GetSecret(ctx context.Context, projectID, namespace, name s } return nil, fmt.Errorf("%w: Secret %s", referenceddata.ErrSourceNotFound, name) } + +// TestIsOptionalRefImagePullSecret pins that a Secret used as an image pull +// credential is never treated as optional. Optional sources are silently +// skipped when missing or oversized; for a pull credential that would swap a +// clear condition on the WorkloadDeployment for an opaque image-pull failure at +// the cell, so the required-ness must win even when the same Secret is also +// referenced with optional=true elsewhere in the template. +func TestIsOptionalRefImagePullSecret(t *testing.T) { + const credName = "registry-creds" + + tmpl := computev1alpha.InstanceTemplateSpec{ + Spec: computev1alpha.InstanceSpec{ + Runtime: computev1alpha.InstanceRuntimeSpec{ + Sandbox: &computev1alpha.SandboxRuntime{ + ImagePullSecrets: []computev1alpha.LocalSecretReference{{Name: credName}}, + Containers: []computev1alpha.SandboxContainer{ + { + Name: "app", + Image: "registry.example.com/private/app:v1", + EnvFrom: []computev1alpha.EnvFromSource{ + // Same Secret, referenced optionally. + {SecretRef: &computev1alpha.SecretEnvSource{Name: credName, Optional: ptr.To(true)}}, + {SecretRef: &computev1alpha.SecretEnvSource{Name: "other-secret", Optional: ptr.To(true)}}, + }, + }, + }, + }, + }, + }, + } + + assert.False(t, isOptionalRef(referenceddata.ObjectRef{Kind: kindSecret, Name: credName, Namespace: "proj"}, tmpl), + "an image pull credential must never be optional") + assert.True(t, isOptionalRef(referenceddata.ObjectRef{Kind: kindSecret, Name: "other-secret", Namespace: "proj"}, tmpl), + "unrelated optional secrets keep their optional semantics") +} + +// ─── Image pull credentials federate like any other referenced Secret ──────── + +// templateWithImagePullSecret returns an InstanceTemplateSpec whose sandbox +// pulls from a private registry using the named Secret. Shape mirrors +// test/e2e/referenced-data-mounts/workload-deployment.yaml. +func templateWithImagePullSecret(pullSecretNames ...string) computev1alpha.InstanceTemplateSpec { + refs := make([]computev1alpha.LocalSecretReference, 0, len(pullSecretNames)) + for _, n := range pullSecretNames { + refs = append(refs, computev1alpha.LocalSecretReference{Name: n}) + } + return computev1alpha.InstanceTemplateSpec{ + Spec: computev1alpha.InstanceSpec{ + Runtime: computev1alpha.InstanceRuntimeSpec{ + Sandbox: &computev1alpha.SandboxRuntime{ + ImagePullSecrets: refs, + Containers: []computev1alpha.SandboxContainer{ + {Name: "app", Image: "registry.example.com/private/app:v1"}, + }, + }, + }, + }, + } +} + +// makeDockerConfigSecret returns a Secret shaped like one produced by +// `kubectl create secret docker-registry`. +func makeDockerConfigSecret(name string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: testProjNS, Name: name}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte( + `{"auths":{"registry.example.com":{"username":"robot","password":"s3cr3t","auth":"cm9ib3Q6czNjcjN0"}}}`), + }, + } +} + +// TestReferencedData_Federated_ImagePullSecretWrittenToHub asserts the add path +// end to end on the hub: a Secret referenced ONLY by imagePullSecrets is copied +// into the downstream ns-{project-uid} namespace, carries the referenced-data +// label that the PropagationPolicy selects on (which is what carries it to the +// cell), preserves the dockerconfigjson type and payload, and is listed in the +// expected-referenced-data annotation so the Instance gate waits for it. +func TestReferencedData_Federated_ImagePullSecretWrittenToHub(t *testing.T) { + projNS := testProjNS + const wdName = "wd-pull-1" + const pullSecretName = "registry-creds" + companionName := referenceddata.CompanionName(kindSecret, pullSecretName) + + projNSObj := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: projNS, UID: testProjNSUID}} + srcSecret := makeDockerConfigSecret(pullSecretName) + wd := makeWD(projNS, wdName, templateWithImagePullSecret(pullSecretName)) + + s := rdTestScheme(t) + require.NoError(t, corev1.AddToScheme(s)) + + projectCl := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(projNSObj, srcSecret, wd). + WithStatusSubresource(wd). + Build() + + hubScheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(hubScheme)) + require.NoError(t, computev1alpha.AddToScheme(hubScheme)) + hubCl := fake.NewClientBuilder().WithScheme(hubScheme).Build() + + c, clusterName := newRDControllerFederated(t, projectCl, hubCl) + + reconcileWD(t, c, clusterName, projNS, wdName) // stamps finalizer + reconcileWD(t, c, clusterName, projNS, wdName) // materialises companion + + var hubSecret corev1.Secret + require.NoError(t, hubCl.Get(context.Background(), + types.NamespacedName{Namespace: testKarmadaNSStr, Name: companionName}, &hubSecret), + "pull credential must be copied to the hub downstream namespace") + assert.Equal(t, corev1.SecretTypeDockerConfigJson, hubSecret.Type, "companion must preserve the docker config Secret type") + assert.Equal(t, srcSecret.Data[corev1.DockerConfigJsonKey], hubSecret.Data[corev1.DockerConfigJsonKey]) + assert.Equal(t, computev1alpha.ReferencedDataLabelValue, hubSecret.Labels[computev1alpha.ReferencedDataLabel], + "companion must carry the referenced-data label the PropagationPolicy selects on to reach the cell") + + wd = getWD(t, projectCl, types.NamespacedName{Namespace: projNS, Name: wdName}) + assert.Equal(t, + []string{referenceddata.CompanionToken(kindSecret, companionName)}, + decodeExpectedAnnotation(t, wd), + "the Instance gate must wait for the pull credential") + + cond := apimeta.FindStatusCondition(wd.Status.Conditions, computev1alpha.ReferencedDataReady) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionTrue, cond.Status) +} + +// TestReferencedData_Federated_ImagePullSecretRemoved_CompanionReleased asserts +// the remove path: dropping one entry from imagePullSecrets releases only that +// companion from the hub, leaving the credential still referenced by the WD in +// place. +func TestReferencedData_Federated_ImagePullSecretRemoved_CompanionReleased(t *testing.T) { + projNS := testProjNS + const wdName = "wd-pull-2" + const keptSecret = "registry-creds" + const droppedSecret = "legacy-registry-creds" + + projNSObj := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: projNS, UID: testProjNSUID}} + wd := makeWD(projNS, wdName, templateWithImagePullSecret(keptSecret, droppedSecret)) + + s := rdTestScheme(t) + require.NoError(t, corev1.AddToScheme(s)) + + projectCl := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(projNSObj, makeDockerConfigSecret(keptSecret), makeDockerConfigSecret(droppedSecret), wd). + WithStatusSubresource(wd). + Build() + + hubScheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(hubScheme)) + require.NoError(t, computev1alpha.AddToScheme(hubScheme)) + hubCl := fake.NewClientBuilder().WithScheme(hubScheme).Build() + + c, clusterName := newRDControllerFederated(t, projectCl, hubCl) + + reconcileWD(t, c, clusterName, projNS, wdName) + reconcileWD(t, c, clusterName, projNS, wdName) + + for _, name := range []string{keptSecret, droppedSecret} { + var hubSecret corev1.Secret + require.NoError(t, hubCl.Get(context.Background(), + types.NamespacedName{Namespace: testKarmadaNSStr, Name: name}, &hubSecret), "companion %q must exist before removal", name) + } + + // Drop one pull credential from the template. + wd = getWD(t, projectCl, types.NamespacedName{Namespace: projNS, Name: wdName}) + wd.Spec.Template.Spec.Runtime.Sandbox.ImagePullSecrets = []computev1alpha.LocalSecretReference{{Name: keptSecret}} + require.NoError(t, projectCl.Update(context.Background(), wd)) + + reconcileWD(t, c, clusterName, projNS, wdName) + + var gone corev1.Secret + assert.Error(t, hubCl.Get(context.Background(), + types.NamespacedName{Namespace: testKarmadaNSStr, Name: droppedSecret}, &gone), + "de-referenced pull credential must be removed from the hub (and therefore from the cell)") + + var kept corev1.Secret + require.NoError(t, hubCl.Get(context.Background(), + types.NamespacedName{Namespace: testKarmadaNSStr, Name: keptSecret}, &kept), + "still-referenced pull credential must survive") + + wd = getWD(t, projectCl, types.NamespacedName{Namespace: projNS, Name: wdName}) + assert.Equal(t, + []string{referenceddata.CompanionToken(kindSecret, keptSecret)}, + decodeExpectedAnnotation(t, wd)) +} + +// TestReferencedData_Federated_ImagePullSecretAlsoMounted_NotReleased pins the +// dedupe invariant across reference sources: a Secret used BOTH as a pull +// credential and as a volume must survive when only the pull-credential +// reference is dropped. The mount would otherwise break. +func TestReferencedData_Federated_ImagePullSecretAlsoMounted_NotReleased(t *testing.T) { + projNS := testProjNS + const wdName = "wd-pull-3" + const sharedSecret = "registry-creds" + + projNSObj := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: projNS, UID: testProjNSUID}} + + tmpl := templateWithImagePullSecret(sharedSecret) + tmpl.Spec.Volumes = []computev1alpha.InstanceVolume{ + { + Name: "creds-vol", + VolumeSource: computev1alpha.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: sharedSecret}}, + }, + } + wd := makeWD(projNS, wdName, tmpl) + + s := rdTestScheme(t) + require.NoError(t, corev1.AddToScheme(s)) + + projectCl := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(projNSObj, makeDockerConfigSecret(sharedSecret), wd). + WithStatusSubresource(wd). + Build() + + hubScheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(hubScheme)) + require.NoError(t, computev1alpha.AddToScheme(hubScheme)) + hubCl := fake.NewClientBuilder().WithScheme(hubScheme).Build() + + c, clusterName := newRDControllerFederated(t, projectCl, hubCl) + + reconcileWD(t, c, clusterName, projNS, wdName) + reconcileWD(t, c, clusterName, projNS, wdName) + + wd = getWD(t, projectCl, types.NamespacedName{Namespace: projNS, Name: wdName}) + assert.Len(t, decodeExpectedAnnotation(t, wd), 1, "a Secret referenced twice must federate exactly once") + + // Drop only the pull-credential reference; the volume still mounts it. + wd.Spec.Template.Spec.Runtime.Sandbox.ImagePullSecrets = nil + require.NoError(t, projectCl.Update(context.Background(), wd)) + + reconcileWD(t, c, clusterName, projNS, wdName) + + var stillThere corev1.Secret + require.NoError(t, hubCl.Get(context.Background(), + types.NamespacedName{Namespace: testKarmadaNSStr, Name: sharedSecret}, &stillThere), + "companion must survive: the volume mount still references it") + + refs, err := decodeRefCount(stillThere.Annotations) + require.NoError(t, err) + assert.Contains(t, refs, types.NamespacedName{Namespace: projNS, Name: wdName}.String()) +} diff --git a/internal/referenceddata/collector.go b/internal/referenceddata/collector.go index 2975c82b..985e4f0c 100644 --- a/internal/referenceddata/collector.go +++ b/internal/referenceddata/collector.go @@ -14,6 +14,7 @@ import ( // - container env.ValueFrom.ConfigMapKeyRef / SecretKeyRef // - container envFrom[].configMapRef / secretRef // - spec.volumes[].configMap and spec.volumes[].secret +// - runtime.sandbox.imagePullSecrets[] // // The namespace field on every returned ObjectRef is set to the provided // namespace (the Workload's namespace). References are always same-namespace. @@ -37,8 +38,16 @@ func CollectFromTemplate(namespace string, template computev1alpha.InstanceTempl }) } - // Collect from sandbox containers. + // Collect from the sandbox runtime. if sb := template.Spec.Runtime.Sandbox; sb != nil { + // Image pull credentials. These are plain Secrets in the Workload's + // namespace, so they federate through the same companion machinery as + // every other referenced Secret — the credential has to exist in the cell + // namespace before the runtime can authenticate to a private registry. + for _, ps := range sb.ImagePullSecrets { + add("Secret", ps.Name) + } + for _, c := range sb.Containers { // env[].valueFrom for _, e := range c.Env { diff --git a/internal/referenceddata/collector_test.go b/internal/referenceddata/collector_test.go index 9a2fb1a3..fd18bc57 100644 --- a/internal/referenceddata/collector_test.go +++ b/internal/referenceddata/collector_test.go @@ -16,6 +16,10 @@ const ( testEnvConfigMap = "app-config" testSharedCfg = "shared-cfg" testCfgRef = "cfg" + testPullSecret = "registry-creds" + testPrivateImage = "registry.example.com/private/app:v1" + testAppContainer = "app" + testZRegistry = "z-registry" ) func TestCollectFromTemplate(t *testing.T) { @@ -199,6 +203,99 @@ func TestCollectFromTemplate(t *testing.T) { // No refs collected — the invalid entry is skipped entirely. want: nil, }, + // Shape mirrors test/e2e/referenced-data-mounts/workload-deployment.yaml + // with a pull credential added: a real template references a ConfigMap by + // volume, a Secret by env, and a registry credential by imagePullSecrets. + "imagePullSecrets alongside env and volume sources": { + template: makeTemplate(func(t *computev1alpha.InstanceTemplateSpec) { + t.Spec.Runtime.Sandbox = &computev1alpha.SandboxRuntime{ + ImagePullSecrets: []computev1alpha.LocalSecretReference{{Name: testPullSecret}}, + Containers: []computev1alpha.SandboxContainer{ + { + Name: testAppContainer, + Image: testPrivateImage, + Env: []corev1.EnvVar{ + { + Name: "DB_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "app-secret"}, + Key: "db.password", + }, + }, + }, + }, + VolumeAttachments: []computev1alpha.VolumeAttachment{ + {Name: "config-vol", MountPath: ptr.To("/etc/config")}, + }, + }, + }, + } + t.Spec.Volumes = []computev1alpha.InstanceVolume{ + { + Name: "config-vol", + VolumeSource: computev1alpha.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: testEnvConfigMap}, + }, + }, + }, + } + }), + want: ReferencedSet{ + {Kind: testKindConfigMap, Name: testEnvConfigMap, Namespace: ns}, + {Kind: testKindSecret, Name: "app-secret", Namespace: ns}, + {Kind: testKindSecret, Name: "registry-creds", Namespace: ns}, + }, + }, + "multiple imagePullSecrets deduplicated and sorted": { + template: makeTemplate(func(t *computev1alpha.InstanceTemplateSpec) { + t.Spec.Runtime.Sandbox = &computev1alpha.SandboxRuntime{ + ImagePullSecrets: []computev1alpha.LocalSecretReference{ + {Name: testZRegistry}, + {Name: "a-registry"}, + {Name: testZRegistry}, + }, + Containers: []computev1alpha.SandboxContainer{ + {Name: testAppContainer, Image: testPrivateImage}, + }, + } + }), + want: ReferencedSet{ + {Kind: testKindSecret, Name: "a-registry", Namespace: ns}, + {Kind: testKindSecret, Name: testZRegistry, Namespace: ns}, + }, + }, + // The same Secret used both as a pull credential and as an env source + // must federate exactly once. + "imagePullSecret shared with env secret collapses to one ref": { + template: makeTemplate(func(t *computev1alpha.InstanceTemplateSpec) { + t.Spec.Runtime.Sandbox = &computev1alpha.SandboxRuntime{ + ImagePullSecrets: []computev1alpha.LocalSecretReference{{Name: testNameDBCreds}}, + Containers: []computev1alpha.SandboxContainer{ + { + Name: "app", + Image: testPrivateImage, + EnvFrom: []computev1alpha.EnvFromSource{{SecretRef: &computev1alpha.SecretEnvSource{Name: testNameDBCreds}}}, + }, + }, + } + }), + want: ReferencedSet{ + {Kind: testKindSecret, Name: testNameDBCreds, Namespace: ns}, + }, + }, + "imagePullSecret with empty name is skipped": { + template: makeTemplate(func(t *computev1alpha.InstanceTemplateSpec) { + t.Spec.Runtime.Sandbox = &computev1alpha.SandboxRuntime{ + ImagePullSecrets: []computev1alpha.LocalSecretReference{{Name: ""}}, + Containers: []computev1alpha.SandboxContainer{ + {Name: testAppContainer, Image: testContainerImage}, + }, + } + }), + want: nil, + }, "mixed sources sorted configmap-first then secret": { template: makeTemplate(func(t *computev1alpha.InstanceTemplateSpec) { t.Spec.Runtime.Sandbox = &computev1alpha.SandboxRuntime{ @@ -268,6 +365,20 @@ func TestTemplateReferencesData(t *testing.T) { }), want: true, }, + // A template whose only referenced data is a pull credential must still + // stamp the ReferencedData scheduling gate — otherwise the Instance is + // admitted to the cell before the credential lands there. + "has only an image pull secret": { + template: makeTemplate(func(t *computev1alpha.InstanceTemplateSpec) { + t.Spec.Runtime.Sandbox = &computev1alpha.SandboxRuntime{ + ImagePullSecrets: []computev1alpha.LocalSecretReference{{Name: testPullSecret}}, + Containers: []computev1alpha.SandboxContainer{ + {Name: testAppContainer, Image: testPrivateImage}, + }, + } + }), + want: true, + }, "has secret ref": { template: makeTemplate(func(t *computev1alpha.InstanceTemplateSpec) { t.Spec.Runtime.Sandbox = &computev1alpha.SandboxRuntime{ diff --git a/test/e2e/referenced-data-pull-secret/chainsaw-test.yaml b/test/e2e/referenced-data-pull-secret/chainsaw-test.yaml new file mode 100644 index 00000000..77805dca --- /dev/null +++ b/test/e2e/referenced-data-pull-secret/chainsaw-test.yaml @@ -0,0 +1,261 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: referenced-data-pull-secret +spec: + description: | + Validates that an image pull credential referenced by + spec.template.spec.runtime.sandbox.imagePullSecrets is federated to the cell + exactly like a ConfigMap or Secret referenced by env/envFrom/volumes, and is + reclaimed when the reference is dropped. + + Without this, a user who wires up a pull secret gets an Instance that cannot + pull its image: the credential never leaves the project control plane. + + [Hop 1] Source Secret (dockerconfigjson) + ConfigMap created in the project + namespace. + [Hop 2] WorkloadDeployment referencing the credential is created; the + ReferencedDataController materialises a companion Secret in + ns-{project-uid} on the Karmada hub and lists it in the + expected-referenced-data annotation. + [Hop 3] Karmada propagates the companion to pop-dfw via the city-dfw + PropagationPolicy Secret selector (label-based, source-agnostic). + [Hop 4] The Instance's ReferencedData scheduling gate clears only once the + credential is present on the cell. + [Hop 5] Dropping the imagePullSecrets entry releases the companion from the + hub and the cell, while the still-referenced ConfigMap companion + survives. + + Scope: cross-plane DELIVERY and RECLAIM of the credential. Mapping the + credential onto the downstream Instance is the provider layer and is not + asserted here. + + template: true + + steps: + + # ─── Hop 1: create source data ───────────────────────────────────────────── + + - name: create-source-data + description: Create the source pull credential and ConfigMap in the project namespace. + try: + - apply: + file: source-pull-secret.yaml + - apply: + file: source-configmap.yaml + + # ─── Hop 2: create WD; assert companion materialisation on the hub ───────── + + - name: create-workload-deployment + description: Create the WorkloadDeployment referencing the pull credential. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-pull-credential-companion-on-hub + description: | + Assert the pull credential was copied into ns-{project-uid} on the Karmada + hub with the referenced-data label the PropagationPolicy selects on. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + type: kubernetes.io/dockerconfigjson + metadata: + namespace: ($companionNS) + name: registry-creds + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-wd-annotation-lists-pull-credential + description: | + Assert the expected-referenced-data annotation lists the pull credential, + so the Instance gate waits for it to arrive at the cell. + try: + - script: + timeout: 60s + content: | + ANNO=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get workloaddeployment test-pullsecret-wd \ + --namespace "$NAMESPACE" \ + -o jsonpath='{.metadata.annotations.compute\.datumapis\.com/expected-referenced-data}') + echo "annotation: $ANNO" + echo "$ANNO" | grep -q 'Secret/registry-creds' || { + echo "ERROR: pull credential missing from expected-referenced-data: '$ANNO'" + exit 1 + } + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($namespace) + name: test-pullsecret-wd + (status.conditions[?type == 'ReferencedDataReady'] | [0]): + status: "True" + reason: "Ready" + + # ─── Hop 3: assert the credential reached the cell ───────────────────────── + + - name: assert-pull-credential-on-cell + description: Assert Karmada propagated the pull credential to pop-dfw. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + type: kubernetes.io/dockerconfigjson + metadata: + namespace: ($downstreamNS) + name: registry-creds + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($downstreamNS) + name: pullsecret-app-config + labels: + compute.datumapis.com/referenced-data: "true" + + # ─── Hop 4: assert the Instance gate cleared ─────────────────────────────── + + - name: assert-referenced-data-gate-cleared + description: | + Assert the ReferencedData scheduling gate cleared, which only happens once + every expected companion — including the pull credential — is on the cell. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-pullsecret-wd-0 + (status.conditions[?type == 'ReferencedDataReady'] | [0]): + status: "True" + reason: "Ready" + - script: + timeout: 60s + content: | + DOWNSTREAM_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + GATES=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + get instance test-pullsecret-wd-0 \ + --namespace "$DOWNSTREAM_NS" \ + -o jsonpath='{.spec.schedulingGates[*].name}') + if echo "$GATES" | grep -qw "ReferencedData"; then + echo "ERROR: ReferencedData gate still present in schedulingGates: '$GATES'" + exit 1 + fi + echo "ReferencedData gate cleared. Remaining gates: '$GATES'" + + # ─── Hop 5: drop the reference; assert reclaim on hub and cell ───────────── + + - name: remove-pull-credential-reference + description: | + Remove the imagePullSecrets entry from the WorkloadDeployment, keeping the + ConfigMap volume reference in place. + try: + - apply: + file: workload-deployment-no-pull-secret.yaml + + - name: assert-pull-credential-reclaimed-from-hub + description: | + Assert the de-referenced credential companion is deleted from the hub while + the still-referenced ConfigMap companion survives. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - error: + timeout: 90s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($companionNS) + name: registry-creds + - assert: + timeout: 30s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($companionNS) + name: pullsecret-app-config + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-pull-credential-reclaimed-from-cell + description: Assert Karmada removed the credential from pop-dfw as well. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - error: + timeout: 90s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($downstreamNS) + name: registry-creds + - assert: + timeout: 30s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($downstreamNS) + name: pullsecret-app-config + labels: + compute.datumapis.com/referenced-data: "true" diff --git a/test/e2e/referenced-data-pull-secret/source-configmap.yaml b/test/e2e/referenced-data-pull-secret/source-configmap.yaml new file mode 100644 index 00000000..5416449f --- /dev/null +++ b/test/e2e/referenced-data-pull-secret/source-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pullsecret-app-config + # namespace injected by Chainsaw from ($namespace) +data: + app.conf: | + mode=test diff --git a/test/e2e/referenced-data-pull-secret/source-pull-secret.yaml b/test/e2e/referenced-data-pull-secret/source-pull-secret.yaml new file mode 100644 index 00000000..970a8ed2 --- /dev/null +++ b/test/e2e/referenced-data-pull-secret/source-pull-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: registry-creds + # namespace injected by Chainsaw from ($namespace) +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + {"auths":{"registry.example.com":{"username":"robot","password":"s3cr3t","auth":"cm9ib3Q6czNjcjN0"}}} diff --git a/test/e2e/referenced-data-pull-secret/workload-deployment-no-pull-secret.yaml b/test/e2e/referenced-data-pull-secret/workload-deployment-no-pull-secret.yaml new file mode 100644 index 00000000..19c6eed1 --- /dev/null +++ b/test/e2e/referenced-data-pull-secret/workload-deployment-no-pull-secret.yaml @@ -0,0 +1,35 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-pullsecret-wd + # namespace injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000004" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + # Explicit empty list, not an omitted field: chainsaw applies a merge + # patch, so an absent key would leave the existing entry in place. + imagePullSecrets: [] + containers: + - name: app + image: docker.io/library/busybox:stable + volumeAttachments: + - name: config-vol + mountPath: /etc/config + volumes: + - name: config-vol + configMap: + name: pullsecret-app-config + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/referenced-data-pull-secret/workload-deployment.yaml b/test/e2e/referenced-data-pull-secret/workload-deployment.yaml new file mode 100644 index 00000000..f32f012f --- /dev/null +++ b/test/e2e/referenced-data-pull-secret/workload-deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-pullsecret-wd + # namespace injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000004" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + imagePullSecrets: + - name: registry-creds + containers: + - name: app + image: docker.io/library/busybox:stable + volumeAttachments: + - name: config-vol + mountPath: /etc/config + volumes: + - name: config-vol + configMap: + name: pullsecret-app-config + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1