diff --git a/deploy/containerlab/resources/fabric-router/base/fabric-lab-patch.yaml b/deploy/containerlab/resources/fabric-router/base/fabric-lab-patch.yaml index 8b3131b1..3dd094d9 100644 --- a/deploy/containerlab/resources/fabric-router/base/fabric-lab-patch.yaml +++ b/deploy/containerlab/resources/fabric-router/base/fabric-lab-patch.yaml @@ -5,19 +5,6 @@ metadata: spec: template: spec: - # Image override only -- this overlay deliberately inherits - # config/fabric-router/'s own affinity (galactic.datumapis.com/fabric=router, - # which every lab worker carries) rather than narrowing it. It used to - # narrow to node in (compute, edge) so that iad's route-reflector node - # could be served by a second DaemonSet (resources/fabric-control/iad/, - # removed) carrying its own frr.conf; that split existed only because a - # shared fabric-config ConfigMap couldn't serve two nodes' configs. - # frr-init now selects a frr.conf. key via NODE_NAME, so one - # DaemonSet plus one ConfigMap covers every role -- the route - # reflector included (../iad/kustomization.yaml's configMapGenerator - # carries iad-worker3's key alongside the compute and edge nodes'). - # Nothing about FRR itself differs by role; only the per-node - # frr.conf each pod picks. initContainers: - name: frr-init image: fabric-router:latest diff --git a/deploy/containerlab/resources/fabric-router/dfw/kustomization.yaml b/deploy/containerlab/resources/fabric-router/dfw/kustomization.yaml index dc3ab2bb..dd1f2622 100644 --- a/deploy/containerlab/resources/fabric-router/dfw/kustomization.yaml +++ b/deploy/containerlab/resources/fabric-router/dfw/kustomization.yaml @@ -2,9 +2,7 @@ namespace: galactic-system resources: - ../base -# One frr.conf. key per node this cluster's fabric DaemonSet -# matches -- frr-init selects its own via the NODE_NAME downward-API env -# var. dfw has three: one compute worker and two edge workers. +# One frr.conf. key per node (dfw: compute, two edge workers) configMapGenerator: - name: fabric-config files: diff --git a/deploy/containerlab/resources/fabric-router/iad/kustomization.yaml b/deploy/containerlab/resources/fabric-router/iad/kustomization.yaml index 34b252c7..e9ed7e11 100644 --- a/deploy/containerlab/resources/fabric-router/iad/kustomization.yaml +++ b/deploy/containerlab/resources/fabric-router/iad/kustomization.yaml @@ -2,8 +2,7 @@ namespace: galactic-system resources: - ../base -# One key per fabric-carrying node -- see ../dfw/kustomization.yaml. iad -# has three: compute, edge, and the lab's single EVPN route reflector. +# One key per fabric-carrying node (iad: compute, edge, route reflector) configMapGenerator: - name: fabric-config files: diff --git a/deploy/containerlab/resources/fabric-router/sjc/kustomization.yaml b/deploy/containerlab/resources/fabric-router/sjc/kustomization.yaml index 8ea3b4b4..34fc8544 100644 --- a/deploy/containerlab/resources/fabric-router/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/fabric-router/sjc/kustomization.yaml @@ -2,7 +2,7 @@ namespace: galactic-system resources: - ../base -# One key per fabric-carrying node -- see ../dfw/kustomization.yaml. +# One key per fabric-carrying node configMapGenerator: - name: fabric-config files: diff --git a/deploy/containerlab/resources/galactic-cni/dfw/ebpf-interfaces-patch.yaml b/deploy/containerlab/resources/galactic-cni/dfw/ebpf-interfaces-patch.yaml index e8ecb338..c8284d93 100644 --- a/deploy/containerlab/resources/galactic-cni/dfw/ebpf-interfaces-patch.yaml +++ b/deploy/containerlab/resources/galactic-cni/dfw/ebpf-interfaces-patch.yaml @@ -5,17 +5,7 @@ metadata: spec: template: spec: - # Both of dfw-worker's uplinks. dfw-worker2/dfw-worker3 also have an - # eth2 (facing the compute node), so this value is valid on every node - # in this cluster -- which matters, since the list is DaemonSet-wide - # and an interface a node does not have fails that node's attach. - # - # This covers the SRv6 uSID decap hook only. galactic-nat's shard XDP - # program is configured separately and names the same two uplinks - # (GALACTIC_NAT_UPLINK_INTERFACES, resources/galactic-nat/dfw/ - # node-patch.yaml) -- the two lists have to agree on a dual-homed node, - # or traffic arriving on an uplink only one of them covers is - # decapsulated but never translated, or vice versa. + # Both uplinks initContainers: - name: install-cni env: diff --git a/deploy/containerlab/resources/galactic-cni/dfw/kustomization.yaml b/deploy/containerlab/resources/galactic-cni/dfw/kustomization.yaml index bc6076d8..2235197b 100644 --- a/deploy/containerlab/resources/galactic-cni/dfw/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-cni/dfw/kustomization.yaml @@ -1,5 +1,4 @@ -# dfw: the one site whose compute node is dual-homed, so the uSID decap hook -# has to attach to both of its uplinks. +# dfw: dual-homed compute node namespace: galactic-system resources: - ../shared diff --git a/deploy/containerlab/resources/galactic-cni/iad/kustomization.yaml b/deploy/containerlab/resources/galactic-cni/iad/kustomization.yaml index cb9e28bf..e8d21cca 100644 --- a/deploy/containerlab/resources/galactic-cni/iad/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-cni/iad/kustomization.yaml @@ -1,5 +1,4 @@ -# iad: single-homed compute node, so the shared eth1-only interface list -# applies unchanged. See ../dfw/ for the dual-homed case. +# iad: single-homed compute node namespace: galactic-system resources: - ../shared diff --git a/deploy/containerlab/resources/galactic-cni/shared/daemonset-patch.yaml b/deploy/containerlab/resources/galactic-cni/shared/daemonset-patch.yaml index 0e8039ef..9f660145 100644 --- a/deploy/containerlab/resources/galactic-cni/shared/daemonset-patch.yaml +++ b/deploy/containerlab/resources/galactic-cni/shared/daemonset-patch.yaml @@ -10,46 +10,13 @@ spec: image: galactic-cni:latest imagePullPolicy: Never env: - # Fabric-wide NAT66 shard membership list (design plan §3, - # internal/plumbing/srv6.EgressDefaultRouteAdd's own doc - # comment) -- the same three shard SIDs - # resources/galactic-nat/{dfw,sjc,iad}/node-patch.yaml - # configure each shard with, shared identically across every - # site here since this list is currently operator-supplied, - # not learned in-cluster (see EnvCNIEgressShardSIDs's own doc - # comment). Written into the static conflist by - # internal/installer.Bootstrap (this init container), read - # back by internal/cnibgp at every CNI ADD. + # Fabric-wide NAT66 shard SIDs - name: GALACTIC_CNI_EGRESS_SHARD_SIDS value: "2001:db8:ff01:9:e001::,2001:db8:ff02:9:e001::,2001:db8:ff03:9:e001::" - # The fabric-wide NAT64 prefix, identical to every shard's own - # GALACTIC_NAT_NAT64_PREFIX. Setting it is what makes each CNI ADD - # install the tenant VRF's route for the prefix, so an IPv6-only - # tenant reaches an IPv4 destination by addressing its synthesized - # form (RFC 6052: the IPv4 address in the low 32 bits). Written - # into the static conflist by this init container and read back by - # internal/cnibgp on every ADD, exactly like the shard list above. + # Fabric-wide NAT64 prefix - name: GALACTIC_CNI_NAT64_PREFIX value: "2001:db8:64::/96" - # Every lab node is dual-homed: eth0 carries the IPv6 default - # route but only reaches the Kind/ContainerLab management - # bridge, while eth1 is the dedicated point-to-point link to - # the transit fabric that actual SRv6-encapsulated VPC traffic - # arrives/departs on -- see credential-refresh's own identical - # env var below for the full history of why this override - # exists at all. This init container needs its own copy, not - # just credential-refresh's: internal/installer.Bootstrap - # (this container) is what resolves and writes it into the - # static conflist's own "ebpf_interfaces" field for - # internal/cnibgp to read back on every CNI ADD (mirroring - # NAT66_SHARD_SIDS just above) -- found live, without it here, - # Bootstrap's own resolution fell back to the same broken - # eth0 auto-detection this override exists to fix, producing - # a wrong-but-plausible srv6.ResolvePublicUplink/ - # ResolveNodeSourceAddress result for every CNI ADD with no - # error at all (this env var being set only on - # credential-refresh, a separate container in the same pod, - # never propagates to this one). + # Fabric uplink the eBPF hook attaches to - name: GALACTIC_CNI_EBPF_INTERFACES value: eth1 containers: @@ -57,18 +24,6 @@ spec: image: galactic-cni:latest imagePullPolicy: Never env: - # Every lab node is dual-homed: eth0 carries the IPv6 default - # route but only reaches the Kind/ContainerLab management - # bridge (kubectl/API-server traffic), while eth1 is the - # dedicated point-to-point link to the transit fabric (tr1-4) - # that actual SRv6-encapsulated VPC traffic arrives on. - # ResolveInterfaces' auto-detection (default-IPv6-route - # heuristic, internal/plumbing/ebpf/attach/interfaces.go) picks - # eth0 here since it's ambiguous between the two -- confirmed - # live by tcpdump: cross-region SRv6 packets arrive on eth1 but - # the eBPF usid_ingress filter was only ever attached to eth0, - # so decapsulation never ran and every VPC ping between sites - # silently blackholed. This override forces the correct - # interface for this topology; see docs/cni/configuration.md. + # Fabric uplink the eBPF hook attaches to - name: GALACTIC_CNI_EBPF_INTERFACES value: eth1 diff --git a/deploy/containerlab/resources/galactic-cni/shared/kustomization.yaml b/deploy/containerlab/resources/galactic-cni/shared/kustomization.yaml index c54dae92..b808bea4 100644 --- a/deploy/containerlab/resources/galactic-cni/shared/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-cni/shared/kustomization.yaml @@ -1,12 +1,4 @@ -# The lab's shared galactic-cni DaemonSet: config/galactic-cni/ (copied onto -# the node at deploy time as ../base -- see scripts/deploy-cni.sh) plus the -# lab-only image/env patch every site needs. -# -# Referenced as a directory by each site overlay rather than being applied -# directly, because one env var has to differ per site: dfw's compute node is -# dual-homed and needs GALACTIC_CNI_EBPF_INTERFACES to name both uplinks, -# while a node without eth2 fails its attach outright if eth2 is listed -# (internal/plumbing/ebpf/attach.attachOne errors on an unknown link). +# Lab-shared galactic-cni layer: base DaemonSet plus the image/env patch every site applies. resources: - ../base patches: diff --git a/deploy/containerlab/resources/galactic-cni/sjc/kustomization.yaml b/deploy/containerlab/resources/galactic-cni/sjc/kustomization.yaml index 0f43197e..7d38d888 100644 --- a/deploy/containerlab/resources/galactic-cni/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-cni/sjc/kustomization.yaml @@ -1,5 +1,4 @@ -# sjc: single-homed compute node, so the shared eth1-only interface list -# applies unchanged. See ../dfw/ for the dual-homed case. +# sjc: single-homed compute node namespace: galactic-system resources: - ../shared diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker.yaml index fd25e05c..97803c21 100644 --- a/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker.yaml @@ -1,8 +1,4 @@ -# Reflector-side session toward dfw-worker (dfw compute). The client side of -# this same session lives in resources/galactic-router/, pointed -# back at this node's fc00:0:8::1:1790. Every galactic-router in the lab -- -# compute and edge alike, in all three clusters -- is a client of this one -# reflector; compute and edge nodes never peer with each other directly. +# Reflector-side session toward dfw-worker (dfw compute) apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker2.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker2.yaml index 590a5464..86f16e6d 100644 --- a/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker2.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker2.yaml @@ -1,8 +1,4 @@ -# Reflector-side session toward dfw-worker2 (dfw edge). The client side of -# this same session lives in resources/galactic-gateway/, pointed -# back at this node's fc00:0:8::1:1790. Every galactic-router in the lab -- -# compute and edge alike, in all three clusters -- is a client of this one -# reflector; compute and edge nodes never peer with each other directly. +# Reflector-side session toward dfw-worker2 (dfw edge) apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker3.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker3.yaml index 75a6d1df..00123659 100644 --- a/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker3.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-dfw-worker3.yaml @@ -1,8 +1,4 @@ -# Reflector-side session toward dfw-worker3 (dfw edge). The client side of -# this same session lives in resources/galactic-gateway/, pointed -# back at this node's fc00:0:8::1:1790. Every galactic-router in the lab -- -# compute and edge alike, in all three clusters -- is a client of this one -# reflector; compute and edge nodes never peer with each other directly. +# Reflector-side session toward dfw-worker3 (dfw edge) apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-iad-worker.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-iad-worker.yaml index 6c1da886..7d1d2cf9 100644 --- a/deploy/containerlab/resources/galactic-control/iad/bgppeer-iad-worker.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-iad-worker.yaml @@ -1,8 +1,4 @@ -# Reflector-side session toward iad-worker (iad compute). The client side of -# this same session lives in resources/galactic-router/, pointed -# back at this node's fc00:0:8::1:1790. Every galactic-router in the lab -- -# compute and edge alike, in all three clusters -- is a client of this one -# reflector; compute and edge nodes never peer with each other directly. +# Reflector-side session toward iad-worker (iad compute) apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-iad-worker2.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-iad-worker2.yaml index 66e34d6c..1bad4e80 100644 --- a/deploy/containerlab/resources/galactic-control/iad/bgppeer-iad-worker2.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-iad-worker2.yaml @@ -1,8 +1,4 @@ -# Reflector-side session toward iad-worker2 (iad edge). The client side of -# this same session lives in resources/galactic-gateway/, pointed -# back at this node's fc00:0:8::1:1790. Every galactic-router in the lab -- -# compute and edge alike, in all three clusters -- is a client of this one -# reflector; compute and edge nodes never peer with each other directly. +# Reflector-side session toward iad-worker2 (iad edge) apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-sjc-worker.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-sjc-worker.yaml index 78c1f703..ad7c2b38 100644 --- a/deploy/containerlab/resources/galactic-control/iad/bgppeer-sjc-worker.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-sjc-worker.yaml @@ -1,8 +1,4 @@ -# Reflector-side session toward sjc-worker (sjc compute). The client side of -# this same session lives in resources/galactic-router/, pointed -# back at this node's fc00:0:8::1:1790. Every galactic-router in the lab -- -# compute and edge alike, in all three clusters -- is a client of this one -# reflector; compute and edge nodes never peer with each other directly. +# Reflector-side session toward sjc-worker (sjc compute) apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-sjc-worker2.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-sjc-worker2.yaml index b342dc28..b8992758 100644 --- a/deploy/containerlab/resources/galactic-control/iad/bgppeer-sjc-worker2.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-sjc-worker2.yaml @@ -1,8 +1,4 @@ -# Reflector-side session toward sjc-worker2 (sjc edge). The client side of -# this same session lives in resources/galactic-gateway/, pointed -# back at this node's fc00:0:8::1:1790. Every galactic-router in the lab -- -# compute and edge alike, in all three clusters -- is a client of this one -# reflector; compute and edge nodes never peer with each other directly. +# Reflector-side session toward sjc-worker2 (sjc edge) apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-control/iad/bgprouter.yaml b/deploy/containerlab/resources/galactic-control/iad/bgprouter.yaml index b20ebbc7..ffe609c3 100644 --- a/deploy/containerlab/resources/galactic-control/iad/bgprouter.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/bgprouter.yaml @@ -10,12 +10,6 @@ spec: localASN: 65000 routerID: "10.255.255.4" srv6Locator: "2001:db8:ff03::/48" - # nodeID 4, not 1: this router shares iad's locator with iad-worker - # (nodeID 1) and iad-worker2 (nodeID 2), and a uSID's Node-ID is what - # separates one node's SID space from another's within a site. The - # reflector originates no tenant path of its own, so nothing is derived - # from this today -- but a duplicate would be indistinguishable from - # iad-worker's own SIDs the moment it did. nodeID: 4 addressFamilies: - afi: l2vpn diff --git a/deploy/containerlab/resources/galactic-gateway/base/gateway-lab-patch.yaml b/deploy/containerlab/resources/galactic-gateway/base/gateway-lab-patch.yaml index 733fd88c..bc549970 100644 --- a/deploy/containerlab/resources/galactic-gateway/base/gateway-lab-patch.yaml +++ b/deploy/containerlab/resources/galactic-gateway/base/gateway-lab-patch.yaml @@ -5,19 +5,7 @@ metadata: spec: template: spec: - # Image override only. This must NOT carry a galactic-router entry: - # `containers` is a strategic-merge list keyed on `name`, so an entry - # for a container config/galactic-gateway/base/daemonset.yaml doesn't - # define is *added* rather than overridden. This patch used to list - # one, from when galactic-router ran as a sidecar in the gateway pod; - # once that moved to its own DaemonSet, the entry stayed behind and - # silently resurrected a galactic-router container with an image and - # nothing else -- no GALACTIC_ROUTER_NODE_NAME, no volumes -- which - # exited 1 ("node name is required") into CrashLoopBackOff on both - # gateway nodes while the gateway container itself stayed Ready (pod - # stuck at 1/2). galactic-router reaches these nodes via - # config/galactic-router/overlays/router/ like every other node, - # since they carry galactic.datumapis.com/galactic=router. + # Image override only. containers: - name: galactic-gateway image: galactic-gateway:latest diff --git a/deploy/containerlab/resources/galactic-gateway/base/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/base/kustomization.yaml index 84e9999c..bc5171b2 100644 --- a/deploy/containerlab/resources/galactic-gateway/base/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/base/kustomization.yaml @@ -1,11 +1,3 @@ -# Mirrors resources/galactic-router/base/'s pattern exactly, pointed at -# config/galactic-gateway/base instead of config/galactic-router/base: "gateway" is copied -# onto the node at deploy time (see scripts/deploy-galactic-router.sh's -# copy_router_gateway_config), nested here so this kustomization's -# "gateway" resource reference resolves. Unlike resources/galactic-router/ -# base/'s "tenant" resource, config/galactic-gateway/base is self-contained (its own -# full single-container DaemonSet spec, not a patch onto config/galactic-router/base), -# so there is no separate "base" resource to nest alongside it here. resources: - gateway patches: diff --git a/deploy/containerlab/resources/galactic-gateway/dfw-worker2/bgppeer.yaml b/deploy/containerlab/resources/galactic-gateway/dfw-worker2/bgppeer.yaml index 28acdc4e..e9c058d1 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw-worker2/bgppeer.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw-worker2/bgppeer.yaml @@ -1,7 +1,4 @@ -# This edge node's own client session to the lab's single EVPN route -# reflector (iad-worker3, fc00:0:8::1 port 1790) -- the same reflector every -# compute node peers with. Edge and compute exchange EVPN paths through it, -# never directly. +# Client session to the lab's single EVPN route reflector. apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/dfw-worker2/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/dfw-worker2/kustomization.yaml index f97c4705..ed31cb0e 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw-worker2/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw-worker2/kustomization.yaml @@ -1,6 +1,4 @@ -# dfw-worker2's instantiation of config/galactic-gateway/base (see that dir's -# kustomization.yaml doc comment for why this role needs one instance per -# gateway node rather than one shared DaemonSet). +# dfw-worker2's edge gateway overlay. namespace: galactic-system resources: - ../base @@ -12,13 +10,7 @@ patches: target: kind: DaemonSet name: galactic-gateway - # Strategic-merge patches can't rename a resource (Kustomize keeps the - # target's original identity regardless of what metadata.name the patch - # body says), so the rename needs a JSON6902 patch: without a distinct - # name, a site with two edge nodes would have both DaemonSets named - # "galactic-gateway" in the same namespace, silently clobbering each - # other. Applied uniformly even where a site has only one edge node, so - # the DaemonSet name always names the node it runs on. + # Renames the DaemonSet to this node's name. - target: kind: DaemonSet name: galactic-gateway diff --git a/deploy/containerlab/resources/galactic-gateway/dfw-worker2/node-patch.yaml b/deploy/containerlab/resources/galactic-gateway/dfw-worker2/node-patch.yaml index e6089b62..6b2f193f 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw-worker2/node-patch.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw-worker2/node-patch.yaml @@ -5,12 +5,7 @@ metadata: spec: template: spec: - # kubernetes.io/hostname pins this DaemonSet instance to exactly one - # node: GALACTIC_GATEWAY_SRV6_ADDRESS below must be unique per gateway - # node (see config/galactic-gateway/base/kustomization.yaml's doc - # comment), so every edge node gets its own DaemonSet rather than one - # DaemonSet matching every edge-labeled node with identical env. dfw - # has two edge nodes and so two instances; sjc and iad have one each. + # Hostname pin: one DaemonSet instance per edge node. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -29,36 +24,12 @@ spec: containers: - name: galactic-gateway env: - # eth1 is this lab's dedicated transit-fabric-facing uplink on - # every node (see resources/galactic-cni/daemonset-patch.yaml's - # comment) -- the same interface edgedsr.c's edge_lb XDP program - # attaches to for both underlay BGP and ingress traffic. + # Transit-fabric-facing uplink (underlay BGP + ingress). - name: GALACTIC_GATEWAY_PUBLIC_INTERFACE value: eth1 - # eth2 is this node's compute-facing link (see - # gvpc.clab.yaml's links section): dfw's compute node is - # dual-homed to both of dfw's edge nodes, and every site's - # compute node reaches the fabric only through its edge tier. - # A backend's reply to a VIP therefore crosses this node, - # where connection tracking holds no record of the forward - # half -- which went to the backend encapsulated, through XDP - # -- and kube-proxy drops it as invalid. edge_return forwards - # those replies before netfilter sees them. + # Compute-facing link. - name: GALACTIC_GATEWAY_INTERNAL_INTERFACES value: eth2 - # uFMT 48+16 uSID over this site's shared locator (2001:db8:ff01::/48, - # see bgprouter.yaml's srv6Locator). nodeID=2, - # Function=End.DT46 (arbitrary -- Argument 0 always misses - # vrf_table, so no Function value is ever consulted for it; see - # internal/plumbing/ebpf/prog/usid.c), Argument=0 (reserved). - # Computed via internal/plumbing/ebpf/uformat.Encode directly, - # bypassing srv6.ComputeSID's argument==0 guard (that guard - # exists for tenant-VRF SID derivation specifically). This is - # this node's own plain SRv6-reachable address, used only as the - # DSR datapath's outer-header encap source -- never a NAT/SNAT - # source and never compared against anything on a receive path - # (see internal/config/gateway.go's EnvGatewaySRv6Address doc - # comment). No in-cluster mechanism yet derives this - # automatically, so it is supplied statically per node here. + # This node's SRv6 address: DSR outer-header encap source. - name: GALACTIC_GATEWAY_SRV6_ADDRESS value: "2001:db8:ff01:2:e000::" diff --git a/deploy/containerlab/resources/galactic-gateway/dfw-worker3/bgppeer.yaml b/deploy/containerlab/resources/galactic-gateway/dfw-worker3/bgppeer.yaml index d89598ae..2174e51b 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw-worker3/bgppeer.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw-worker3/bgppeer.yaml @@ -1,7 +1,4 @@ -# This edge node's own client session to the lab's single EVPN route -# reflector (iad-worker3, fc00:0:8::1 port 1790) -- the same reflector every -# compute node peers with. Edge and compute exchange EVPN paths through it, -# never directly. +# Client session to the lab's single EVPN route reflector. apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/dfw-worker3/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/dfw-worker3/kustomization.yaml index 65e3e697..5c457fc5 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw-worker3/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw-worker3/kustomization.yaml @@ -1,6 +1,4 @@ -# dfw-worker3's instantiation of config/galactic-gateway/base (see that dir's -# kustomization.yaml doc comment for why this role needs one instance per -# gateway node rather than one shared DaemonSet). +# dfw-worker3's edge gateway overlay. namespace: galactic-system resources: - ../base @@ -12,13 +10,7 @@ patches: target: kind: DaemonSet name: galactic-gateway - # Strategic-merge patches can't rename a resource (Kustomize keeps the - # target's original identity regardless of what metadata.name the patch - # body says), so the rename needs a JSON6902 patch: without a distinct - # name, a site with two edge nodes would have both DaemonSets named - # "galactic-gateway" in the same namespace, silently clobbering each - # other. Applied uniformly even where a site has only one edge node, so - # the DaemonSet name always names the node it runs on. + # Renames the DaemonSet to this node's name. - target: kind: DaemonSet name: galactic-gateway diff --git a/deploy/containerlab/resources/galactic-gateway/dfw-worker3/node-patch.yaml b/deploy/containerlab/resources/galactic-gateway/dfw-worker3/node-patch.yaml index 40f794b2..a85911b6 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw-worker3/node-patch.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw-worker3/node-patch.yaml @@ -5,12 +5,7 @@ metadata: spec: template: spec: - # kubernetes.io/hostname pins this DaemonSet instance to exactly one - # node: GALACTIC_GATEWAY_SRV6_ADDRESS below must be unique per gateway - # node (see config/galactic-gateway/base/kustomization.yaml's doc - # comment), so every edge node gets its own DaemonSet rather than one - # DaemonSet matching every edge-labeled node with identical env. dfw - # has two edge nodes and so two instances; sjc and iad have one each. + # Hostname pin: one DaemonSet instance per edge node. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -29,36 +24,12 @@ spec: containers: - name: galactic-gateway env: - # eth1 is this lab's dedicated transit-fabric-facing uplink on - # every node (see resources/galactic-cni/daemonset-patch.yaml's - # comment) -- the same interface edgedsr.c's edge_lb XDP program - # attaches to for both underlay BGP and ingress traffic. + # Transit-fabric-facing uplink (underlay BGP + ingress). - name: GALACTIC_GATEWAY_PUBLIC_INTERFACE value: eth1 - # eth2 is this node's compute-facing link (see - # gvpc.clab.yaml's links section): dfw's compute node is - # dual-homed to both of dfw's edge nodes, and every site's - # compute node reaches the fabric only through its edge tier. - # A backend's reply to a VIP therefore crosses this node, - # where connection tracking holds no record of the forward - # half -- which went to the backend encapsulated, through XDP - # -- and kube-proxy drops it as invalid. edge_return forwards - # those replies before netfilter sees them. + # Compute-facing link. - name: GALACTIC_GATEWAY_INTERNAL_INTERFACES value: eth2 - # uFMT 48+16 uSID over this site's shared locator (2001:db8:ff01::/48, - # see bgprouter.yaml's srv6Locator). nodeID=3, - # Function=End.DT46 (arbitrary -- Argument 0 always misses - # vrf_table, so no Function value is ever consulted for it; see - # internal/plumbing/ebpf/prog/usid.c), Argument=0 (reserved). - # Computed via internal/plumbing/ebpf/uformat.Encode directly, - # bypassing srv6.ComputeSID's argument==0 guard (that guard - # exists for tenant-VRF SID derivation specifically). This is - # this node's own plain SRv6-reachable address, used only as the - # DSR datapath's outer-header encap source -- never a NAT/SNAT - # source and never compared against anything on a receive path - # (see internal/config/gateway.go's EnvGatewaySRv6Address doc - # comment). No in-cluster mechanism yet derives this - # automatically, so it is supplied statically per node here. + # This node's SRv6 address: DSR outer-header encap source. - name: GALACTIC_GATEWAY_SRV6_ADDRESS value: "2001:db8:ff01:3:e000::" diff --git a/deploy/containerlab/resources/galactic-gateway/dfw/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/dfw/kustomization.yaml index 1348b3e9..710c7544 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw/kustomization.yaml @@ -1,8 +1,5 @@ -# The single entry point for dfw's edge tier: every edge node's -# DaemonSet+BGP+NetworkGateway instantiation, plus this site's own ns60 -# NetworkRule and its backend's ServiceVIPBinding. -# scripts/deploy-galactic-router.sh applies this one path per site rather -# than each node overlay separately. +# Single entry point for dfw's edge tier: per-edge-node DaemonSet+BGP+ +# NetworkGateway, plus this site's ns60 NetworkRule and ServiceVIPBinding. namespace: galactic-system resources: - ../dfw-worker2 diff --git a/deploy/containerlab/resources/galactic-gateway/dfw/networkrule-ns60.yaml b/deploy/containerlab/resources/galactic-gateway/dfw/networkrule-ns60.yaml index 473de3fd..51cd4d9e 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw/networkrule-ns60.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw/networkrule-ns60.yaml @@ -1,25 +1,4 @@ -# ns60: a live web service (resources/tenants/ns60/, vpc=60/vpcattachment=60, -# see ns60/dfw/nad.yaml) exposed through the edge XDP Maglev/DSR gateway on -# a real, routable ingress VIP. This rule's backend address is the nginx -# Deployment's actual live pod address (`kubectl get pod -n ns60 -o wide`) -# and its VIP is IPv6, matching the datapath's current scope: IPv6-only, -# plain TCP/UDP, no IPv4 support yet (see edgedsr.c). -# -# 2001:db8:6060::1 is a documentation-range (RFC 3849, 2001:db8::/32) address -# picked to avoid colliding with anything else in this topology (transit -# mesh: 2001:db8:0:*::/64 and 2001:db8:1:*::/64; SRv6 locators: -# 2001:db8:ff0X::/48; NAT66 shard public addresses: 2001:db8:9966::/32). -# -# The *same* VIP is used in all three clusters, each with its own site-local -# backend -- a genuine anycast service, not three separate ones. Both edge nodes -# originates it identically (DSR's anycast model: no primary/secondary -# status.primaryNode assignment, no BGP local-preference split -- see -# internal/controller/networkgateway_controller.go), and every edge node's -# FRR also originates the covering 2001:db8:6060::/48 into the underlay -# (resources/fabric-router/dfw/frr.conf.dfw-worker2), so the transit can -# forward a packet whose literal destination is the VIP. Which site answers -# a given client is then just transit best-path; which gateway within that -# site answers is Maglev's consistent-hash ring. +# ns60 live web service exposed on a routable IPv6 ingress VIP. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/dfw/servicevipbinding-ns60.yaml b/deploy/containerlab/resources/galactic-gateway/dfw/servicevipbinding-ns60.yaml index fb8c86e5..f8148099 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw/servicevipbinding-ns60.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw/servicevipbinding-ns60.yaml @@ -1,22 +1,4 @@ -# Sample ServiceVIPBinding for dfw's ns60 backend, alongside -# networkrule-ns60.yaml (same VIP/backend pair -- see that file's own -# comment for the address provenance). ns60's nginx Deployment -# (resources/tenants/ns60/) has no node pin of its own beyond excluding -# control-plane nodes, and lands on dfw-worker in practice: it is this -# cluster's only untainted worker, since every edge node carries -# galactic.datumapis.com/node=edge:NoSchedule and iad's reflector carries -# galactic.datumapis.com/galactic=control:NoSchedule. Confirm the live pod's -# actual node with `kubectl get pod -n ns60 -o wide`. -# -# egressKind: veth, since ns60's backend is a plain container Deployment -# (veth attach, not a tap/VM backend) -- see ServiceVIPBinding's own doc -# comment for the veth/tap fork. backendAddress/backendPort match -# networkrule-ns60.yaml's own backend entry exactly -- required for veth -# too, not just tap: a decapsulated ingress packet is delivered into vpc60's -# own VRF routing table, which has no route to vipAddress at all (it is only -# bound on the node's own root-namespace galactic-vip0 interface, outside -# that VRF) -- found live in this lab. ServiceVIPBindingReconciler registers -# the same vip_xlat_table substitution tap already used. +# ServiceVIPBinding for dfw's ns60 backend. apiVersion: network.datumapis.com/v1alpha1 kind: ServiceVIPBinding metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/iad-worker2/bgppeer.yaml b/deploy/containerlab/resources/galactic-gateway/iad-worker2/bgppeer.yaml index badfd98f..31db4c5a 100644 --- a/deploy/containerlab/resources/galactic-gateway/iad-worker2/bgppeer.yaml +++ b/deploy/containerlab/resources/galactic-gateway/iad-worker2/bgppeer.yaml @@ -1,7 +1,4 @@ -# This edge node's own client session to the lab's single EVPN route -# reflector (iad-worker3, fc00:0:8::1 port 1790) -- the same reflector every -# compute node peers with. Edge and compute exchange EVPN paths through it, -# never directly. +# Client session to the lab's single EVPN route reflector. apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/iad-worker2/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/iad-worker2/kustomization.yaml index c999e7bd..6ddb2593 100644 --- a/deploy/containerlab/resources/galactic-gateway/iad-worker2/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/iad-worker2/kustomization.yaml @@ -1,6 +1,4 @@ -# iad-worker2's instantiation of config/galactic-gateway/base (see that dir's -# kustomization.yaml doc comment for why this role needs one instance per -# gateway node rather than one shared DaemonSet). +# iad-worker2's edge gateway overlay. namespace: galactic-system resources: - ../base @@ -12,13 +10,7 @@ patches: target: kind: DaemonSet name: galactic-gateway - # Strategic-merge patches can't rename a resource (Kustomize keeps the - # target's original identity regardless of what metadata.name the patch - # body says), so the rename needs a JSON6902 patch: without a distinct - # name, a site with two edge nodes would have both DaemonSets named - # "galactic-gateway" in the same namespace, silently clobbering each - # other. Applied uniformly even where a site has only one edge node, so - # the DaemonSet name always names the node it runs on. + # Renames the DaemonSet to this node's name. - target: kind: DaemonSet name: galactic-gateway diff --git a/deploy/containerlab/resources/galactic-gateway/iad-worker2/node-patch.yaml b/deploy/containerlab/resources/galactic-gateway/iad-worker2/node-patch.yaml index 0f983fdb..088cf05d 100644 --- a/deploy/containerlab/resources/galactic-gateway/iad-worker2/node-patch.yaml +++ b/deploy/containerlab/resources/galactic-gateway/iad-worker2/node-patch.yaml @@ -5,12 +5,7 @@ metadata: spec: template: spec: - # kubernetes.io/hostname pins this DaemonSet instance to exactly one - # node: GALACTIC_GATEWAY_SRV6_ADDRESS below must be unique per gateway - # node (see config/galactic-gateway/base/kustomization.yaml's doc - # comment), so every edge node gets its own DaemonSet rather than one - # DaemonSet matching every edge-labeled node with identical env. dfw - # has two edge nodes and so two instances; sjc and iad have one each. + # Hostname pin: one DaemonSet instance per edge node. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -29,36 +24,12 @@ spec: containers: - name: galactic-gateway env: - # eth1 is this lab's dedicated transit-fabric-facing uplink on - # every node (see resources/galactic-cni/daemonset-patch.yaml's - # comment) -- the same interface edgedsr.c's edge_lb XDP program - # attaches to for both underlay BGP and ingress traffic. + # Transit-fabric-facing uplink (underlay BGP + ingress). - name: GALACTIC_GATEWAY_PUBLIC_INTERFACE value: eth1 - # eth2 is this node's compute-facing link (see - # gvpc.clab.yaml's links section): dfw's compute node is - # dual-homed to both of dfw's edge nodes, and every site's - # compute node reaches the fabric only through its edge tier. - # A backend's reply to a VIP therefore crosses this node, - # where connection tracking holds no record of the forward - # half -- which went to the backend encapsulated, through XDP - # -- and kube-proxy drops it as invalid. edge_return forwards - # those replies before netfilter sees them. + # Compute-facing link. - name: GALACTIC_GATEWAY_INTERNAL_INTERFACES value: eth2 - # uFMT 48+16 uSID over this site's shared locator (2001:db8:ff03::/48, - # see bgprouter.yaml's srv6Locator). nodeID=2, - # Function=End.DT46 (arbitrary -- Argument 0 always misses - # vrf_table, so no Function value is ever consulted for it; see - # internal/plumbing/ebpf/prog/usid.c), Argument=0 (reserved). - # Computed via internal/plumbing/ebpf/uformat.Encode directly, - # bypassing srv6.ComputeSID's argument==0 guard (that guard - # exists for tenant-VRF SID derivation specifically). This is - # this node's own plain SRv6-reachable address, used only as the - # DSR datapath's outer-header encap source -- never a NAT/SNAT - # source and never compared against anything on a receive path - # (see internal/config/gateway.go's EnvGatewaySRv6Address doc - # comment). No in-cluster mechanism yet derives this - # automatically, so it is supplied statically per node here. + # This node's SRv6 address: DSR outer-header encap source. - name: GALACTIC_GATEWAY_SRV6_ADDRESS value: "2001:db8:ff03:2:e000::" diff --git a/deploy/containerlab/resources/galactic-gateway/iad/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/iad/kustomization.yaml index e8e687d6..c6e2c77e 100644 --- a/deploy/containerlab/resources/galactic-gateway/iad/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/iad/kustomization.yaml @@ -1,8 +1,5 @@ -# The single entry point for iad's edge tier: every edge node's -# DaemonSet+BGP+NetworkGateway instantiation, plus this site's own ns60 -# NetworkRule and its backend's ServiceVIPBinding. -# scripts/deploy-galactic-router.sh applies this one path per site rather -# than each node overlay separately. +# Single entry point for iad's edge tier: per-edge-node DaemonSet+BGP+ +# NetworkGateway, plus this site's ns60 NetworkRule and ServiceVIPBinding. namespace: galactic-system resources: - ../iad-worker2 diff --git a/deploy/containerlab/resources/galactic-gateway/iad/networkrule-ns60.yaml b/deploy/containerlab/resources/galactic-gateway/iad/networkrule-ns60.yaml index e67eca8f..3a42ce45 100644 --- a/deploy/containerlab/resources/galactic-gateway/iad/networkrule-ns60.yaml +++ b/deploy/containerlab/resources/galactic-gateway/iad/networkrule-ns60.yaml @@ -1,25 +1,4 @@ -# ns60: a live web service (resources/tenants/ns60/, vpc=60/vpcattachment=60, -# see ns60/iad/nad.yaml) exposed through the edge XDP Maglev/DSR gateway on -# a real, routable ingress VIP. This rule's backend address is the nginx -# Deployment's actual live pod address (`kubectl get pod -n ns60 -o wide`) -# and its VIP is IPv6, matching the datapath's current scope: IPv6-only, -# plain TCP/UDP, no IPv4 support yet (see edgedsr.c). -# -# 2001:db8:6060::1 is a documentation-range (RFC 3849, 2001:db8::/32) address -# picked to avoid colliding with anything else in this topology (transit -# mesh: 2001:db8:0:*::/64 and 2001:db8:1:*::/64; SRv6 locators: -# 2001:db8:ff0X::/48; NAT66 shard public addresses: 2001:db8:9966::/32). -# -# The *same* VIP is used in all three clusters, each with its own site-local -# backend -- a genuine anycast service, not three separate ones. This site's edge node -# originates it identically (DSR's anycast model: no primary/secondary -# status.primaryNode assignment, no BGP local-preference split -- see -# internal/controller/networkgateway_controller.go), and every edge node's -# FRR also originates the covering 2001:db8:6060::/48 into the underlay -# (resources/fabric-router/iad/frr.conf.iad-worker2), so the transit can -# forward a packet whose literal destination is the VIP. Which site answers -# a given client is then just transit best-path; which gateway within that -# site answers is Maglev's consistent-hash ring. +# ns60 live web service exposed on a routable IPv6 ingress VIP. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/iad/servicevipbinding-ns60.yaml b/deploy/containerlab/resources/galactic-gateway/iad/servicevipbinding-ns60.yaml index b43ac3fc..9fa0fdd0 100644 --- a/deploy/containerlab/resources/galactic-gateway/iad/servicevipbinding-ns60.yaml +++ b/deploy/containerlab/resources/galactic-gateway/iad/servicevipbinding-ns60.yaml @@ -1,22 +1,4 @@ -# Sample ServiceVIPBinding for iad's ns60 backend, alongside -# networkrule-ns60.yaml (same VIP/backend pair -- see that file's own -# comment for the address provenance). ns60's nginx Deployment -# (resources/tenants/ns60/) has no node pin of its own beyond excluding -# control-plane nodes, and lands on iad-worker in practice: it is this -# cluster's only untainted worker, since every edge node carries -# galactic.datumapis.com/node=edge:NoSchedule and iad's reflector carries -# galactic.datumapis.com/galactic=control:NoSchedule. Confirm the live pod's -# actual node with `kubectl get pod -n ns60 -o wide`. -# -# egressKind: veth, since ns60's backend is a plain container Deployment -# (veth attach, not a tap/VM backend) -- see ServiceVIPBinding's own doc -# comment for the veth/tap fork. backendAddress/backendPort match -# networkrule-ns60.yaml's own backend entry exactly -- required for veth -# too, not just tap: a decapsulated ingress packet is delivered into vpc60's -# own VRF routing table, which has no route to vipAddress at all (it is only -# bound on the node's own root-namespace galactic-vip0 interface, outside -# that VRF) -- found live in this lab. ServiceVIPBindingReconciler registers -# the same vip_xlat_table substitution tap already used. +# ServiceVIPBinding for iad's ns60 backend. apiVersion: network.datumapis.com/v1alpha1 kind: ServiceVIPBinding metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/sjc-worker2/bgppeer.yaml b/deploy/containerlab/resources/galactic-gateway/sjc-worker2/bgppeer.yaml index 69dbe3b4..727f5b8e 100644 --- a/deploy/containerlab/resources/galactic-gateway/sjc-worker2/bgppeer.yaml +++ b/deploy/containerlab/resources/galactic-gateway/sjc-worker2/bgppeer.yaml @@ -1,7 +1,4 @@ -# This edge node's own client session to the lab's single EVPN route -# reflector (iad-worker3, fc00:0:8::1 port 1790) -- the same reflector every -# compute node peers with. Edge and compute exchange EVPN paths through it, -# never directly. +# Client session to the lab's single EVPN route reflector. apiVersion: network.datumapis.com/v1alpha1 kind: BGPPeer metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/sjc-worker2/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/sjc-worker2/kustomization.yaml index c0019688..188d3e4d 100644 --- a/deploy/containerlab/resources/galactic-gateway/sjc-worker2/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/sjc-worker2/kustomization.yaml @@ -1,6 +1,4 @@ -# sjc-worker2's instantiation of config/galactic-gateway/base (see that dir's -# kustomization.yaml doc comment for why this role needs one instance per -# gateway node rather than one shared DaemonSet). +# sjc-worker2's edge gateway overlay. namespace: galactic-system resources: - ../base @@ -12,13 +10,7 @@ patches: target: kind: DaemonSet name: galactic-gateway - # Strategic-merge patches can't rename a resource (Kustomize keeps the - # target's original identity regardless of what metadata.name the patch - # body says), so the rename needs a JSON6902 patch: without a distinct - # name, a site with two edge nodes would have both DaemonSets named - # "galactic-gateway" in the same namespace, silently clobbering each - # other. Applied uniformly even where a site has only one edge node, so - # the DaemonSet name always names the node it runs on. + # Renames the DaemonSet to this node's name. - target: kind: DaemonSet name: galactic-gateway diff --git a/deploy/containerlab/resources/galactic-gateway/sjc-worker2/node-patch.yaml b/deploy/containerlab/resources/galactic-gateway/sjc-worker2/node-patch.yaml index 68def373..e1426d39 100644 --- a/deploy/containerlab/resources/galactic-gateway/sjc-worker2/node-patch.yaml +++ b/deploy/containerlab/resources/galactic-gateway/sjc-worker2/node-patch.yaml @@ -5,12 +5,7 @@ metadata: spec: template: spec: - # kubernetes.io/hostname pins this DaemonSet instance to exactly one - # node: GALACTIC_GATEWAY_SRV6_ADDRESS below must be unique per gateway - # node (see config/galactic-gateway/base/kustomization.yaml's doc - # comment), so every edge node gets its own DaemonSet rather than one - # DaemonSet matching every edge-labeled node with identical env. dfw - # has two edge nodes and so two instances; sjc and iad have one each. + # Hostname pin: one DaemonSet instance per edge node. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -29,36 +24,12 @@ spec: containers: - name: galactic-gateway env: - # eth1 is this lab's dedicated transit-fabric-facing uplink on - # every node (see resources/galactic-cni/daemonset-patch.yaml's - # comment) -- the same interface edgedsr.c's edge_lb XDP program - # attaches to for both underlay BGP and ingress traffic. + # Transit-fabric-facing uplink (underlay BGP + ingress). - name: GALACTIC_GATEWAY_PUBLIC_INTERFACE value: eth1 - # eth2 is this node's compute-facing link (see - # gvpc.clab.yaml's links section): dfw's compute node is - # dual-homed to both of dfw's edge nodes, and every site's - # compute node reaches the fabric only through its edge tier. - # A backend's reply to a VIP therefore crosses this node, - # where connection tracking holds no record of the forward - # half -- which went to the backend encapsulated, through XDP - # -- and kube-proxy drops it as invalid. edge_return forwards - # those replies before netfilter sees them. + # Compute-facing link. - name: GALACTIC_GATEWAY_INTERNAL_INTERFACES value: eth2 - # uFMT 48+16 uSID over this site's shared locator (2001:db8:ff02::/48, - # see bgprouter.yaml's srv6Locator). nodeID=2, - # Function=End.DT46 (arbitrary -- Argument 0 always misses - # vrf_table, so no Function value is ever consulted for it; see - # internal/plumbing/ebpf/prog/usid.c), Argument=0 (reserved). - # Computed via internal/plumbing/ebpf/uformat.Encode directly, - # bypassing srv6.ComputeSID's argument==0 guard (that guard - # exists for tenant-VRF SID derivation specifically). This is - # this node's own plain SRv6-reachable address, used only as the - # DSR datapath's outer-header encap source -- never a NAT/SNAT - # source and never compared against anything on a receive path - # (see internal/config/gateway.go's EnvGatewaySRv6Address doc - # comment). No in-cluster mechanism yet derives this - # automatically, so it is supplied statically per node here. + # This node's SRv6 address: DSR outer-header encap source. - name: GALACTIC_GATEWAY_SRV6_ADDRESS value: "2001:db8:ff02:2:e000::" diff --git a/deploy/containerlab/resources/galactic-gateway/sjc/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/sjc/kustomization.yaml index 77c61d85..73e9a0a3 100644 --- a/deploy/containerlab/resources/galactic-gateway/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/sjc/kustomization.yaml @@ -1,8 +1,5 @@ -# The single entry point for sjc's edge tier: every edge node's -# DaemonSet+BGP+NetworkGateway instantiation, plus this site's own ns60 -# NetworkRule and its backend's ServiceVIPBinding. -# scripts/deploy-galactic-router.sh applies this one path per site rather -# than each node overlay separately. +# Single entry point for sjc's edge tier: per-edge-node DaemonSet+BGP+ +# NetworkGateway, plus this site's ns60 NetworkRule and ServiceVIPBinding. namespace: galactic-system resources: - ../sjc-worker2 diff --git a/deploy/containerlab/resources/galactic-gateway/sjc/networkrule-ns60.yaml b/deploy/containerlab/resources/galactic-gateway/sjc/networkrule-ns60.yaml index 354d1a74..ef223871 100644 --- a/deploy/containerlab/resources/galactic-gateway/sjc/networkrule-ns60.yaml +++ b/deploy/containerlab/resources/galactic-gateway/sjc/networkrule-ns60.yaml @@ -1,25 +1,4 @@ -# ns60: a live web service (resources/tenants/ns60/, vpc=60/vpcattachment=60, -# see ns60/sjc/nad.yaml) exposed through the edge XDP Maglev/DSR gateway on -# a real, routable ingress VIP. This rule's backend address is the nginx -# Deployment's actual live pod address (`kubectl get pod -n ns60 -o wide`) -# and its VIP is IPv6, matching the datapath's current scope: IPv6-only, -# plain TCP/UDP, no IPv4 support yet (see edgedsr.c). -# -# 2001:db8:6060::1 is a documentation-range (RFC 3849, 2001:db8::/32) address -# picked to avoid colliding with anything else in this topology (transit -# mesh: 2001:db8:0:*::/64 and 2001:db8:1:*::/64; SRv6 locators: -# 2001:db8:ff0X::/48; NAT66 shard public addresses: 2001:db8:9966::/32). -# -# The *same* VIP is used in all three clusters, each with its own site-local -# backend -- a genuine anycast service, not three separate ones. This site's edge node -# originates it identically (DSR's anycast model: no primary/secondary -# status.primaryNode assignment, no BGP local-preference split -- see -# internal/controller/networkgateway_controller.go), and every edge node's -# FRR also originates the covering 2001:db8:6060::/48 into the underlay -# (resources/fabric-router/sjc/frr.conf.sjc-worker2), so the transit can -# forward a packet whose literal destination is the VIP. Which site answers -# a given client is then just transit best-path; which gateway within that -# site answers is Maglev's consistent-hash ring. +# ns60 live web service exposed on a routable IPv6 ingress VIP. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: diff --git a/deploy/containerlab/resources/galactic-gateway/sjc/servicevipbinding-ns60.yaml b/deploy/containerlab/resources/galactic-gateway/sjc/servicevipbinding-ns60.yaml index 88c48d41..7c22863c 100644 --- a/deploy/containerlab/resources/galactic-gateway/sjc/servicevipbinding-ns60.yaml +++ b/deploy/containerlab/resources/galactic-gateway/sjc/servicevipbinding-ns60.yaml @@ -1,22 +1,4 @@ -# Sample ServiceVIPBinding for sjc's ns60 backend, alongside -# networkrule-ns60.yaml (same VIP/backend pair -- see that file's own -# comment for the address provenance). ns60's nginx Deployment -# (resources/tenants/ns60/) has no node pin of its own beyond excluding -# control-plane nodes, and lands on sjc-worker in practice: it is this -# cluster's only untainted worker, since every edge node carries -# galactic.datumapis.com/node=edge:NoSchedule and iad's reflector carries -# galactic.datumapis.com/galactic=control:NoSchedule. Confirm the live pod's -# actual node with `kubectl get pod -n ns60 -o wide`. -# -# egressKind: veth, since ns60's backend is a plain container Deployment -# (veth attach, not a tap/VM backend) -- see ServiceVIPBinding's own doc -# comment for the veth/tap fork. backendAddress/backendPort match -# networkrule-ns60.yaml's own backend entry exactly -- required for veth -# too, not just tap: a decapsulated ingress packet is delivered into vpc60's -# own VRF routing table, which has no route to vipAddress at all (it is only -# bound on the node's own root-namespace galactic-vip0 interface, outside -# that VRF) -- found live in this lab. ServiceVIPBindingReconciler registers -# the same vip_xlat_table substitution tap already used. +# ServiceVIPBinding for sjc's ns60 backend. apiVersion: network.datumapis.com/v1alpha1 kind: ServiceVIPBinding metadata: diff --git a/deploy/containerlab/resources/galactic-nat/base/kustomization.yaml b/deploy/containerlab/resources/galactic-nat/base/kustomization.yaml index d0f6efbb..97e483a7 100644 --- a/deploy/containerlab/resources/galactic-nat/base/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-nat/base/kustomization.yaml @@ -1,13 +1,3 @@ -# Mirrors resources/galactic-gateway/base/'s pattern exactly, pointed at -# config/galactic-nat/base instead of config/galactic-gateway/base: -# "nat" is copied onto the node at deploy time by -# scripts/deploy-galactic-nat.sh's copy_nat_config helper (mirroring -# scripts/deploy-galactic-router.sh's copy_router_gateway_config), nested -# here so this kustomization's "nat" resource reference resolves. -# config/galactic-nat/base is self-contained (its own full -# single-container DaemonSet spec, not a patch onto some other config/ -# base), so there is no separate "base" resource to nest alongside it here -# -- same as config/galactic-gateway/base's own equivalent note. resources: - nat patches: diff --git a/deploy/containerlab/resources/galactic-nat/base/nat-lab-patch.yaml b/deploy/containerlab/resources/galactic-nat/base/nat-lab-patch.yaml index cf8ad567..b00830ad 100644 --- a/deploy/containerlab/resources/galactic-nat/base/nat-lab-patch.yaml +++ b/deploy/containerlab/resources/galactic-nat/base/nat-lab-patch.yaml @@ -1,9 +1,3 @@ -# Lab-only image override, same idea as resources/galactic-gateway/base/ -# router-lab-patch.yaml: Kind loads locally-built images by tag, so the -# real ghcr.io/datum-cloud/galactic-nat: reference -# config/galactic-nat/base/daemonset.yaml ships is swapped for the -# locally-tagged image with imagePullPolicy: Never so the kubelet never -# tries to pull it from GHCR. apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/deploy/containerlab/resources/galactic-nat/dfw/egressshard.yaml b/deploy/containerlab/resources/galactic-nat/dfw/egressshard.yaml index 6d59cad2..f00e5300 100644 --- a/deploy/containerlab/resources/galactic-nat/dfw/egressshard.yaml +++ b/deploy/containerlab/resources/galactic-nat/dfw/egressshard.yaml @@ -1,11 +1,3 @@ -# Sample EgressShard for dfw-worker, this cluster's own shard node -- -# alongside node-patch.yaml the same way resources/galactic-gateway/ -# /networkgateway.yaml sits alongside its own node-patch.yaml. -# Status is left empty: EgressShardReconciler (running inside the -# galactic-nat process on dfw-worker itself, per node-patch.yaml's -# GALACTIC_NAT_SHARD_SID/_SHARD_PUB_ADDR6) fills in -# status.shardAddress/status.shardSID from its own config at startup -- -# this object only needs to exist and target the right node. apiVersion: network.datumapis.com/v1alpha1 kind: EgressShard metadata: diff --git a/deploy/containerlab/resources/galactic-nat/dfw/kustomization.yaml b/deploy/containerlab/resources/galactic-nat/dfw/kustomization.yaml index 13456079..54ca2852 100644 --- a/deploy/containerlab/resources/galactic-nat/dfw/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-nat/dfw/kustomization.yaml @@ -1,10 +1,3 @@ -# dfw's instantiation of ../base as this site's NAT66 shard, reusing the -# already-existing dfw-worker node (see node-patch.yaml and ../README.md) -# rather than a new dedicated shard-labeled node. Mirrors -# resources/galactic-router/dfw/'s per-site pattern (no rename needed the -# way resources/galactic-gateway// require: dfw/iad/sjc -# are three separate clusters, so "galactic-nat" never collides with -# itself the way two DaemonSets in the same iad cluster would). namespace: galactic-system resources: - ../base diff --git a/deploy/containerlab/resources/galactic-nat/dfw/node-patch.yaml b/deploy/containerlab/resources/galactic-nat/dfw/node-patch.yaml index db8a80a8..77a7ebcf 100644 --- a/deploy/containerlab/resources/galactic-nat/dfw/node-patch.yaml +++ b/deploy/containerlab/resources/galactic-nat/dfw/node-patch.yaml @@ -5,135 +5,19 @@ metadata: spec: template: spec: - # config/galactic-nat/base/daemonset.yaml's own affinity requires - # galactic.datumapis.com/node: compute -- dfw-worker - # (node_files/dfw/config.yaml) carries it, reusing that existing - # compute worker as this site's shard rather than introducing a - # dedicated node, per the redesign plan's own §8 suggestion. No - # affinity override needed here: dfw-worker is the only node in - # this cluster carrying that label, so the base's own affinity - # already resolves to exactly it -- unlike - # resources/galactic-gateway/dfw-worker2/, which needs a - # kubernetes.io/hostname pin because iad has *two* edge-labeled - # nodes needing different per-node env. GALACTIC_NAT_SHARD_SID/ - # _SHARD_PUB_ADDR6 below must be unique per shard node (see - # config/galactic-nat/kustomization.yaml's doc comment), so - # dfw/iad/sjc are three separate per-node overlays rather than one - # overlay matching all three site workers with identical env. containers: - name: galactic-nat env: - # dfw-worker is the lab's one dual-homed compute node: eth1 to - # dfw-worker2 and eth2 to dfw-worker3 (gvpc.clab.yaml), so it - # names both here. internal/plumbing/ebpf/natprog's XDP program - # attaches to every interface in this list. - # - # Both, not just eth1, and this is the whole point of the list - # (#545): the datapath claims a packet only on an interface it - # is attached to, so an encapsulated tenant packet arriving on - # an unattached uplink would reach no translation program and be - # forwarded untranslated and uncounted. While this took a single - # name, dfw's FRR config had to de-prefer eth2 to keep egress- - # shard traffic off it -- that workaround is gone now (see - # resources/fabric-router/dfw/frr.conf.dfw-worker), and losing - # either edge node no longer degrades this node's shard role. - # - # Matches GALACTIC_CNI_EBPF_INTERFACES on this same site, which - # has always named both uplinks - # (resources/galactic-cni/dfw/ebpf-interfaces-patch.yaml). - # sjc and iad are single-homed and name eth1 alone. + # dual-homed node: eth1, eth2 uplinks - name: GALACTIC_NAT_UPLINK_INTERFACES value: eth1,eth2 - # uFMT 48+16 uSID over dfw's own locator (2001:db8:ff01::/48, - # see resources/galactic-router/dfw/bgprouter.yaml's - # srv6Locator), nodeID=9 -- a dedicated Node-ID reserved for - # this shard, deliberately NOT dfw-worker's own real nodeID=1 - # (an earlier version of this file reused it, on the theory - # that this shard "runs on that exact physical node, not a - # separate dedicated node"). That reuse was a real bug, found - # live: internal/plumbing/ebpf/natprog/nat.c's own - # locator_matches only checks the top 64 bits (Block+Node-ID) - # of a packet's outer destination against shard_sid -- see - # that function's own doc comment -- so reusing dfw-worker's - # Node-ID meant this shard's XDP program silently hijacked - # *every* ordinary tenant ingress packet addressed to - # dfw-worker's own real delivery uSIDs (same Block+Node-ID, - # nexthdr also 41) before usid_ingress's TC hook ever saw - # them. Physical co-location (which node this pod is - # scheduled to) and uSID identity (which Node-ID a shard's own - # address claims) are independent; nodeID=9 is free at this - # site (dfw-worker is the only BGPRouter on this locator, - # nodeID=1 -- see resources/galactic-router/dfw/bgprouter.yaml). - # Function=End.DT46 (0xE), matching resources/galactic-gateway/ - # the edge nodes' node-patch.yaml convention, even though this - # address is never actually looked up through - # function_table/vrf_table the way a real uSID is -- - # EgressShardStatus.ShardSID's own doc comment says it's - # advertised as a plain node-reachability route "no - # VRFID/Function" -- the uFMT encoding here only keeps the - # address inside dfw's locator block. - # - # Argument=1 is a placeholder and carries no meaning: it names - # no tenant and never could, this one configured value being - # shared by every VRF on every node. Each CNI ADD overwrites - # it with that attachment's own VRFID before installing the - # tenant's egress route (internal/cnibgp's - # shardSIDsForTenant), which is what lets this shard tell two - # same-node tenants apart. What this address does have to - # reserve for itself is its whole Block+Node-ID: that /64 is - # what EgressShardReconciler advertises and what - # locator_matches compares against. - # - # Still a lab-only value needing real allocation logic (the - # same gap GALACTIC_GATEWAY_SRV6_ADDRESS and EnvNATShardSID's - # own doc comment both flag today) before production. + # shard SID: uFMT uSID over dfw's locator (2001:db8:ff01::/48), nodeID=9 - name: GALACTIC_NAT_SHARD_SID value: "2001:db8:ff01:9:e001::" - # Lab-only placeholder masquerade source address, same - # "operator-supplied, no in-cluster derivation yet" status as - # GALACTIC_GATEWAY_SRV6_ADDRESS (see - # internal/config/gateway.go's doc comment). 2001:db8:9966::/32 - # is a new documentation-range (RFC 3849) block reserved here - # for NAT66 shard public addresses specifically, picked to - # avoid colliding with any address already in use elsewhere - # in this topology (transit mesh: 2001:db8:0:*::/64 and - # 2001:db8:1:*::/64; SRv6 locators: 2001:db8:ff0X::/48; the - # ns60 ingress VIP: 2001:db8:6060::1) -- "9966" as a mnemonic - # for "NAT66". ::1 here is dfw's shard (shard 1 of 3). - # - # Reachable from outside the fabric only because this node's - # fabric-router originates 2001:db8:9966:1::/64 into the underlay - # (resources/fabric-router/dfw/frr.conf.dfw-worker): the EVPN path - # EgressShardReconciler advertises for this address never leaves - # galactic-router's iBGP mesh, so without that origination a - # reply from the off-fabric host is discarded by the first - # transit router holding no route for it (#549). + # NAT66 masquerade source address, docs-range 2001:db8:9966::/32 block - name: GALACTIC_NAT_SHARD_PUB_ADDR6 value: "2001:db8:9966:1::1" - # Turning NAT64 on for this shard takes both of the next two - # together: the IPv4 masquerade source an IPv4-only destination - # sees, and the /96 whose synthesized addresses this shard - # translates down to IPv4. The prefix is fabric-wide and identical - # on every shard and on galactic-cni (which installs the tenant - # VRF's route for it); only the address is per-shard. - # - # 192.0.2.0/24 is RFC 5737 TEST-NET-1, picked for the same reason - # 2001:db8:9966::/32 was picked above -- a documentation range - # that cannot collide with anything real. ::1 is this site's - # shard, matching its IPv6 counterpart's numbering. - # - # A NAT64 reply arrives from the IPv4 internet rather than - # across the fabric, so nothing galactic-nat advertises can make - # this address reachable -- the underlay has to attract it to - # this node. That is exactly what this site's shard node now - # originates for itself (resources/fabric-router/dfw/ - # frr.conf.dfw-worker), the IPv4 half of the same origination its - # IPv6 counterpart above needs; without it, outbound translation - # works and replies never arrive (#549). The other half of the - # return path, delivering an un-translated reply back to a tenant - # on another node, is #550: the shard re-encapsulates toward the - # outer source the tenant's node stamped, which has to be that - # node's own SID. + # NAT64 masquerade source address (RFC 5737 TEST-NET-1) - name: GALACTIC_NAT_SHARD_PUB_ADDR4 value: "192.0.2.1" - name: GALACTIC_NAT_NAT64_PREFIX diff --git a/deploy/containerlab/resources/galactic-nat/iad/egressshard.yaml b/deploy/containerlab/resources/galactic-nat/iad/egressshard.yaml index 09453dc8..0e3d7159 100644 --- a/deploy/containerlab/resources/galactic-nat/iad/egressshard.yaml +++ b/deploy/containerlab/resources/galactic-nat/iad/egressshard.yaml @@ -1,11 +1,3 @@ -# Sample EgressShard for iad-worker, this cluster's own shard node -- -# alongside node-patch.yaml the same way resources/galactic-gateway/ -# /networkgateway.yaml sits alongside its own node-patch.yaml. -# Status is left empty: EgressShardReconciler (running inside the -# galactic-nat process on iad-worker itself, per node-patch.yaml's -# GALACTIC_NAT_SHARD_SID/_SHARD_PUB_ADDR6) fills in -# status.shardAddress/status.shardSID from its own config at startup -- -# this object only needs to exist and target the right node. apiVersion: network.datumapis.com/v1alpha1 kind: EgressShard metadata: diff --git a/deploy/containerlab/resources/galactic-nat/iad/kustomization.yaml b/deploy/containerlab/resources/galactic-nat/iad/kustomization.yaml index 48633544..54ca2852 100644 --- a/deploy/containerlab/resources/galactic-nat/iad/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-nat/iad/kustomization.yaml @@ -1,10 +1,3 @@ -# iad's instantiation of ../base as this site's NAT66 shard, reusing the -# already-existing iad-worker node (see node-patch.yaml and ../README.md) -# rather than a new dedicated shard-labeled node. Mirrors -# resources/galactic-router/iad/'s per-site pattern (no rename needed the -# way resources/galactic-gateway// require: dfw/iad/sjc -# are three separate clusters, so "galactic-nat" never collides with -# itself the way two DaemonSets in the same iad cluster would). namespace: galactic-system resources: - ../base diff --git a/deploy/containerlab/resources/galactic-nat/iad/node-patch.yaml b/deploy/containerlab/resources/galactic-nat/iad/node-patch.yaml index 930484ba..eef73407 100644 --- a/deploy/containerlab/resources/galactic-nat/iad/node-patch.yaml +++ b/deploy/containerlab/resources/galactic-nat/iad/node-patch.yaml @@ -5,129 +5,19 @@ metadata: spec: template: spec: - # config/galactic-nat/base/daemonset.yaml's own affinity requires - # galactic.datumapis.com/node: compute -- iad-worker - # (node_files/iad/config.yaml, worker-index 1) carries it, reusing - # that existing compute worker as this site's shard rather than - # introducing a dedicated node, per the redesign plan's own §8 - # suggestion. iad-worker3 (the route reflector, - # galactic.datumapis.com/galactic=control, not a - # galactic.datumapis.com/node value at all) and iad-worker2 (the - # tainted galactic.datumapis.com/node=edge node, see gvpc.clab.yaml - # and README.md's node table) never carry the - # compute label, so no affinity override or hostname pin is needed - # here: iad-worker is the only node in this cluster carrying it, - # and the base's own affinity already resolves to exactly it -- - # unlike resources/galactic-gateway/dfw-worker2/, which needs a - # kubernetes.io/hostname pin because iad has *two* edge-labeled - # nodes needing different per-node env. GALACTIC_NAT_SHARD_SID/ - # _SHARD_PUB_ADDR6 below must be unique per shard node (see - # config/galactic-nat/kustomization.yaml's doc comment), so - # dfw/iad/sjc are three separate per-node overlays rather than one - # overlay matching all three site workers with identical env. containers: - name: galactic-nat env: - # eth1 is this node's only transit-fabric-facing uplink (see - # resources/galactic-cni/daemonset-patch.yaml's comment) -- the - # same interface internal/plumbing/ebpf/natprog's XDP program - # attaches to. A single-homed site, so a one-element list; - # dfw's compute node is dual-homed and names both of its - # uplinks here (resources/galactic-nat/dfw/node-patch.yaml). + # single-homed node: eth1 uplink - name: GALACTIC_NAT_UPLINK_INTERFACES value: eth1 - # uFMT 48+16 uSID over iad's own locator (2001:db8:ff03::/48, - # see resources/galactic-router/iad/bgprouter.yaml's - # srv6Locator), nodeID=9 -- a dedicated Node-ID reserved for - # this shard, deliberately NOT iad-worker's own real nodeID=1 - # (an earlier version of this file reused it, on the theory - # that this shard "runs on that exact physical node, not a - # separate dedicated node"). That reuse was a real bug, found - # live: internal/plumbing/ebpf/natprog/nat.c's own - # locator_matches only checks the top 64 bits (Block+Node-ID) - # of a packet's outer destination against shard_sid -- see - # that function's own doc comment -- so reusing iad-worker's - # Node-ID meant this shard's XDP program silently hijacked - # *every* ordinary tenant ingress packet addressed to - # iad-worker's own real delivery uSIDs (same Block+Node-ID, - # nexthdr also 41) before usid_ingress's TC hook ever saw - # them. Physical co-location (which node this pod is - # scheduled to) and uSID identity (which Node-ID a shard's own - # address claims) are independent; nodeID=9 is free at this - # site (iad-worker=1, iad-worker2=2, iad-worker3=4 -- see - # resources/galactic-router/iad/, resources/galactic-control/ - # iad/, and resources/galactic-gateway/iad-worker2/'s own - # bgprouter.yaml files). Function=End.DT46 (0xE), matching - # resources/galactic-gateway/iad-worker2/node-patch.yaml's - # convention, - # even though this address is never actually looked up - # through function_table/vrf_table the way a real uSID is -- - # EgressShardStatus.ShardSID's own doc comment says it's - # advertised as a plain node-reachability route "no - # VRFID/Function" -- the uFMT encoding here only keeps the - # address inside iad's locator block. - # - # Argument=1 is a placeholder and carries no meaning: it names - # no tenant and never could, this one configured value being - # shared by every VRF on every node. Each CNI ADD overwrites - # it with that attachment's own VRFID before installing the - # tenant's egress route (internal/cnibgp's - # shardSIDsForTenant), which is what lets this shard tell two - # same-node tenants apart. What this address does have to - # reserve for itself is its whole Block+Node-ID: that /64 is - # what EgressShardReconciler advertises and what - # locator_matches compares against. - # - # Still a lab-only value needing real allocation logic (the - # same gap GALACTIC_GATEWAY_SRV6_ADDRESS and EnvNATShardSID's - # own doc comment both flag today) before production. + # shard SID: uFMT uSID over iad's locator (2001:db8:ff03::/48), nodeID=9 - name: GALACTIC_NAT_SHARD_SID value: "2001:db8:ff03:9:e001::" - # Lab-only placeholder masquerade source address, same - # "operator-supplied, no in-cluster derivation yet" status as - # GALACTIC_GATEWAY_SRV6_ADDRESS (see - # internal/config/gateway.go's doc comment). 2001:db8:9966::/32 - # is a new documentation-range (RFC 3849) block reserved here - # for NAT66 shard public addresses specifically, picked to - # avoid colliding with any address already in use elsewhere - # in this topology (transit mesh: 2001:db8:0:*::/64 and - # 2001:db8:1:*::/64; SRv6 locators: 2001:db8:ff0X::/48; the - # ns60 ingress VIP: 2001:db8:6060::1) -- "9966" as a mnemonic - # for "NAT66". ::2 here is iad's shard (shard 2 of 3). - # - # Reachable from outside the fabric only because this node's - # fabric-router originates 2001:db8:9966:2::/64 into the underlay - # (resources/fabric-router/iad/frr.conf.iad-worker): the EVPN path - # EgressShardReconciler advertises for this address never leaves - # galactic-router's iBGP mesh, so without that origination a - # reply from the off-fabric host is discarded by the first - # transit router holding no route for it (#549). + # NAT66 masquerade source address, docs-range 2001:db8:9966::/32 block - name: GALACTIC_NAT_SHARD_PUB_ADDR6 value: "2001:db8:9966:2::1" - # Turning NAT64 on for this shard takes both of the next two - # together: the IPv4 masquerade source an IPv4-only destination - # sees, and the /96 whose synthesized addresses this shard - # translates down to IPv4. The prefix is fabric-wide and identical - # on every shard and on galactic-cni (which installs the tenant - # VRF's route for it); only the address is per-shard. - # - # 192.0.2.0/24 is RFC 5737 TEST-NET-1, picked for the same reason - # 2001:db8:9966::/32 was picked above -- a documentation range - # that cannot collide with anything real. ::2 is this site's - # shard, matching its IPv6 counterpart's numbering. - # - # A NAT64 reply arrives from the IPv4 internet rather than - # across the fabric, so nothing galactic-nat advertises can make - # this address reachable -- the underlay has to attract it to - # this node. That is exactly what this site's shard node now - # originates for itself (resources/fabric-router/iad/ - # frr.conf.iad-worker), the IPv4 half of the same origination its - # IPv6 counterpart above needs; without it, outbound translation - # works and replies never arrive (#549). The other half of the - # return path, delivering an un-translated reply back to a tenant - # on another node, is #550: the shard re-encapsulates toward the - # outer source the tenant's node stamped, which has to be that - # node's own SID. + # NAT64 masquerade source address (RFC 5737 TEST-NET-1) - name: GALACTIC_NAT_SHARD_PUB_ADDR4 value: "192.0.2.2" - name: GALACTIC_NAT_NAT64_PREFIX diff --git a/deploy/containerlab/resources/galactic-nat/sjc/egressshard.yaml b/deploy/containerlab/resources/galactic-nat/sjc/egressshard.yaml index 3c4ba930..17398cc2 100644 --- a/deploy/containerlab/resources/galactic-nat/sjc/egressshard.yaml +++ b/deploy/containerlab/resources/galactic-nat/sjc/egressshard.yaml @@ -1,11 +1,3 @@ -# Sample EgressShard for sjc-worker, this cluster's own shard node -- -# alongside node-patch.yaml the same way resources/galactic-gateway/ -# /networkgateway.yaml sits alongside its own node-patch.yaml. -# Status is left empty: EgressShardReconciler (running inside the -# galactic-nat process on sjc-worker itself, per node-patch.yaml's -# GALACTIC_NAT_SHARD_SID/_SHARD_PUB_ADDR6) fills in -# status.shardAddress/status.shardSID from its own config at startup -- -# this object only needs to exist and target the right node. apiVersion: network.datumapis.com/v1alpha1 kind: EgressShard metadata: diff --git a/deploy/containerlab/resources/galactic-nat/sjc/kustomization.yaml b/deploy/containerlab/resources/galactic-nat/sjc/kustomization.yaml index 6d560176..54ca2852 100644 --- a/deploy/containerlab/resources/galactic-nat/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-nat/sjc/kustomization.yaml @@ -1,10 +1,3 @@ -# sjc's instantiation of ../base as this site's NAT66 shard, reusing the -# already-existing sjc-worker node (see node-patch.yaml and ../README.md) -# rather than a new dedicated shard-labeled node. Mirrors -# resources/galactic-router/sjc/'s per-site pattern (no rename needed the -# way resources/galactic-gateway// require: dfw/iad/sjc -# are three separate clusters, so "galactic-nat" never collides with -# itself the way two DaemonSets in the same iad cluster would). namespace: galactic-system resources: - ../base diff --git a/deploy/containerlab/resources/galactic-nat/sjc/node-patch.yaml b/deploy/containerlab/resources/galactic-nat/sjc/node-patch.yaml index e53a57f5..4468970c 100644 --- a/deploy/containerlab/resources/galactic-nat/sjc/node-patch.yaml +++ b/deploy/containerlab/resources/galactic-nat/sjc/node-patch.yaml @@ -5,124 +5,19 @@ metadata: spec: template: spec: - # config/galactic-nat/base/daemonset.yaml's own affinity requires - # galactic.datumapis.com/node: compute -- sjc-worker - # (node_files/sjc/config.yaml) carries it, reusing that existing - # compute worker as this site's shard rather than introducing a - # dedicated node, per the redesign plan's own §8 suggestion. No - # affinity override needed here: sjc-worker is the only node in - # this cluster carrying that label, so the base's own affinity - # already resolves to exactly it -- unlike - # resources/galactic-gateway/dfw-worker2/, which needs a - # kubernetes.io/hostname pin because iad has *two* edge-labeled - # nodes needing different per-node env. GALACTIC_NAT_SHARD_SID/ - # _SHARD_PUB_ADDR6 below must be unique per shard node (see - # config/galactic-nat/kustomization.yaml's doc comment), so - # dfw/iad/sjc are three separate per-node overlays rather than one - # overlay matching all three site workers with identical env. containers: - name: galactic-nat env: - # eth1 is this node's only transit-fabric-facing uplink (see - # resources/galactic-cni/daemonset-patch.yaml's comment) -- the - # same interface internal/plumbing/ebpf/natprog's XDP program - # attaches to. A single-homed site, so a one-element list; - # dfw's compute node is dual-homed and names both of its - # uplinks here (resources/galactic-nat/dfw/node-patch.yaml). + # single-homed node: eth1 uplink - name: GALACTIC_NAT_UPLINK_INTERFACES value: eth1 - # uFMT 48+16 uSID over sjc's own locator (2001:db8:ff02::/48, - # see resources/galactic-router/sjc/bgprouter.yaml's - # srv6Locator), nodeID=9 -- a dedicated Node-ID reserved for - # this shard, deliberately NOT sjc-worker's own real nodeID=1 - # (an earlier version of this file reused it, on the theory - # that this shard "runs on that exact physical node, not a - # separate dedicated node"). That reuse was a real bug, found - # live: internal/plumbing/ebpf/natprog/nat.c's own - # locator_matches only checks the top 64 bits (Block+Node-ID) - # of a packet's outer destination against shard_sid -- see - # that function's own doc comment -- so reusing sjc-worker's - # Node-ID meant this shard's XDP program silently hijacked - # *every* ordinary tenant ingress packet addressed to - # sjc-worker's own real delivery uSIDs (same Block+Node-ID, - # nexthdr also 41) before usid_ingress's TC hook ever saw - # them. Physical co-location (which node this pod is - # scheduled to) and uSID identity (which Node-ID a shard's own - # address claims) are independent; nodeID=9 is free at this - # site (sjc-worker is the only BGPRouter on this locator, - # nodeID=1 -- see resources/galactic-router/sjc/bgprouter.yaml). - # Function=End.DT46 (0xE), matching resources/galactic-gateway/ - # the edge nodes' node-patch.yaml convention, even though this - # address is never actually looked up through - # function_table/vrf_table the way a real uSID is -- - # EgressShardStatus.ShardSID's own doc comment says it's - # advertised as a plain node-reachability route "no - # VRFID/Function" -- the uFMT encoding here only keeps the - # address inside sjc's locator block (see - # resources/galactic-nat/iad/node-patch.yaml's fuller - # rationale on the Node-ID fix). - # - # Argument=1 is a placeholder and carries no meaning: it names - # no tenant and never could, this one configured value being - # shared by every VRF on every node. Each CNI ADD overwrites - # it with that attachment's own VRFID before installing the - # tenant's egress route (internal/cnibgp's - # shardSIDsForTenant), which is what lets this shard tell two - # same-node tenants apart. What this address does have to - # reserve for itself is its whole Block+Node-ID: that /64 is - # what EgressShardReconciler advertises and what - # locator_matches compares against. - # - # Still a lab-only value needing real allocation logic (the - # same gap GALACTIC_GATEWAY_SRV6_ADDRESS and EnvNATShardSID's - # own doc comment both flag today) before production. + # shard SID: uFMT uSID over sjc's locator (2001:db8:ff02::/48), nodeID=9 - name: GALACTIC_NAT_SHARD_SID value: "2001:db8:ff02:9:e001::" - # Lab-only placeholder masquerade source address, same - # "operator-supplied, no in-cluster derivation yet" status as - # GALACTIC_GATEWAY_SRV6_ADDRESS (see - # internal/config/gateway.go's doc comment). 2001:db8:9966::/32 - # is a new documentation-range (RFC 3849) block reserved here - # for NAT66 shard public addresses specifically, picked to - # avoid colliding with any address already in use elsewhere - # in this topology (transit mesh: 2001:db8:0:*::/64 and - # 2001:db8:1:*::/64; SRv6 locators: 2001:db8:ff0X::/48; the - # ns60 ingress VIP: 2001:db8:6060::1) -- "9966" as a mnemonic - # for "NAT66". ::3 here is sjc's shard (shard 3 of 3). - # - # Reachable from outside the fabric only because this node's - # fabric-router originates 2001:db8:9966:3::/64 into the underlay - # (resources/fabric-router/sjc/frr.conf.sjc-worker): the EVPN path - # EgressShardReconciler advertises for this address never leaves - # galactic-router's iBGP mesh, so without that origination a - # reply from the off-fabric host is discarded by the first - # transit router holding no route for it (#549). + # NAT66 masquerade source address, docs-range 2001:db8:9966::/32 block - name: GALACTIC_NAT_SHARD_PUB_ADDR6 value: "2001:db8:9966:3::1" - # Turning NAT64 on for this shard takes both of the next two - # together: the IPv4 masquerade source an IPv4-only destination - # sees, and the /96 whose synthesized addresses this shard - # translates down to IPv4. The prefix is fabric-wide and identical - # on every shard and on galactic-cni (which installs the tenant - # VRF's route for it); only the address is per-shard. - # - # 192.0.2.0/24 is RFC 5737 TEST-NET-1, picked for the same reason - # 2001:db8:9966::/32 was picked above -- a documentation range - # that cannot collide with anything real. ::3 is this site's - # shard, matching its IPv6 counterpart's numbering. - # - # A NAT64 reply arrives from the IPv4 internet rather than - # across the fabric, so nothing galactic-nat advertises can make - # this address reachable -- the underlay has to attract it to - # this node. That is exactly what this site's shard node now - # originates for itself (resources/fabric-router/sjc/ - # frr.conf.sjc-worker), the IPv4 half of the same origination its - # IPv6 counterpart above needs; without it, outbound translation - # works and replies never arrive (#549). The other half of the - # return path, delivering an un-translated reply back to a tenant - # on another node, is #550: the shard re-encapsulates toward the - # outer source the tenant's node stamped, which has to be that - # node's own SID. + # NAT64 masquerade source address (RFC 5737 TEST-NET-1) - name: GALACTIC_NAT_SHARD_PUB_ADDR4 value: "192.0.2.3" - name: GALACTIC_NAT_NAT64_PREFIX diff --git a/deploy/containerlab/resources/tenants/base/kustomization.yaml b/deploy/containerlab/resources/tenants/base/kustomization.yaml index 24772e73..d6476286 100644 --- a/deploy/containerlab/resources/tenants/base/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/base/kustomization.yaml @@ -1,8 +1,3 @@ -# Shared by every tenant under resources/tenants//base/ — a bare -# Namespace + netshoot Deployment. Each tenant's own base/kustomization.yaml -# sets `namespace:` (renames the Namespace and injects metadata.namespace -# everywhere else) and patches the default-network annotation and, for the -# two single-site tenants, `replicas:`. resources: - namespace.yaml - pod.yaml diff --git a/deploy/containerlab/resources/tenants/base/namespace.yaml b/deploy/containerlab/resources/tenants/base/namespace.yaml index 57d6f6d3..b0eac6b9 100644 --- a/deploy/containerlab/resources/tenants/base/namespace.yaml +++ b/deploy/containerlab/resources/tenants/base/namespace.yaml @@ -1,7 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - # Overwritten by each tenant's base/kustomization.yaml `namespace:` field — - # kustomize's namespace transformer renames a Namespace-kind resource's own - # metadata.name to match, not just metadata.namespace on everything else. name: tenant-placeholder diff --git a/deploy/containerlab/resources/tenants/base/pod.yaml b/deploy/containerlab/resources/tenants/base/pod.yaml index f331618e..5d2a6f85 100644 --- a/deploy/containerlab/resources/tenants/base/pod.yaml +++ b/deploy/containerlab/resources/tenants/base/pod.yaml @@ -14,11 +14,6 @@ spec: labels: app: private annotations: - # Placeholder — Multus resolves an unqualified default-network name - # against the pod's own namespace only when nothing else claims that - # role; in this lab it resolves against kube-system instead (verified - # empirically), so every tenant's base/kustomization.yaml patches - # this to the namespace-qualified "/private" form. v1.multus-cni.io/default-network: private spec: affinity: diff --git a/deploy/containerlab/resources/tenants/ns30/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns30/base/kustomization.yaml index 51306733..22dd3412 100644 --- a/deploy/containerlab/resources/tenants/ns30/base/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns30/base/kustomization.yaml @@ -1,15 +1,3 @@ -# ns30 is single-site (dfw only, see ../dfw/). This is the first of its two -# attachments — ../dfw/pod-b.yaml is the second, on its own NAD -# (ns30/private-b). They deliberately use two distinct vpcattachment values -# under the same vpc rather than one NAD scaled to replicas: 2: galactic-veth -# derives its host/guest veth interface names from (vpc, vpcAttachment) -# alone (internal/plumbing/intf), so two pods sharing one vpcAttachment on -# the same node collide on that name — internal/cni/veth's "stale veth" -# self-heal would delete whichever pod's veth got there second, silently -# breaking it (see the incident this fixture layout replaced). Two distinct -# vpcattachments avoids that collision and lands both pods in one shared VRF -# (internal/plumbing/vrf is keyed by vpc alone) for same-node, same-VPC -# pod-to-pod connectivity testing rather than cross-site. namespace: ns30 resources: - ../../base diff --git a/deploy/containerlab/resources/tenants/ns30/dfw/pod-b.yaml b/deploy/containerlab/resources/tenants/ns30/dfw/pod-b.yaml index 615f2a7f..e9774722 100644 --- a/deploy/containerlab/resources/tenants/ns30/dfw/pod-b.yaml +++ b/deploy/containerlab/resources/tenants/ns30/dfw/pod-b.yaml @@ -1,10 +1,3 @@ -# The second of ns30's two attachments — see ../base/kustomization.yaml's -# doc comment for why this is a standalone Deployment on its own NAD -# (nad-b.yaml, vpcattachment "31") rather than a second replica of ../base's -# Deployment. Mirrors ../../base/pod.yaml (the shared tenant Deployment -# template) directly rather than composing it through kustomize, since a -# second copy needs a different name/label/annotation than kustomize's -# nameSuffix + patch ordering can reliably guarantee. apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/containerlab/resources/tenants/ns40/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns40/base/kustomization.yaml index 17513ebf..920ef186 100644 --- a/deploy/containerlab/resources/tenants/ns40/base/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns40/base/kustomization.yaml @@ -1,13 +1,3 @@ -# ns40 is single-site (iad only, see ../iad/), landing on iad-worker (the -# only untainted worker in that cluster). This is the first of its two -# attachments — ../iad/pod-b.yaml is the second, on its own NAD -# (ns40/private-b). They deliberately use two distinct vpcattachment values -# under the same vpc rather than one NAD scaled to replicas: 2 — see -# ../../ns30/base/kustomization.yaml's doc comment for why (galactic-veth's -# host/guest veth naming collides when two pods share one vpcattachment on -# the same node). Two distinct vpcattachments avoids that collision and -# lands both pods in one shared VRF for same-node, same-VPC pod-to-pod -# connectivity testing rather than cross-site. namespace: ns40 resources: - ../../base diff --git a/deploy/containerlab/resources/tenants/ns40/iad/pod-b.yaml b/deploy/containerlab/resources/tenants/ns40/iad/pod-b.yaml index abc4a8f3..7390ba2d 100644 --- a/deploy/containerlab/resources/tenants/ns40/iad/pod-b.yaml +++ b/deploy/containerlab/resources/tenants/ns40/iad/pod-b.yaml @@ -1,10 +1,3 @@ -# The second of ns40's two attachments — see ../base/kustomization.yaml's -# doc comment for why this is a standalone Deployment on its own NAD -# (nad-b.yaml, vpcattachment "41") rather than a second replica of ../base's -# Deployment. Mirrors ../../base/pod.yaml (the shared tenant Deployment -# template) directly rather than composing it through kustomize, since a -# second copy needs a different name/label/annotation than kustomize's -# nameSuffix + patch ordering can reliably guarantee. apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/containerlab/resources/tenants/ns60/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns60/base/kustomization.yaml index 92587c41..fa1c0b83 100644 --- a/deploy/containerlab/resources/tenants/ns60/base/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns60/base/kustomization.yaml @@ -1,12 +1,3 @@ -# ns60's own base — unlike every other tenant, ns60 doesn't reuse -# ../../base's netshoot Deployment: it exists to expose a real web service -# through the edge XDP NAT+LB gateway (see -# resources/galactic-gateway/iad/networkrule-ns60.yaml), so its pod -# is an nginx server instead. namespace.yaml is a local duplicate of -# ../../base/namespace.yaml rather than a direct reference to it — see that -# file's own doc comment for why. pod.yaml here is ns60's own, already -# namespace-qualified in its multus annotation, so no patches: block is -# needed the way the netshoot-based tenants require. namespace: ns60 resources: - namespace.yaml diff --git a/deploy/containerlab/resources/tenants/ns60/base/namespace.yaml b/deploy/containerlab/resources/tenants/ns60/base/namespace.yaml index 77e0761b..b0eac6b9 100644 --- a/deploy/containerlab/resources/tenants/ns60/base/namespace.yaml +++ b/deploy/containerlab/resources/tenants/ns60/base/namespace.yaml @@ -1,14 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - # Overwritten by this directory's kustomization.yaml `namespace:` field, - # exactly like ../../base/namespace.yaml (the shared placeholder every - # netshoot-based tenant reuses). Duplicated here rather than referenced - # directly (`../../base/namespace.yaml`) because kustomize's default load - # restrictor only allows a raw resource *file* reference to resolve - # within the referencing kustomization's own root — unlike referencing an - # entire sibling *kustomization directory* (what every other tenant's - # base/kustomization.yaml does via `../../base`), which ns60 can't do - # here without also pulling in that shared kustomization's netshoot - # pod.yaml. name: tenant-placeholder diff --git a/deploy/containerlab/resources/tenants/ns60/base/pod.yaml b/deploy/containerlab/resources/tenants/ns60/base/pod.yaml index bdf3b14d..e0bcb29b 100644 --- a/deploy/containerlab/resources/tenants/ns60/base/pod.yaml +++ b/deploy/containerlab/resources/tenants/ns60/base/pod.yaml @@ -14,10 +14,6 @@ spec: labels: app: private annotations: - # Namespace-qualified for the same reason as tenants/base/pod.yaml's - # identical annotation: Multus resolves an unqualified - # default-network name against kube-system in this lab, not the - # pod's own namespace (verified empirically). v1.multus-cni.io/default-network: ns60/private spec: affinity: diff --git a/deploy/containerlab/resources/tenants/ns70/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns70/base/kustomization.yaml index 35000891..ab727eaf 100644 --- a/deploy/containerlab/resources/tenants/ns70/base/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns70/base/kustomization.yaml @@ -1,19 +1,3 @@ -# ns70 exists to exercise one specific datapath property: two different -# tenants whose pods hold the *same* ULA, on different worker nodes, egressing -# through the same shard. -# -# Every other tenant here is given a site-unique ipv6_subnet, so no two of -# them ever present the same inner source address to a shard and the -# collision path is never reached. ns70 deliberately gives iad and sjc the -# identical subnet. ULAs are RFC 4193 locally-assigned and not globally -# coordinated, so two tenants picking the same one is a real possibility -# rather than a contrived one -- it is precisely the case the SRv6 Argument -# was meant to disambiguate, and the case the connection key's encapsulation -# source now covers. -# -# iad and sjc are the two sites on purpose: both resolve dfw's shard SID -# first (a node cannot resolve its own self-originated SID), so both land on -# dfw-worker's shard and arrive there encapsulated from different nodes. namespace: ns70 resources: - ../../base diff --git a/deploy/containerlab/resources/tenants/ns71/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns71/base/kustomization.yaml index ed7a3c56..e5118d82 100644 --- a/deploy/containerlab/resources/tenants/ns71/base/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns71/base/kustomization.yaml @@ -1,31 +1,3 @@ -# ns71 exists to exercise one specific datapath property: two different -# tenants whose pods hold the *same* ULA, on the *same* worker node, -# egressing through the same shard. -# -# It is the same-node counterpart to ns70. ns70 puts its two pods on -# different nodes, where the connection key's encapsulation source tells -# them apart on its own; here both flows arrive at the shard from one node -# and carry an identical encapsulation source, so the SRv6 Argument is the -# only field left that can separate them. That is what #538 was about: the -# CNI registered the operator-configured shard SID verbatim, every VRF on a -# node encapsulated toward a byte-identical destination, and two tenants -# whose inner tuples also matched shared one connection row and one -# masquerade port -- the second tenant's replies delivered to the first. -# -# The two attachments are two distinct VPCs (71 and 72), not two -# attachments of one VPC: a shared VRF would be one tenant, which is not -# the case under test. -# -# Both pods take their address from ipam.addresses rather than a pool, -# which is what makes the collision exist by construction. The pool path -# cannot produce it: internal/cni/ipam keys its on-disk allocation state by -# the pool CIDR alone (node-local, not per-VPC), so two VPCs configured -# with the same subnet on one node are handed two *different* /96s and -# never collide. The values here are byte-for-byte what that pool path -# would hand the first pod out of fd20:71:ffff::/48 -- gateway at the -# pool's first address, pod in the first /96 after the reserved one -- so -# nothing downstream of IPAM behaves differently from any other tenant in -# this lab. namespace: ns71 resources: - ../../base diff --git a/deploy/containerlab/resources/tenants/ns71/sjc/kustomization.yaml b/deploy/containerlab/resources/tenants/ns71/sjc/kustomization.yaml index fb2332f0..eb4a2e90 100644 --- a/deploy/containerlab/resources/tenants/ns71/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns71/sjc/kustomization.yaml @@ -1,11 +1,3 @@ -# sjc is the site on purpose, for two independent reasons. It has exactly -# one schedulable worker -- sjc-worker, the compute node; sjc-worker2 is -# edge-labeled and NoSchedule-tainted (node_files/sjc/config.yaml) -- so -# "same node" is structural here rather than a scheduling coincidence, and -# the nodeSelector in ../base/kustomization.yaml states it anyway. And a -# node cannot resolve its own site's self-originated shard SID, so both -# pods land on dfw's shard, which is where verify:nat-collision-samenode -# captures. resources: - ../base - nad.yaml diff --git a/deploy/containerlab/resources/tenants/ns71/sjc/nad-b.yaml b/deploy/containerlab/resources/tenants/ns71/sjc/nad-b.yaml index 72caeebd..b859a46f 100644 --- a/deploy/containerlab/resources/tenants/ns71/sjc/nad-b.yaml +++ b/deploy/containerlab/resources/tenants/ns71/sjc/nad-b.yaml @@ -1,8 +1,4 @@ --- -# The second of ns71's two tenants: a different VPC (72, so a different VRF -# and a different Argument on the same node), holding the identical address -# nad.yaml's pod holds. Every field of the shard's connection key matches -# across the two except the Argument -- see ../base/kustomization.yaml. apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition metadata: diff --git a/deploy/containerlab/resources/tenants/ns71/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns71/sjc/nad.yaml index b5e964c9..bf75bbcd 100644 --- a/deploy/containerlab/resources/tenants/ns71/sjc/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns71/sjc/nad.yaml @@ -1,7 +1,4 @@ --- -# The first of ns71's two tenants. Its address and gateway are identical to -# nad-b.yaml's by design -- see ../base/kustomization.yaml for why that -# collision is stated here rather than left to IPAM. apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition metadata: diff --git a/deploy/containerlab/resources/tenants/ns71/sjc/pod-b.yaml b/deploy/containerlab/resources/tenants/ns71/sjc/pod-b.yaml index e00bd08b..92a79249 100644 --- a/deploy/containerlab/resources/tenants/ns71/sjc/pod-b.yaml +++ b/deploy/containerlab/resources/tenants/ns71/sjc/pod-b.yaml @@ -1,13 +1,3 @@ -# The second of ns71's two tenants -- see ../base/kustomization.yaml's doc -# comment for what this fixture is for. Mirrors ../../base/pod.yaml (the -# shared tenant Deployment template) directly rather than composing it -# through kustomize, for the same reason ns30/dfw/pod-b.yaml does: a second -# copy needs a different name/label/annotation than kustomize's nameSuffix + -# patch ordering can reliably guarantee. -# -# The nodeSelector is the point of the fixture, not a scheduling -# convenience: both pods must land on one node for their flows to reach the -# shard with an identical encapsulation source. apiVersion: apps/v1 kind: Deployment metadata: