Keep large responses from stalling on VPC networks

api/v1alpha/network_types.go

@@ -30,10 +30,15 @@ type NetworkSpec struct {
 
 	// Network MTU. May be between 1300 and 8856.
 	//
+	// Defaults to 1440. Traffic between locations is encapsulated with a
+	// 40-byte outer IPv6 header, and some provider paths drop larger frames
+	// without returning Packet Too Big, so a larger MTU can hang connections
+	// instead of fragmenting or failing fast.
+	//
 	// +kubebuilder:validation:Minimum=1300
 	// +kubebuilder:validation:Maximum=8856
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default=1460
+	// +kubebuilder:default=1440
 	MTU int32 `json:"mtu,omitempty"`
 }
 

config/crd/bases/networking.datumapis.com_networks.yaml

@@ -95,8 +95,14 @@ spec:
                 - mode
                 type: object
               mtu:
-                default: 1460
-                description: Network MTU. May be between 1300 and 8856.
+                default: 1440
+                description: |-
+                  Network MTU. May be between 1300 and 8856.
+
+                  Defaults to 1440. Traffic between locations is encapsulated with a
+                  40-byte outer IPv6 header, and some provider paths drop larger frames
+                  without returning Packet Too Big, so a larger MTU can hang connections
+                  instead of fragmenting or failing fast.
                 format: int32
                 maximum: 8856
                 minimum: 1300

docs/api/networks.md

@@ -106,10 +106,15 @@ NetworkSpec defines the desired state of a Network
         <td><b>mtu</b></td>
         <td>integer</td>
         <td>
-          Network MTU. May be between 1300 and 8856.<br/>
+          Network MTU. May be between 1300 and 8856.
+
+Defaults to 1440. Traffic between locations is encapsulated with a
+40-byte outer IPv6 header, and some provider paths drop larger frames
+without returning Packet Too Big, so a larger MTU can hang connections
+instead of fragmenting or failing fast.<br/>
           <br/>
             <i>Format</i>: int32<br/>
-            <i>Default</i>: 1460<br/>
+            <i>Default</i>: 1440<br/>
             <i>Minimum</i>: 1300<br/>
             <i>Maximum</i>: 8856<br/>
         </td>

docs/enhancements/network-in-every-location.md

@@ -560,7 +560,7 @@ spec:
   ipFamilies:
     - IPv6
     - IPv4
-  mtu: 1460
+  mtu: 1440
 
   # The Network generation these were read from. An operator comparing this to
   # the Network answers "has this location caught up" without guessing.

docs/enhancements/network-interfaces.md

@@ -209,7 +209,7 @@ spec:
     uid: 8c1d…
   interfaceName: eth0
   attachmentMode: Hypervisor
-  mtu: 1460
+  mtu: 1440
   reclaimPolicy: Retain
   addresses:
     - family: IPv6   address: fd20:a1b:2c3d:1:0:1::/96   gateway: fd20:a1b:2c3d:1::1   primary: true
@@ -362,7 +362,7 @@ spec:
   interfaceName: eth0
 
   # Resolved from Network.spec.mtu, so a provider never reads the network.
-  mtu: 1460
+  mtu: 1440
 
   # The addresses inside the network. One entry per family, exactly one primary.
   # For IPv6 this is the endpoint's whole /96 block, not a single address; the

internal/controller/network_controller_test.go

@@ -93,7 +93,7 @@ func (s *networkScenario) createNetwork(families ...networkingv1alpha.IPFamily)
 	network.Spec = networkingv1alpha.NetworkSpec{
 		IPAM:       networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto},
 		IPFamilies: families,
-		MTU:        1460,
+		MTU:        1440,
 	}
 	require.NoError(s.t, s.client.Create(s.ctx, network))
 	return network

internal/controller/networkcontext_controller_test.go

@@ -86,7 +86,7 @@ func (s *networkContextScenario) createContext(families ...networkingv1alpha.IPF
 		Network:    networkingv1alpha.LocalNetworkRef{Name: testNetworkName},
 		Location:   networkingv1alpha.LocationReference{Name: testLocationName},
 		IPFamilies: families,
-		MTU:        1460,
+		MTU:        1440,
 	}
 	require.NoError(s.t, s.client.Create(s.ctx, networkContext))
 	return networkContext
@@ -296,7 +296,7 @@ func TestNetworkContextSubnetLiesInsideTheNetworkRange(t *testing.T) {
 	network.Spec = networkingv1alpha.NetworkSpec{
 		IPAM:       networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto},
 		IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol},
-		MTU:        1460,
+		MTU:        1440,
 	}
 	require.NoError(t, s.client.Create(s.ctx, network))
 
@@ -580,7 +580,7 @@ func TestDeletingANetworkTakesDownTheLocationsHoldingItsSubnets(t *testing.T) {
 	network.Spec = networkingv1alpha.NetworkSpec{
 		IPAM:       networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto},
 		IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol},
-		MTU:        1460,
+		MTU:        1440,
 	}
 	require.NoError(t, s.client.Create(s.ctx, network))
 
@@ -639,7 +639,7 @@ func TestANetworkTakesDownOnlyItsOwnLocations(t *testing.T) {
 		network.Spec = networkingv1alpha.NetworkSpec{
 			IPAM:       networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto},
 			IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol},
-			MTU:        1460,
+			MTU:        1440,
 		}
 		require.NoError(t, cl.Create(ctx, network))
 		return network

internal/controller/networkinterface_projection_test.go

@@ -193,7 +193,7 @@ func (v *visibility) interfaceOnCell() *networkingv1alpha.NetworkInterface {
 		Network:       networkingv1alpha.LocalNetworkRef{Name: "default"},
 		ClaimRef:      &networkingv1alpha.NetworkInterfaceClaimRef{Name: name},
 		InterfaceName: "eth0",
-		MTU:           1460,
+		MTU:           1440,
 		Addresses: []networkingv1alpha.NetworkInterfaceAddress{{
 			Family:  networkingv1alpha.IPv6Protocol,
 			Address: "fd20:1abc:2def:1::/96",
@@ -286,7 +286,7 @@ func TestInterfaceReachesTheProjectControlPlane(t *testing.T) {
 
 	require.Equal(t, "default", copied.Spec.Network.Name)
 	require.Equal(t, "eth0", copied.Spec.InterfaceName)
-	require.Equal(t, int32(1460), copied.Spec.MTU)
+	require.Equal(t, int32(1440), copied.Spec.MTU)
 	require.Equal(t, "fd20:1abc:2def:1::/96", copied.Spec.Addresses[0].Address)
 	require.Equal(t, "fd20:1abc:2def:1::1", copied.Spec.Addresses[0].Gateway)
 	require.Equal(t, "198.51.100.11", copied.Spec.ExternalAddresses[0].Address)
@@ -411,11 +411,11 @@ func TestEditingACopyDoesNotSurvive(t *testing.T) {
 	v.handToProject()
 
 	copied, _ = v.projectCopy()
-	require.Equal(t, int32(1460), copied.Spec.MTU, "the cell stays the only writer")
+	require.Equal(t, int32(1440), copied.Spec.MTU, "the cell stays the only writer")
 
 	var onCell networkingv1alpha.NetworkInterface
 	require.NoError(t, v.cell.Get(v.ctx, client.ObjectKey{Namespace: v.cellNamespace, Name: boundInterfaceName}, &onCell))
-	require.Equal(t, int32(1460), onCell.Spec.MTU, "an edit to a copy never reaches the cell")
+	require.Equal(t, int32(1440), onCell.Spec.MTU, "an edit to a copy never reaches the cell")
 }
 
 func TestDeletingOnTheCellRemovesBothCopies(t *testing.T) {

internal/controller/networkinterfaceclaim_controller_test.go

@@ -447,7 +447,7 @@ func newScenario(t *testing.T, labelled bool, networkFamilies []networkingv1alph
 	network.Spec = networkingv1alpha.NetworkSpec{
 		IPAM:       networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto},
 		IPFamilies: networkFamilies,
-		MTU:        1460,
+		MTU:        1440,
 	}
 	require.NoError(t, cl.Create(ctx, network))
 
@@ -471,7 +471,7 @@ func newScenario(t *testing.T, labelled bool, networkFamilies []networkingv1alph
 	// The claim reconciler runs in a cell and reads the propagated context, not
 	// the network. Families are set explicitly throughout so each case names the
 	// families it exercises rather than inheriting either default.
-	s.createNetworkContext("default", networkFamilies, 1460)
+	s.createNetworkContext("default", networkFamilies, 1440)
 
 	return s
 }
@@ -642,7 +642,7 @@ func TestNetworkInterfaceClaimBindsDualStack(t *testing.T) {
 
 	iface, err := s.getInterface("web-0-eth0")
 	require.NoError(t, err)
-	require.Equal(t, int32(1460), iface.Spec.MTU)
+	require.Equal(t, int32(1440), iface.Spec.MTU)
 	require.Equal(t, "eth0", iface.Spec.InterfaceName)
 	require.Equal(t, networkingv1alpha.NetworkInterfacePhaseBound, iface.Status.Phase)
 	require.NotNil(t, iface.Spec.ClaimRef)
@@ -1176,10 +1176,10 @@ func TestAdoptionRefusesAnInterfaceOnAnotherNetwork(t *testing.T) {
 	other.Spec = networkingv1alpha.NetworkSpec{
 		IPAM:       networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto},
 		IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol},
-		MTU:        1460,
+		MTU:        1440,
 	}
 	require.NoError(t, s.client.Create(s.ctx, other))
-	s.createNetworkContext("other", []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, 1460)
+	s.createNetworkContext("other", []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, 1440)
 
 	spec := networkingv1alpha.NetworkInterfaceClaimSpec{
 		InterfaceName: "eth0",
@@ -1669,7 +1669,7 @@ func TestNetworkInterfaceClaimTakesMTUFromTheNetworkContext(t *testing.T) {
 	iface, err := s.getInterface("jumbo")
 	require.NoError(t, err)
 	require.Equal(t, int32(1500), iface.Spec.MTU,
-		"the context carries the MTU, and the network the cell cannot read carries 1460")
+		"the context carries the MTU, and the network the cell cannot read carries 1440")
 }
 
 // A network that has not reached the location is a different answer from a

internal/controller/networkpresence_controller_test.go

@@ -118,7 +118,7 @@ func newPresenceScenario(t *testing.T, opts presenceOptions) *presenceScenario {
 		}
 		mtu := opts.mtu
 		if mtu == 0 {
-			mtu = 1460
+			mtu = 1440
 		}
 
 		network := &networkingv1alpha.Network{}
@@ -450,7 +450,7 @@ func TestNetworkPresenceLeavesTheReplicatedHubCopyAlone(t *testing.T) {
 	replicated.Spec.Network = networkingv1alpha.LocalNetworkRef{Name: s.networkName}
 	replicated.Spec.Location = networkingv1alpha.LocationReference{Name: s.locationName}
 	replicated.Spec.IPFamilies = []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}
-	replicated.Spec.MTU = 1460
+	replicated.Spec.MTU = 1440
 	require.NoError(t, s.hub.Create(s.ctx, replicated))
 
 	s.reconcile()
@@ -546,14 +546,14 @@ func TestNetworkPresenceIsTornDownByTheLastConsumerGoingAway(t *testing.T) {
 func TestNetworkPresenceConvergesOnANetworkEdit(t *testing.T) {
 	s := newPresenceScenario(t, presenceOptions{
 		families: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol},
-		mtu:      1460,
+		mtu:      1440,
 	})
 	s.createBinding("consumer-a")
 	s.reconcile()
 
 	networkContext, ok := s.networkContext()
 	require.True(t, ok)
-	require.Equal(t, int32(1460), networkContext.Spec.MTU)
+	require.Equal(t, int32(1440), networkContext.Spec.MTU)
 	generationBefore := networkContext.Spec.NetworkGeneration
 
 	s.network.Spec.MTU = 8856

internal/controller/networkpresence_sync_test.go

@@ -68,7 +68,7 @@ func drainPresenceEvents(t *testing.T, events chan event.GenericEvent) []client.
 func TestNetworkPresenceSyncEnqueuesANetworkEditAgainstAReadyContext(t *testing.T) {
 	s := newPresenceScenario(t, presenceOptions{
 		families: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol},
-		mtu:      1460,
+		mtu:      1440,
 	})
 	s.createBinding("consumer-a")
 	s.reconcile()

internal/webhook/v1alpha/network_webhook_test.go

@@ -17,7 +17,7 @@ func networkWithFamilies(families ...networkingv1alpha.IPFamily) *networkingv1al
 		Spec: networkingv1alpha.NetworkSpec{
 			IPAM:       networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto},
 			IPFamilies: families,
-			MTU:        1460,
+			MTU:        1440,
 		},
 	}
 }

test/e2e-edge/networkservice-endpoints/chainsaw-test.yaml

@@ -92,7 +92,7 @@ spec:
                   name: us-central-1
                 ipFamilies:
                   - IPv4
-                mtu: 1460
+                mtu: 1440
 
     - name: Three claims bind and their interfaces carry what a service selects
       description: |
@@ -717,7 +717,7 @@ spec:
                   name: us-central-1
                 ipFamilies:
                   - IPv4
-                mtu: 1460
+                mtu: 1440
         - create:
             resource:
               apiVersion: networking.datumapis.com/v1alpha

test/e2e/network-ipv6-prefix/chainsaw-test.yaml

@@ -122,7 +122,7 @@ spec:
                   name: us-central-1
                 ipFamilies:
                   - IPv6
-                mtu: 1460
+                mtu: 1440
         - create:
             resource:
               apiVersion: networking.datumapis.com/v1alpha

test/e2e/network-presence-network-edit/chainsaw-test.yaml

@@ -48,7 +48,7 @@ spec:
                   mode: Auto
                 ipFamilies:
                   - IPv6
-                mtu: 1460
+                mtu: 1440
 
     - name: Make the network present in the location
       try:
@@ -80,7 +80,7 @@ spec:
                 name: presence-edit-net-presence-edit-loc
                 namespace: presence-edit-project
               spec:
-                mtu: 1460
+                mtu: 1440
                 ipFamilies:
                   - IPv6
 

test/e2e/networkcontext-ipv6-subnet/chainsaw-test.yaml

@@ -50,7 +50,7 @@ spec:
                   name: us-central-1
                 ipFamilies:
                   - IPv6
-                mtu: 1460
+                mtu: 1440
         - assert:
             timeout: 120s
             resource:

test/e2e/networkinterface-project-visibility/chainsaw-test.yaml

@@ -32,7 +32,7 @@ spec:
                   name: us-central-1
                 ipFamilies:
                   - IPv6
-                mtu: 1460
+                mtu: 1440
 
     - name: A bound interface is published to the hub
       try:
@@ -87,7 +87,7 @@ spec:
                 network:
                   name: nic-visible-net
                 interfaceName: eth0
-                mtu: 1460
+                mtu: 1440
       catch:
         - script:
             content: |

test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml

@@ -43,7 +43,7 @@ spec:
                 ipFamilies:
                   - IPv4
                   - IPv6
-                mtu: 1460
+                mtu: 1440
 
     - name: A bound claim holds three addresses
       try:

test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml

@@ -37,7 +37,7 @@ spec:
                 ipFamilies:
                   - IPv4
                   - IPv6
-                mtu: 1460
+                mtu: 1440
 
     - name: Create a dual-stack network
       try:

test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml

@@ -37,7 +37,7 @@ spec:
                 ipFamilies:
                   - IPv6
                   - IPv4
-                mtu: 1460
+                mtu: 1440
 
     - name: A claim naming datum-public-v4 binds
       try:

test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml

@@ -42,7 +42,7 @@ spec:
                   name: us-central-1
                 ipFamilies:
                   - IPv6
-                mtu: 1460
+                mtu: 1440
 
     - name: A claim asking for IPv4 on an IPv6 network is refused
       try:

test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml

@@ -41,7 +41,7 @@ spec:
                 ipFamilies:
                   - IPv4
                   - IPv6
-                mtu: 1460
+                mtu: 1440
 
     - name: A retained claim binds and its interface carries a finalizer
       description: |

test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml

@@ -42,7 +42,7 @@ spec:
                 ipFamilies:
                   - IPv4
                   - IPv6
-                mtu: 1460
+                mtu: 1440
 
     - name: A claim holds a family address and a named class
       try:

test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml

@@ -43,7 +43,7 @@ spec:
                 ipFamilies:
                   - IPv6
                   - IPv4
-                mtu: 1460
+                mtu: 1440
 
     - name: A claim binds and holds one address
       try: