fix(iac/aws): route private egress through a persistent fck-nat ENI

terraform/environments/aws/ci-cd-permissions/README.md

@@ -9,7 +9,7 @@ CI/CD pipeline uses to deploy infrastructure on AWS. It optionally sets up keyle
 | Resource | Purpose |
 | --- | --- |
 | `aws_iam_role.cudly_deploy` | The deploy role; assumed by GitHub Actions or a human operator |
-| `aws_iam_policy.networking` | VPC, subnets, security groups, ALB, ECS cluster |
+| `aws_iam_policy.networking` | VPC, subnets, security groups, ALB, ECS cluster, and the persistent fck-nat ENI (`ec2:CreateNetworkInterface`, `ec2:ModifyNetworkInterfaceAttribute`, `ec2:DetachNetworkInterface`, added for #716; re-apply before merging the change that creates it) |
 | `aws_iam_policy.compute` | ECS services/tasks, ECR, CloudWatch Logs, SSM |
 | `aws_iam_policy.compute_b` | Overflow for `aws_iam_policy.compute`, which is close to the 6144-character managed-policy limit |
 | `aws_iam_policy.data` | RDS, ElastiCache, S3 (state bucket), Secrets Manager |

terraform/environments/aws/ci-cd-permissions/fck_nat_guard_test.go

@@ -0,0 +1,167 @@
+package cicdpermissions_test
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+// networkInterfaceActions are the EC2 actions the module's
+// aws_network_interface.fck_nat (#716) needs on the deploy role. The
+// ModifyNetworkInterfaceAttribute call is easy to miss: the provider turns
+// source_dest_check off in a second call after the create. Without any one of
+// these, plan stays green and the first apply that creates, updates or
+// replaces the ENI fails with AccessDenied.
+var networkInterfaceActions = []string{
+	"ec2:CreateNetworkInterface",
+	"ec2:DeleteNetworkInterface",
+	"ec2:DescribeNetworkInterfaces",
+	"ec2:ModifyNetworkInterfaceAttribute",
+	"ec2:DetachNetworkInterface",
+	"ec2:CreateTags",
+	"ec2:DeleteTags",
+}
+
+func TestDeployPolicyAllowsFckNatNetworkInterface(t *testing.T) {
+	var allowed []string
+	var createTagsStmts []string
+	for _, file := range deployPolicyFiles(t) {
+		for _, stmt := range findStatements(t, file, func(s string) bool { return statementEffect(s) == "Allow" }) {
+			actions := statementActions(stmt)
+			allowed = append(allowed, actions...)
+			if actionPermitted(actions, "ec2:CreateTags") {
+				createTagsStmts = append(createTagsStmts, stmt)
+			}
+		}
+	}
+	for _, action := range networkInterfaceActions {
+		if !actionPermitted(allowed, action) {
+			t.Errorf("no deploy policy document allows %s, which aws_network_interface.fck_nat needs (#716)", action)
+		}
+	}
+
+	// A CreateTags grant conditioned on ec2:CreateAction must include
+	// CreateNetworkInterface, or tagging the ENI at creation is denied.
+	// At least one grant must also be usable for it.
+	usable := false
+	for _, stmt := range createTagsStmts {
+		cond := statementConditionBody(stmt)
+		if cond == "" || (strings.Contains(cond, "ec2:CreateAction") && strings.Contains(cond, "CreateNetworkInterface")) {
+			usable = true
+			continue
+		}
+		if strings.Contains(cond, "ec2:CreateAction") {
+			t.Errorf("statement %q grants ec2:CreateTags conditioned on ec2:CreateAction without CreateNetworkInterface; tagging the fck-nat ENI at create time would be denied", statementSid(stmt))
+		}
+	}
+	if !usable {
+		t.Errorf("no ec2:CreateTags grant is unconditioned or allows ec2:CreateAction=CreateNetworkInterface")
+	}
+}
+
+// resourceBlock returns the body of `resource "type" "name" {...}` from
+// comment-stripped HCL, or `resource "type" {...}` when name is empty.
+func resourceBlock(t *testing.T, src, typ, name string) string {
+	t.Helper()
+	head := `resource "` + typ + `"`
+	if name != "" {
+		head += ` "` + name + `"`
+	}
+	re := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(head) + ` \{`)
+	loc := re.FindStringIndex(src)
+	if loc == nil {
+		t.Fatalf("block %s not found in networking main.tf", head)
+	}
+	open := loc[1] - 1
+	return src[open:balancedBraceEnd(src, open)]
+}
+
+// bracketValue returns the text of the first `key = [ ... ]` list in block,
+// matching nested brackets (index expressions like fck_nat[0]).
+func bracketValue(t *testing.T, block, key string) string {
+	t.Helper()
+	loc := regexp.MustCompile(regexp.QuoteMeta(key) + `\s*=\s*\[`).FindStringIndex(block)
+	if loc == nil {
+		return ""
+	}
+	depth := 0
+	for i := loc[1] - 1; i < len(block); i++ {
+		switch block[i] {
+		case '[':
+			depth++
+		case ']':
+			depth--
+			if depth == 0 {
+				return block[loc[1]:i]
+			}
+		}
+	}
+	return ""
+}
+
+// TestFckNatApplyOrderAndWiring pins HCL that mock-provider terraform tests
+// cannot see (they expose no apply ordering) and that fails silently in a
+// live account: a wrong order only shows as an egress outage.
+func TestFckNatApplyOrderAndWiring(t *testing.T) {
+	path := filepath.Join(modulesDir, "networking", "aws", "main.tf")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	src := stripCommentLines(string(raw))
+
+	route := resourceBlock(t, src, "aws_route", "private_internet_ipv4")
+	asg := resourceBlock(t, src, "aws_autoscaling_group", "fck_nat")
+	eni := resourceBlock(t, src, "aws_network_interface", "fck_nat")
+	policy := resourceBlock(t, src, "aws_iam_role_policy", "fck_nat_ec2")
+
+	dependsOn := func(block, target string) bool {
+		for _, dep := range strings.Split(bracketValue(t, block, "depends_on"), ",") {
+			if strings.TrimSpace(dep) == target {
+				return true
+			}
+		}
+		return false
+	}
+	if !dependsOn(route, "aws_autoscaling_group.fck_nat") {
+		t.Error("aws_route.private_internet_ipv4 must depends_on aws_autoscaling_group.fck_nat: otherwise the routes move to the new ENI before an instance that attaches it is InService")
+	}
+	if !dependsOn(asg, "aws_iam_role_policy.fck_nat_ec2") {
+		t.Error("aws_autoscaling_group.fck_nat must depends_on aws_iam_role_policy.fck_nat_ec2: the attach permission must exist before an instance boots")
+	}
+
+	if !regexp.MustCompile(`vpc_zone_identifier\s*=\s*\[aws_network_interface\.fck_nat\[0\]\.subnet_id\]`).MatchString(asg) {
+		t.Error("the fck-nat ASG must launch only into the persistent ENI's subnet; an ENI attaches only within its own AZ")
+	}
+	if !regexp.MustCompile(`subnet_id\s*=\s*local\.public_subnet_ids\[0\]`).MatchString(eni) {
+		t.Error("the persistent ENI must live in local.public_subnet_ids[0]")
+	}
+
+	if got := strings.TrimSpace(bracketValue(t, asg, "replace_triggered_by")); got != "aws_launch_template.fck_nat[count.index].user_data" {
+		t.Errorf("replace_triggered_by must be exactly [aws_launch_template.fck_nat[count.index].user_data] (not the ENI, which would also fire on unrelated ENI attribute changes, and not image_id); got [%s]", got)
+	}
+
+	sid := strings.Index(policy, `Sid      = "AttachPersistentENI"`)
+	if sid < 0 {
+		t.Fatal("AttachPersistentENI statement not found in aws_iam_role_policy.fck_nat_ec2")
+	}
+	stmt := policy[sid:]
+	if end := strings.Index(stmt, "},"); end >= 0 {
+		stmt = stmt[:end]
+	}
+	res := bracketValue(t, stmt, "Resource")
+	if res == "" {
+		t.Fatalf("AttachPersistentENI Resource must be a list (a bare \"*\" string is rejected); statement: %s", stmt)
+	}
+	if !strings.Contains(res, "aws_network_interface.fck_nat[0].arn") {
+		t.Errorf("AttachNetworkInterface must be scoped to the persistent ENI ARN; got %s", res)
+	}
+	if !strings.Contains(res, ":instance/*") {
+		t.Errorf("AttachNetworkInterface is authorized against the instance too; Resource must include an instance/* ARN; got %s", res)
+	}
+	if regexp.MustCompile(`"\*"`).MatchString(res) {
+		t.Errorf("AttachNetworkInterface Resource must not be \"*\"; got %s", res)
+	}
+}

terraform/environments/aws/ci-cd-permissions/policy_networking.tf

@@ -33,6 +33,15 @@ resource "aws_iam_policy" "networking" {
           "ec2:CreateLaunchTemplateVersion",
           "ec2:DeleteLaunchTemplateVersions",
           "ec2:ModifyLaunchTemplate",
+          # The persistent fck-nat ENI the private routes target (#716):
+          # CreateNetworkInterface to create it, ModifyNetworkInterfaceAttribute
+          # because the provider turns source_dest_check off in a second call
+          # after the create, and DetachNetworkInterface because destroying or
+          # replacing it detaches it from the NAT instance first.
+          # DeleteNetworkInterface and DescribeNetworkInterfaces are below.
+          "ec2:CreateNetworkInterface",
+          "ec2:ModifyNetworkInterfaceAttribute",
+          "ec2:DetachNetworkInterface",
           "ec2:CreateRoute",
           "ec2:CreateRouteTable",
           "ec2:CreateSecurityGroup",
@@ -104,7 +113,7 @@ resource "aws_iam_policy" "networking" {
       },
       {
         # ec2:RunInstances is needed for the fck-nat AutoScaling group
-        # (one t4g.nano per AZ). Restricting allowed instance types prevents
+        # (one t4g.nano NAT instance in total). Restricting allowed instance types prevents
         # the deploy SA from launching arbitrary large instances and attaching
         # CUDly IAM roles to exfiltrate credentials or run compute at account
         # cost.

terraform/modules/networking/aws/README.md

@@ -114,6 +114,40 @@ The `enable_nat_gateway` option deploys a cost-effective NAT alternative:
 - Provides IPv4 egress for services without IPv6 support (like AWS SES)
 - Automatically deployed in the first public subnet
 - 90% cheaper than AWS NAT Gateway (~$32/month)
+- Private IPv4 routes target a persistent ENI that each instance attaches at
+  boot, so egress survives the ASG replacing the instance. With an existing
+  VPC, route private subnets to the `nat_network_interface_id` output.
+
+### Rollback runbook (persistent ENI, #716)
+
+If egress from the private subnets is down after the apply that introduced the
+persistent ENI, run `rollback.yml` against the pre-merge `main` SHA first. Note
+that `rollback.yml` applies the environment root from that ref's checkout.
+
+Fallback by hand, when the workflow cannot run. Placeholders: `<eni-id>` is the
+`nat_network_interface_id` output, `<instance-id>` the running fck-nat instance
+(`nat_autoscaling_group_name` output, then `aws autoscaling describe-auto-scaling-groups`),
+`<rtb-id>` each private route table.
+
+```bash
+# 1. Is the ENI attached as the instance's second interface?
+aws ec2 describe-network-interfaces --network-interface-ids <eni-id> \
+  --query 'NetworkInterfaces[0].Attachment'
+
+# 2. If not, attach it by hand (device index 1; the instance must be in the ENI's AZ).
+aws ec2 attach-network-interface --network-interface-id <eni-id> \
+  --instance-id <instance-id> --device-index 1
+
+# 3. If the route is a blackhole, point it back at a working target
+#    (the instance's primary ENI, or the ENI once step 2 succeeded).
+aws ec2 replace-route --route-table-id <rtb-id> \
+  --destination-cidr-block 0.0.0.0/0 --network-interface-id <eni-id>
+```
+
+A persistent attach failure shows as an instance replaced about every 5 minutes
+(`aws autoscaling describe-scaling-activities --auto-scaling-group-name <asg-name>`):
+user data bounds the service restart at 300 s and powers the instance off, and
+the launch template's `terminate` shutdown behaviour has the ASG replace it.
 
 ## Security Groups
 
@@ -138,6 +172,8 @@ The module outputs:
 - `database_security_group_id`: Security group for database
 - `lambda_vpc_config`: Convenience object for Lambda module
 - `database_vpc_config`: Convenience object for database module
+- `nat_network_interface_id`: Persistent fck-nat ENI to route IPv4 egress to (if enabled)
+- `nat_autoscaling_group_name`: fck-nat Auto Scaling group name (if enabled)
 
 ## Cost Comparison