Add self-hosted Grafana

infra/deployments/forms/dns/dependencies.tf

@@ -1,3 +1,15 @@
+data "terraform_remote_state" "forms_health" {
+  backend = "s3"
+
+  config = {
+    key    = "health.tfstate"
+    bucket = var.bucket
+    region = "eu-west-2"
+
+    use_lockfile = true
+  }
+}
+
 data "terraform_remote_state" "account" {
   backend = "s3"
 

infra/deployments/forms/dns/grafana.tf

@@ -0,0 +1,13 @@
+resource "aws_route53_record" "grafana" {
+  #checkov:skip=CKV2_AWS_23:Not applicable to alias records
+  count   = var.grafana_settings.enabled ? 1 : 0
+  zone_id = data.terraform_remote_state.account.outputs.route53_hosted_zone_id
+  name    = "grafana.${var.root_domain}"
+  type    = "A"
+
+  alias {
+    name                   = data.terraform_remote_state.forms_health.outputs.grafana_alb_dns_name
+    zone_id                = data.terraform_remote_state.forms_health.outputs.grafana_alb_zone_id
+    evaluate_target_health = true
+  }
+}

infra/deployments/forms/environment/outputs.tf

@@ -10,6 +10,14 @@ output "private_subnet_ids" {
   value = module.environment.private_subnet_ids
 }
 
+output "public_subnet_ids" {
+  value = module.environment.public_subnet_ids
+}
+
+output "alb_logs_bucket_name" {
+  value = module.environment.alb_logs_bucket_name
+}
+
 output "alb_arn_suffix" {
   value = module.environment.alb_arn_suffix
 }

infra/deployments/forms/health/dependencies.tf

@@ -57,3 +57,15 @@ data "terraform_remote_state" "forms_product_page" {
     use_lockfile = true
   }
 }
+
+data "terraform_remote_state" "account" {
+  backend = "s3"
+
+  config = {
+    key    = "account.tfstate"
+    bucket = var.bucket
+    region = "eu-west-2"
+
+    use_lockfile = true
+  }
+}

infra/deployments/forms/health/grafana.tf

@@ -0,0 +1,30 @@
+module "grafana" {
+  count  = var.grafana_settings.enabled ? 1 : 0
+  source = "../../../modules/grafana"
+
+  # The module passes the provider on to the certificate module, which
+  # requires it to be passed explicitly here.
+  providers = {
+    aws = aws
+  }
+
+  environment_name = var.environment_name
+  root_domain      = var.root_domain
+
+  vpc_id                      = data.terraform_remote_state.forms_environment.outputs.vpc_id
+  vpc_cidr_block              = data.terraform_remote_state.forms_environment.outputs.vpc_cidr_block
+  private_subnet_ids          = data.terraform_remote_state.forms_environment.outputs.private_subnet_ids
+  public_subnet_ids           = data.terraform_remote_state.forms_environment.outputs.public_subnet_ids
+  ecs_cluster_arn             = data.terraform_remote_state.forms_environment.outputs.ecs_cluster_arn
+  alb_access_logs_bucket_name = data.terraform_remote_state.forms_environment.outputs.alb_logs_bucket_name
+
+  kinesis_subscription_role_arn = data.terraform_remote_state.account.outputs.kinesis_subscription_role_arn
+
+  cpu                          = var.grafana_settings.cpu
+  memory                       = var.grafana_settings.memory
+  github_allowed_organizations = var.grafana_settings.github_allowed_organizations
+  github_admin_team            = var.grafana_settings.github_admin_team
+  github_editor_teams          = var.grafana_settings.github_editor_teams
+  seconds_until_auto_pause     = var.grafana_settings.seconds_until_auto_pause
+  rds_maintenance_window       = var.environmental_settings.rds_maintenance_window
+}

infra/deployments/forms/health/outputs.tf

@@ -0,0 +1,7 @@
+output "grafana_alb_dns_name" {
+  value = try(module.grafana[0].alb_dns_name, null)
+}
+
+output "grafana_alb_zone_id" {
+  value = try(module.grafana[0].alb_zone_id, null)
+}

infra/deployments/forms/inputs.tf

@@ -233,3 +233,17 @@ variable "end_to_end_test_settings" {
     email_receiver_s3_bucket_name = string
   })
 }
+
+variable "grafana_settings" {
+  description = "Configuration for the self-hosted Grafana service"
+  type = object({
+    enabled                      = optional(bool, false)
+    cpu                          = number
+    memory                       = number
+    github_allowed_organizations = list(string) # GitHub organisations whose members may sign in
+    github_admin_team            = string       # GitHub team (org/team) granted Grafana server admin
+    github_editor_teams          = list(string) # GitHub teams (org/team) granted the Editor role; other org members are denied
+    seconds_until_auto_pause     = optional(number, 3600)
+  })
+  nullable = false
+}

infra/deployments/forms/tfvars/dev.tfvars

@@ -128,3 +128,11 @@ end_to_end_test_settings = {
   s3_form_id                    = "12457"
   email_receiver_s3_bucket_name = "govuk-forms-dev-test-emails"
 }
+grafana_settings = {
+  enabled                      = true
+  cpu                          = 512
+  memory                       = 1024
+  github_allowed_organizations = ["govuk-forms"]
+  github_admin_team            = "govuk-forms/govuk-forms-infrastructure"
+  github_editor_teams          = ["govuk-forms/govuk-forms-devs"]
+}

infra/deployments/forms/tfvars/production.tfvars

@@ -185,3 +185,11 @@ end_to_end_test_settings = {
   s3_form_id                    = "5086"
   email_receiver_s3_bucket_name = "govuk-forms-production-test-emails"
 }
+grafana_settings = {
+  enabled                      = false
+  cpu                          = 512
+  memory                       = 1024
+  github_allowed_organizations = ["govuk-forms"]
+  github_admin_team            = "govuk-forms/govuk-forms-infrastructure"
+  github_editor_teams          = ["govuk-forms/govuk-forms-devs"]
+}

infra/deployments/forms/tfvars/staging.tfvars

@@ -93,3 +93,11 @@ end_to_end_test_settings = {
   s3_form_id                    = "13657"
   email_receiver_s3_bucket_name = "govuk-forms-staging-test-emails"
 }
+grafana_settings = {
+  enabled                      = false
+  cpu                          = 512
+  memory                       = 1024
+  github_allowed_organizations = ["govuk-forms"]
+  github_admin_team            = "govuk-forms/govuk-forms-infrastructure"
+  github_editor_teams          = ["govuk-forms/govuk-forms-devs"]
+}

infra/modules/acm-cert-with-dns-validation/main.tf

@@ -1,7 +1,7 @@
 # Manages an AWS ACM certificate and the necessary DNS records for its
 # validation.  The certificate domain name and its subject alternative names
-# must be valid within a single Route53 hosted zone for the var.domain_name
-# passed.
+# must be valid within a single Route53 hosted zone, named by var.zone_name
+# or, when that is not set, var.domain_name.
 
 resource "aws_acm_certificate" "cert" {
   #checkov:skip=CKV2_AWS_71:we can't tell Checkov that domain_name won't include * here
@@ -17,7 +17,7 @@ resource "aws_acm_certificate" "cert" {
 }
 
 data "aws_route53_zone" "cert_domain" {
-  name         = var.domain_name
+  name         = coalesce(var.zone_name, var.domain_name)
   private_zone = false
 }
 

infra/modules/acm-cert-with-dns-validation/variables.tf

@@ -3,6 +3,12 @@ variable "domain_name" {
   description = "The primary domain name for the certificate"
 }
 
+variable "zone_name" {
+  type        = string
+  description = "The Route53 hosted zone in which to create the validation records. Defaults to domain_name."
+  default     = null
+}
+
 variable "subject_alternative_names" {
   type        = list(string)
   description = "Subject alternative names for the certificate. Must be within the same domain as domain_name"

infra/modules/deployer-access/policy.tf

@@ -486,7 +486,9 @@ data "aws_iam_policy_document" "iam" {
       "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-runner-ecs-task-policy",
       "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-runner-adot-collector",
       "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-ecs-task-policy",
-      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-adot-collector"
+      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-adot-collector",
+      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-grafana-ecs-task-policy",
+      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-grafana-ecs-task-execution-additional"
     ]
     sid = "ManageEcsPolicies"
   }
@@ -550,7 +552,9 @@ data "aws_iam_policy_document" "iam" {
       "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-admin-ecs-task-execution",
       "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-ecs-task-execution",
       "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-product-page-ecs-task-execution",
-      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-queue-worker-ecs-task-execution"
+      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-queue-worker-ecs-task-execution",
+      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-grafana-ecs-task",
+      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-grafana-ecs-task-execution"
     ]
     sid = "ManageTaskAndTaskExecutionRoles"
   }
@@ -807,6 +811,7 @@ data "aws_iam_policy_document" "logs" {
       "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-runner-queue-worker-${var.environment_name}/adot-collector:*",
       "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-product-page-${var.environment_name}:*",
       "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-product-page-${var.environment_name}/adot-collector:*",
+      "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/grafana:*",
       "arn:aws:logs:us-east-1:${var.account_id}:log-group:aws-waf-logs-${var.environment_name}*",
       "arn:aws:logs:eu-west-2:${var.account_id}:log-group:aws-waf-logs-alb-${var.environment_name}*"
     ]

infra/modules/environment/alb.tf

@@ -50,7 +50,8 @@ data "aws_iam_policy_document" "allow_logs" {
     actions = ["s3:PutObject"]
     resources = [
       "arn:aws:s3:::${module.logs_bucket.bucket_name}/${var.env_name}/AWSLogs/${local.account_id}/*",
-      "arn:aws:s3:::${module.logs_bucket.bucket_name}/forms-internal/AWSLogs/${local.account_id}/*"
+      "arn:aws:s3:::${module.logs_bucket.bucket_name}/forms-internal/AWSLogs/${local.account_id}/*",
+      "arn:aws:s3:::${module.logs_bucket.bucket_name}/grafana/AWSLogs/${local.account_id}/*"
     ]
   }
 }

infra/modules/environment/outputs.tf

@@ -14,6 +14,18 @@ output "private_subnet_ids" {
   ]
 }
 
+output "public_subnet_ids" {
+  value = [
+    aws_subnet.public_a.id,
+    aws_subnet.public_b.id,
+    aws_subnet.public_c.id
+  ]
+}
+
+output "alb_logs_bucket_name" {
+  value = module.logs_bucket.bucket_name
+}
+
 output "alb_arn_suffix" {
   value = aws_lb.alb.arn_suffix
 }

infra/modules/grafana/README.md

@@ -0,0 +1,42 @@
+# Grafana
+
+Runs Grafana OSS as a single Fargate task behind its own public ALB at
+`grafana.<root_domain>`, backed by its own Aurora Serverless v2 PostgreSQL
+cluster. The load balancer is public and is not behind CloudFront or a WAF;
+GitHub sign in is the only access control. Data sources for CloudWatch and
+X-Ray (Application Signals) are provisioned at start up and authenticate with
+the task role. Grafana alerting is disabled.
+
+## Signing in
+
+Users sign in with GitHub and are sent straight to GitHub when they visit
+Grafana. Access is limited to members of the organisations in
+`github_allowed_organizations`, and a role is assigned from team membership:
+`github_admin_team` members are Grafana server admins, `github_editor_teams`
+members are editors, and anybody else is refused.
+
+The built-in `admin` user remains for use when GitHub sign in is unavailable.
+Its password is in the SSM parameter `/grafana/admin-password` and the login
+form is reachable at `/login?disableAutoLogin=true`.
+
+## Setting up an environment
+
+1. Apply `forms/account` (deployer permissions), `forms/environment` (public
+   subnet outputs and the ALB logs bucket policy), then `forms/health` (this
+   module) and `forms/dns` (the record).
+2. Create a GitHub OAuth App in the organisation with the homepage URL
+   `https://grafana.<root_domain>` and the authorization callback URL
+   `https://grafana.<root_domain>/login/github`. If the organisation restricts
+   third-party OAuth apps, approve it for the organisation.
+3. Put the app's client ID and secret in the SSM parameters
+   `/grafana/github/client-id` and `/grafana/github/client-secret`.
+4. Force a new deployment of the `grafana` ECS service so the task picks the
+   values up.
+
+## Notes
+
+- The database is allowed to pause after `seconds_until_auto_pause` seconds
+  without connections, but Grafana keeps a connection pool open, so in
+  practice it only pauses while the service is stopped.
+- The plugins are downloaded from grafana.com when the task starts, so the
+  task needs internet egress and a start takes a minute or two.

infra/modules/grafana/alb.tf

@@ -0,0 +1,111 @@
+module "certificate" {
+  source = "../acm-cert-with-dns-validation"
+  providers = {
+    aws             = aws
+    aws.certificate = aws
+  }
+
+  domain_name               = local.host_name
+  zone_name                 = var.root_domain
+  subject_alternative_names = []
+}
+
+resource "aws_security_group" "alb" {
+  #checkov:skip=CKV_AWS_260:Port 80 only redirects to HTTPS
+  name        = "${local.name}-alb"
+  description = "Allows public inbound on 80 and 443 and outbound to the Grafana container"
+  vpc_id      = var.vpc_id
+
+  ingress {
+    description = "Port 80 from public, redirected to 443"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "Port 443 from public"
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "Grafana container port within VPC"
+    from_port   = local.container_port
+    to_port     = local.container_port
+    protocol    = "tcp"
+    cidr_blocks = [var.vpc_cidr_block]
+  }
+}
+
+resource "aws_lb" "grafana" {
+  #checkov:skip=CKV2_AWS_28:Access is limited to GitHub team members by Grafana, and a WAF is not considered necessary
+
+  name                       = local.name
+  internal                   = false
+  load_balancer_type         = "application"
+  enable_deletion_protection = true
+  drop_invalid_header_fields = true
+  security_groups            = [aws_security_group.alb.id]
+  subnets                    = var.public_subnet_ids
+
+  access_logs {
+    bucket  = var.alb_access_logs_bucket_name
+    prefix  = local.name
+    enabled = true
+  }
+}
+
+resource "aws_lb_listener" "http" {
+  load_balancer_arn = aws_lb.grafana.arn
+  port              = "80"
+  protocol          = "HTTP"
+
+  default_action {
+    type = "redirect"
+
+    redirect {
+      port        = "443"
+      protocol    = "HTTPS"
+      status_code = "HTTP_301"
+    }
+  }
+}
+
+resource "aws_lb_listener" "https" {
+  load_balancer_arn = aws_lb.grafana.arn
+  port              = "443"
+  protocol          = "HTTPS"
+  ssl_policy        = "ELBSecurityPolicy-TLS13-1-2-2021-06"
+  certificate_arn   = module.certificate.arn
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.grafana.arn
+  }
+}
+
+resource "aws_lb_target_group" "grafana" {
+  #checkov:skip=CKV_AWS_378: We're happy that this is internal traffic within our vpc and we do not want the complexity cost of setting up TLS between the load balancer and application
+  name        = local.name
+  port        = local.container_port
+  protocol    = "HTTP"
+  vpc_id      = var.vpc_id
+  target_type = "ip"
+
+  deregistration_delay = "60"
+
+  health_check {
+    path     = "/api/health"
+    matcher  = "200"
+    protocol = "HTTP"
+
+    interval            = 11
+    timeout             = 10
+    unhealthy_threshold = 3
+    healthy_threshold   = 2
+  }
+}