diff --git a/infra/deployments/forms/dns/dependencies.tf b/infra/deployments/forms/dns/dependencies.tf index fe7b6180e..1a0331faf 100644 --- a/infra/deployments/forms/dns/dependencies.tf +++ b/infra/deployments/forms/dns/dependencies.tf @@ -1,3 +1,15 @@ +data "terraform_remote_state" "forms_health" { + backend = "s3" + + config = { + key = "health.tfstate" + bucket = var.bucket + region = "eu-west-2" + + use_lockfile = true + } +} + data "terraform_remote_state" "account" { backend = "s3" diff --git a/infra/deployments/forms/dns/grafana.tf b/infra/deployments/forms/dns/grafana.tf new file mode 100644 index 000000000..19ab1e81a --- /dev/null +++ b/infra/deployments/forms/dns/grafana.tf @@ -0,0 +1,13 @@ +resource "aws_route53_record" "grafana" { + #checkov:skip=CKV2_AWS_23:Not applicable to alias records + count = var.grafana_settings.enabled ? 1 : 0 + zone_id = data.terraform_remote_state.account.outputs.route53_hosted_zone_id + name = "grafana.${var.root_domain}" + type = "A" + + alias { + name = data.terraform_remote_state.forms_health.outputs.grafana_alb_dns_name + zone_id = data.terraform_remote_state.forms_health.outputs.grafana_alb_zone_id + evaluate_target_health = true + } +} diff --git a/infra/deployments/forms/environment/outputs.tf b/infra/deployments/forms/environment/outputs.tf index 7d93ff624..7fb7f77dd 100644 --- a/infra/deployments/forms/environment/outputs.tf +++ b/infra/deployments/forms/environment/outputs.tf @@ -10,6 +10,14 @@ output "private_subnet_ids" { value = module.environment.private_subnet_ids } +output "public_subnet_ids" { + value = module.environment.public_subnet_ids +} + +output "alb_logs_bucket_name" { + value = module.environment.alb_logs_bucket_name +} + output "alb_arn_suffix" { value = module.environment.alb_arn_suffix } diff --git a/infra/deployments/forms/health/dependencies.tf b/infra/deployments/forms/health/dependencies.tf index d10ab8adc..541d72686 100644 --- a/infra/deployments/forms/health/dependencies.tf +++ b/infra/deployments/forms/health/dependencies.tf @@ -57,3 +57,15 @@ data "terraform_remote_state" "forms_product_page" { use_lockfile = true } } + +data "terraform_remote_state" "account" { + backend = "s3" + + config = { + key = "account.tfstate" + bucket = var.bucket + region = "eu-west-2" + + use_lockfile = true + } +} diff --git a/infra/deployments/forms/health/grafana.tf b/infra/deployments/forms/health/grafana.tf new file mode 100644 index 000000000..25e8a602b --- /dev/null +++ b/infra/deployments/forms/health/grafana.tf @@ -0,0 +1,30 @@ +module "grafana" { + count = var.grafana_settings.enabled ? 1 : 0 + source = "../../../modules/grafana" + + # The module passes the provider on to the certificate module, which + # requires it to be passed explicitly here. + providers = { + aws = aws + } + + environment_name = var.environment_name + root_domain = var.root_domain + + vpc_id = data.terraform_remote_state.forms_environment.outputs.vpc_id + vpc_cidr_block = data.terraform_remote_state.forms_environment.outputs.vpc_cidr_block + private_subnet_ids = data.terraform_remote_state.forms_environment.outputs.private_subnet_ids + public_subnet_ids = data.terraform_remote_state.forms_environment.outputs.public_subnet_ids + ecs_cluster_arn = data.terraform_remote_state.forms_environment.outputs.ecs_cluster_arn + alb_access_logs_bucket_name = data.terraform_remote_state.forms_environment.outputs.alb_logs_bucket_name + + kinesis_subscription_role_arn = data.terraform_remote_state.account.outputs.kinesis_subscription_role_arn + + cpu = var.grafana_settings.cpu + memory = var.grafana_settings.memory + github_allowed_organizations = var.grafana_settings.github_allowed_organizations + github_admin_team = var.grafana_settings.github_admin_team + github_editor_teams = var.grafana_settings.github_editor_teams + seconds_until_auto_pause = var.grafana_settings.seconds_until_auto_pause + rds_maintenance_window = var.environmental_settings.rds_maintenance_window +} diff --git a/infra/deployments/forms/health/outputs.tf b/infra/deployments/forms/health/outputs.tf new file mode 100644 index 000000000..dcf26ce0d --- /dev/null +++ b/infra/deployments/forms/health/outputs.tf @@ -0,0 +1,7 @@ +output "grafana_alb_dns_name" { + value = try(module.grafana[0].alb_dns_name, null) +} + +output "grafana_alb_zone_id" { + value = try(module.grafana[0].alb_zone_id, null) +} diff --git a/infra/deployments/forms/inputs.tf b/infra/deployments/forms/inputs.tf index 9dcb14624..4a0fd801c 100644 --- a/infra/deployments/forms/inputs.tf +++ b/infra/deployments/forms/inputs.tf @@ -233,3 +233,17 @@ variable "end_to_end_test_settings" { email_receiver_s3_bucket_name = string }) } + +variable "grafana_settings" { + description = "Configuration for the self-hosted Grafana service" + type = object({ + enabled = optional(bool, false) + cpu = number + memory = number + github_allowed_organizations = list(string) # GitHub organisations whose members may sign in + github_admin_team = string # GitHub team (org/team) granted Grafana server admin + github_editor_teams = list(string) # GitHub teams (org/team) granted the Editor role; other org members are denied + seconds_until_auto_pause = optional(number, 3600) + }) + nullable = false +} diff --git a/infra/deployments/forms/tfvars/dev.tfvars b/infra/deployments/forms/tfvars/dev.tfvars index 147bb6ac4..a94919714 100644 --- a/infra/deployments/forms/tfvars/dev.tfvars +++ b/infra/deployments/forms/tfvars/dev.tfvars @@ -128,3 +128,11 @@ end_to_end_test_settings = { s3_form_id = "12457" email_receiver_s3_bucket_name = "govuk-forms-dev-test-emails" } +grafana_settings = { + enabled = true + cpu = 512 + memory = 1024 + github_allowed_organizations = ["govuk-forms"] + github_admin_team = "govuk-forms/govuk-forms-infrastructure" + github_editor_teams = ["govuk-forms/govuk-forms-devs"] +} diff --git a/infra/deployments/forms/tfvars/production.tfvars b/infra/deployments/forms/tfvars/production.tfvars index 2227a3828..77bc1d6f6 100644 --- a/infra/deployments/forms/tfvars/production.tfvars +++ b/infra/deployments/forms/tfvars/production.tfvars @@ -185,3 +185,11 @@ end_to_end_test_settings = { s3_form_id = "5086" email_receiver_s3_bucket_name = "govuk-forms-production-test-emails" } +grafana_settings = { + enabled = false + cpu = 512 + memory = 1024 + github_allowed_organizations = ["govuk-forms"] + github_admin_team = "govuk-forms/govuk-forms-infrastructure" + github_editor_teams = ["govuk-forms/govuk-forms-devs"] +} diff --git a/infra/deployments/forms/tfvars/staging.tfvars b/infra/deployments/forms/tfvars/staging.tfvars index f142f19b8..7f1f88bdf 100644 --- a/infra/deployments/forms/tfvars/staging.tfvars +++ b/infra/deployments/forms/tfvars/staging.tfvars @@ -93,3 +93,11 @@ end_to_end_test_settings = { s3_form_id = "13657" email_receiver_s3_bucket_name = "govuk-forms-staging-test-emails" } +grafana_settings = { + enabled = false + cpu = 512 + memory = 1024 + github_allowed_organizations = ["govuk-forms"] + github_admin_team = "govuk-forms/govuk-forms-infrastructure" + github_editor_teams = ["govuk-forms/govuk-forms-devs"] +} diff --git a/infra/modules/acm-cert-with-dns-validation/main.tf b/infra/modules/acm-cert-with-dns-validation/main.tf index 95cd15d6c..a8673ca7b 100644 --- a/infra/modules/acm-cert-with-dns-validation/main.tf +++ b/infra/modules/acm-cert-with-dns-validation/main.tf @@ -1,7 +1,7 @@ # Manages an AWS ACM certificate and the necessary DNS records for its # validation. The certificate domain name and its subject alternative names -# must be valid within a single Route53 hosted zone for the var.domain_name -# passed. +# must be valid within a single Route53 hosted zone, named by var.zone_name +# or, when that is not set, var.domain_name. resource "aws_acm_certificate" "cert" { #checkov:skip=CKV2_AWS_71:we can't tell Checkov that domain_name won't include * here @@ -17,7 +17,7 @@ resource "aws_acm_certificate" "cert" { } data "aws_route53_zone" "cert_domain" { - name = var.domain_name + name = coalesce(var.zone_name, var.domain_name) private_zone = false } diff --git a/infra/modules/acm-cert-with-dns-validation/variables.tf b/infra/modules/acm-cert-with-dns-validation/variables.tf index 35f1f6488..228358305 100644 --- a/infra/modules/acm-cert-with-dns-validation/variables.tf +++ b/infra/modules/acm-cert-with-dns-validation/variables.tf @@ -3,6 +3,12 @@ variable "domain_name" { description = "The primary domain name for the certificate" } +variable "zone_name" { + type = string + description = "The Route53 hosted zone in which to create the validation records. Defaults to domain_name." + default = null +} + variable "subject_alternative_names" { type = list(string) description = "Subject alternative names for the certificate. Must be within the same domain as domain_name" diff --git a/infra/modules/deployer-access/policy.tf b/infra/modules/deployer-access/policy.tf index 3851e2b5e..831971bf1 100644 --- a/infra/modules/deployer-access/policy.tf +++ b/infra/modules/deployer-access/policy.tf @@ -486,7 +486,9 @@ data "aws_iam_policy_document" "iam" { "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-runner-ecs-task-policy", "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-runner-adot-collector", "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-ecs-task-policy", - "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-adot-collector" + "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-adot-collector", + "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-grafana-ecs-task-policy", + "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-grafana-ecs-task-execution-additional" ] sid = "ManageEcsPolicies" } @@ -550,7 +552,9 @@ data "aws_iam_policy_document" "iam" { "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-admin-ecs-task-execution", "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-ecs-task-execution", "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-product-page-ecs-task-execution", - "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-queue-worker-ecs-task-execution" + "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-queue-worker-ecs-task-execution", + "arn:aws:iam::${var.account_id}:role/${var.environment_name}-grafana-ecs-task", + "arn:aws:iam::${var.account_id}:role/${var.environment_name}-grafana-ecs-task-execution" ] sid = "ManageTaskAndTaskExecutionRoles" } @@ -807,6 +811,7 @@ data "aws_iam_policy_document" "logs" { "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-runner-queue-worker-${var.environment_name}/adot-collector:*", "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-product-page-${var.environment_name}:*", "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-product-page-${var.environment_name}/adot-collector:*", + "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/grafana:*", "arn:aws:logs:us-east-1:${var.account_id}:log-group:aws-waf-logs-${var.environment_name}*", "arn:aws:logs:eu-west-2:${var.account_id}:log-group:aws-waf-logs-alb-${var.environment_name}*" ] diff --git a/infra/modules/environment/alb.tf b/infra/modules/environment/alb.tf index f01457401..eae4d6e4d 100644 --- a/infra/modules/environment/alb.tf +++ b/infra/modules/environment/alb.tf @@ -50,7 +50,8 @@ data "aws_iam_policy_document" "allow_logs" { actions = ["s3:PutObject"] resources = [ "arn:aws:s3:::${module.logs_bucket.bucket_name}/${var.env_name}/AWSLogs/${local.account_id}/*", - "arn:aws:s3:::${module.logs_bucket.bucket_name}/forms-internal/AWSLogs/${local.account_id}/*" + "arn:aws:s3:::${module.logs_bucket.bucket_name}/forms-internal/AWSLogs/${local.account_id}/*", + "arn:aws:s3:::${module.logs_bucket.bucket_name}/grafana/AWSLogs/${local.account_id}/*" ] } } diff --git a/infra/modules/environment/outputs.tf b/infra/modules/environment/outputs.tf index 3310df861..1c67631d7 100644 --- a/infra/modules/environment/outputs.tf +++ b/infra/modules/environment/outputs.tf @@ -14,6 +14,18 @@ output "private_subnet_ids" { ] } +output "public_subnet_ids" { + value = [ + aws_subnet.public_a.id, + aws_subnet.public_b.id, + aws_subnet.public_c.id + ] +} + +output "alb_logs_bucket_name" { + value = module.logs_bucket.bucket_name +} + output "alb_arn_suffix" { value = aws_lb.alb.arn_suffix } diff --git a/infra/modules/grafana/README.md b/infra/modules/grafana/README.md new file mode 100644 index 000000000..259cd8f8b --- /dev/null +++ b/infra/modules/grafana/README.md @@ -0,0 +1,42 @@ +# Grafana + +Runs Grafana OSS as a single Fargate task behind its own public ALB at +`grafana.`, backed by its own Aurora Serverless v2 PostgreSQL +cluster. The load balancer is public and is not behind CloudFront or a WAF; +GitHub sign in is the only access control. Data sources for CloudWatch and +X-Ray (Application Signals) are provisioned at start up and authenticate with +the task role. Grafana alerting is disabled. + +## Signing in + +Users sign in with GitHub and are sent straight to GitHub when they visit +Grafana. Access is limited to members of the organisations in +`github_allowed_organizations`, and a role is assigned from team membership: +`github_admin_team` members are Grafana server admins, `github_editor_teams` +members are editors, and anybody else is refused. + +The built-in `admin` user remains for use when GitHub sign in is unavailable. +Its password is in the SSM parameter `/grafana/admin-password` and the login +form is reachable at `/login?disableAutoLogin=true`. + +## Setting up an environment + +1. Apply `forms/account` (deployer permissions), `forms/environment` (public + subnet outputs and the ALB logs bucket policy), then `forms/health` (this + module) and `forms/dns` (the record). +2. Create a GitHub OAuth App in the organisation with the homepage URL + `https://grafana.` and the authorization callback URL + `https://grafana./login/github`. If the organisation restricts + third-party OAuth apps, approve it for the organisation. +3. Put the app's client ID and secret in the SSM parameters + `/grafana/github/client-id` and `/grafana/github/client-secret`. +4. Force a new deployment of the `grafana` ECS service so the task picks the + values up. + +## Notes + +- The database is allowed to pause after `seconds_until_auto_pause` seconds + without connections, but Grafana keeps a connection pool open, so in + practice it only pauses while the service is stopped. +- The plugins are downloaded from grafana.com when the task starts, so the + task needs internet egress and a start takes a minute or two. diff --git a/infra/modules/grafana/alb.tf b/infra/modules/grafana/alb.tf new file mode 100644 index 000000000..8c64d06fa --- /dev/null +++ b/infra/modules/grafana/alb.tf @@ -0,0 +1,111 @@ +module "certificate" { + source = "../acm-cert-with-dns-validation" + providers = { + aws = aws + aws.certificate = aws + } + + domain_name = local.host_name + zone_name = var.root_domain + subject_alternative_names = [] +} + +resource "aws_security_group" "alb" { + #checkov:skip=CKV_AWS_260:Port 80 only redirects to HTTPS + name = "${local.name}-alb" + description = "Allows public inbound on 80 and 443 and outbound to the Grafana container" + vpc_id = var.vpc_id + + ingress { + description = "Port 80 from public, redirected to 443" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "Port 443 from public" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + description = "Grafana container port within VPC" + from_port = local.container_port + to_port = local.container_port + protocol = "tcp" + cidr_blocks = [var.vpc_cidr_block] + } +} + +resource "aws_lb" "grafana" { + #checkov:skip=CKV2_AWS_28:Access is limited to GitHub team members by Grafana, and a WAF is not considered necessary + + name = local.name + internal = false + load_balancer_type = "application" + enable_deletion_protection = true + drop_invalid_header_fields = true + security_groups = [aws_security_group.alb.id] + subnets = var.public_subnet_ids + + access_logs { + bucket = var.alb_access_logs_bucket_name + prefix = local.name + enabled = true + } +} + +resource "aws_lb_listener" "http" { + load_balancer_arn = aws_lb.grafana.arn + port = "80" + protocol = "HTTP" + + default_action { + type = "redirect" + + redirect { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } +} + +resource "aws_lb_listener" "https" { + load_balancer_arn = aws_lb.grafana.arn + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" + certificate_arn = module.certificate.arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.grafana.arn + } +} + +resource "aws_lb_target_group" "grafana" { + #checkov:skip=CKV_AWS_378: We're happy that this is internal traffic within our vpc and we do not want the complexity cost of setting up TLS between the load balancer and application + name = local.name + port = local.container_port + protocol = "HTTP" + vpc_id = var.vpc_id + target_type = "ip" + + deregistration_delay = "60" + + health_check { + path = "/api/health" + matcher = "200" + protocol = "HTTP" + + interval = 11 + timeout = 10 + unhealthy_threshold = 3 + healthy_threshold = 2 + } +} diff --git a/infra/modules/grafana/datasources.tf b/infra/modules/grafana/datasources.tf new file mode 100644 index 000000000..81d98ac18 --- /dev/null +++ b/infra/modules/grafana/datasources.tf @@ -0,0 +1,54 @@ +# Data sources are provisioned from a file Grafana reads at start up. The +# file is rendered here and written into the container by its command, see +# ecs.tf. All of them authenticate with the task role. +locals { + aws_datasource_json_data = { + authType = "default" + defaultRegion = data.aws_region.current.region + } + + datasources = [ + { + name = "CloudWatch" + uid = "cloudwatch" + type = "cloudwatch" + access = "proxy" + isDefault = true + editable = false + jsonData = local.aws_datasource_json_data + }, + { + name = "X-Ray" + uid = "xray" + type = "grafana-x-ray-datasource" + access = "proxy" + editable = false + jsonData = local.aws_datasource_json_data + }, + { + # Queries metrics ingested through the CloudWatch OTLP endpoint with + # PromQL. The Amazon Managed Service for Prometheus plugin is pointed + # at CloudWatch's Prometheus-compatible API rather than a workspace, + # so requests must be signed for the "monitoring" service, see + # https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-PromQL-Grafana.html + name = "CloudWatch PromQL" + uid = "cloudwatch-promql" + type = "grafana-amazonprometheus-datasource" + access = "proxy" + url = "https://monitoring.${data.aws_region.current.region}.amazonaws.com" + editable = false + jsonData = { + httpMethod = "POST" + sigV4Auth = true + sigV4AuthType = "default" + sigV4Region = data.aws_region.current.region + sigv4Service = "monitoring" + } + } + ] + + datasources_yaml = yamlencode({ + apiVersion = 1 + datasources = local.datasources + }) +} diff --git a/infra/modules/grafana/ecs.tf b/infra/modules/grafana/ecs.tf new file mode 100644 index 000000000..4e4a72a93 --- /dev/null +++ b/infra/modules/grafana/ecs.tf @@ -0,0 +1,154 @@ +locals { + data_path = "/var/lib/grafana" + provisioning_path = "${local.data_path}/provisioning" + + # Grafana teams are exposed to the role mapping as "@/" + github_role_attribute_path = join(" || ", concat( + ["contains(groups[*], '@${var.github_admin_team}') && 'GrafanaAdmin'"], + [for team in var.github_editor_teams : "contains(groups[*], '@${team}') && 'Editor'"] + )) + + environment_variables = [ + { name = "GF_SERVER_ROOT_URL", value = "https://${local.host_name}/" }, + { name = "GF_SERVER_ENABLE_GZIP", value = "true" }, + { name = "GF_PATHS_DATA", value = local.data_path }, + { name = "GF_PATHS_PLUGINS", value = "${local.data_path}/plugins" }, + { name = "GF_PATHS_LOGS", value = "${local.data_path}/logs" }, + { name = "GF_PATHS_PROVISIONING", value = local.provisioning_path }, + { name = "GF_LOG_MODE", value = "console" }, + { name = "GF_DATABASE_TYPE", value = "postgres" }, + { name = "GF_DATABASE_HOST", value = "${aws_rds_cluster.grafana.endpoint}:${local.rds_port}" }, + { name = "GF_DATABASE_NAME", value = aws_rds_cluster.grafana.database_name }, + { name = "GF_DATABASE_USER", value = aws_rds_cluster.grafana.master_username }, + { name = "GF_DATABASE_SSL_MODE", value = "require" }, + { name = "GF_SECURITY_ADMIN_USER", value = "admin" }, + { name = "GF_SECURITY_COOKIE_SECURE", value = "true" }, + { name = "GF_USERS_ALLOW_SIGN_UP", value = "false" }, + { name = "GF_AUTH_GITHUB_ENABLED", value = "true" }, + { name = "GF_AUTH_GITHUB_AUTO_LOGIN", value = "true" }, + { name = "GF_AUTH_GITHUB_ALLOW_SIGN_UP", value = "true" }, + { name = "GF_AUTH_GITHUB_SCOPES", value = "user:email,read:org" }, + { name = "GF_AUTH_GITHUB_ALLOWED_ORGANIZATIONS", value = join(" ", var.github_allowed_organizations) }, + { name = "GF_AUTH_GITHUB_ROLE_ATTRIBUTE_PATH", value = local.github_role_attribute_path }, + { name = "GF_AUTH_GITHUB_ROLE_ATTRIBUTE_STRICT", value = "true" }, + { name = "GF_AUTH_GITHUB_ALLOW_ASSIGN_GRAFANA_ADMIN", value = "true" }, + { name = "GF_UNIFIED_ALERTING_ENABLED", value = "false" }, + { name = "GF_ANALYTICS_REPORTING_ENABLED", value = "false" }, + { name = "GF_ANALYTICS_CHECK_FOR_UPDATES", value = "false" }, + { name = "GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES", value = "false" }, + { name = "GF_NEWS_NEWS_FEED_ENABLED", value = "false" }, + { name = "GF_AWS_ALLOWED_AUTH_PROVIDERS", value = "default" }, + { name = "GF_PLUGINS_PREINSTALL_SYNC", value = "grafana-amazonprometheus-datasource@3.2.0,grafana-x-ray-datasource@2.17.1" }, + { name = "GRAFANA_DATASOURCES_YAML", value = local.datasources_yaml }, + ] + + secrets = [ + { name = "GF_DATABASE_PASSWORD", valueFrom = aws_ssm_parameter.database_password.arn }, + { name = "GF_SECURITY_ADMIN_PASSWORD", valueFrom = aws_ssm_parameter.admin_password.arn }, + { name = "GF_SECURITY_SECRET_KEY", valueFrom = aws_ssm_parameter.secret_key.arn }, + { name = "GF_AUTH_GITHUB_CLIENT_ID", valueFrom = aws_ssm_parameter.github_client_id.arn }, + { name = "GF_AUTH_GITHUB_CLIENT_SECRET", valueFrom = aws_ssm_parameter.github_client_secret.arn }, + ] + + # Write the provisioning file, then hand over to the image's own entrypoint + command = <<-EOT + mkdir -p "$GF_PATHS_PROVISIONING/datasources" "$GF_PATHS_PROVISIONING/dashboards" "$GF_PATHS_PROVISIONING/plugins" "$GF_PATHS_PROVISIONING/alerting" "$GF_PATHS_PROVISIONING/notifiers" && + printf '%s' "$GRAFANA_DATASOURCES_YAML" > "$GF_PATHS_PROVISIONING/datasources/aws.yaml" && + exec /run.sh + EOT + + container_definition = { + name = local.name + image = var.image + essential = true + environment = local.environment_variables + secrets = local.secrets + entryPoint = ["/bin/sh", "-c"] + command = [local.command] + # The image has no VOLUME for its data directory, so a bind mount there + # would not be writable by the grafana user. Grafana writes plugins, + # provisioning and cache files under /var/lib/grafana in the container + # layer instead. + readonlyRootFilesystem = false + portMappings = [ + { + hostPort = local.container_port + protocol = "tcp" + containerPort = local.container_port + } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.grafana.name + awslogs-region = data.aws_region.current.region + awslogs-stream-prefix = local.name + } + } + healthCheck = { + command = ["CMD-SHELL", "curl -sf http://localhost:${local.container_port}/api/health || exit 1"] + interval = 30 + timeout = 5 + retries = 5 + startPeriod = 180 # plugins install before Grafana listens, and the database may be resuming + } + } +} + +resource "aws_ecs_task_definition" "grafana" { + #checkov:skip=CKV_AWS_336:The image has no VOLUME for its data directory, so the root filesystem must be writable + family = local.name + container_definitions = jsonencode([local.container_definition]) + + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "ARM64" + } + + execution_role_arn = aws_iam_role.task_execution.arn + task_role_arn = aws_iam_role.task.arn + + requires_compatibilities = ["FARGATE"] + cpu = var.cpu + memory = var.memory + + network_mode = "awsvpc" + + enable_fault_injection = false +} + +resource "aws_ecs_service" "grafana" { + #checkov:skip=CKV_AWS_332:We don't want to target "LATEST" and get a surprise when a new version is released. + #checkov:skip=CKV2_FORMS_AWS_2:We don't autoscale this service + name = local.name + cluster = var.ecs_cluster_arn + task_definition = aws_ecs_task_definition.grafana.arn + deployment_maximum_percent = "200" + deployment_minimum_healthy_percent = "100" + desired_count = 1 + # Matches the container health check startPeriod, see above + health_check_grace_period_seconds = 180 + + launch_type = "FARGATE" + platform_version = "1.4.0" + + load_balancer { + target_group_arn = aws_lb_target_group.grafana.arn + container_name = local.name + container_port = local.container_port + } + + network_configuration { + subnets = var.private_subnet_ids + security_groups = [aws_security_group.grafana.id] + assign_public_ip = false + } + + # The target group must be attached to the load balancer before the service + # can be created, which only happens once the listener exists. + depends_on = [aws_lb_listener.https] + + lifecycle { + prevent_destroy = true # ECS services cannot be destructively replaced without downtime. This helps to avoid accidentally doing so. + } +} diff --git a/infra/modules/grafana/iam.tf b/infra/modules/grafana/iam.tf new file mode 100644 index 000000000..26eeb6bdb --- /dev/null +++ b/infra/modules/grafana/iam.tf @@ -0,0 +1,152 @@ +## +# Task role: what Grafana itself may do +## +resource "aws_iam_role" "task" { + name = "${var.environment_name}-${local.name}-ecs-task" + description = "Used by Grafana tasks when running" + assume_role_policy = data.aws_iam_policy_document.ecs_tasks_assume_role.json +} + +data "aws_iam_policy_document" "ecs_tasks_assume_role" { + statement { + sid = "AllowECS" + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + +resource "aws_iam_policy" "task" { + #checkov:skip=CKV_AWS_355:CloudWatch and X-Ray read operations cannot be scoped to specific resources + #checkov:skip=CKV_AWS_356:CloudWatch and X-Ray read operations cannot be scoped to specific resources + name = "${var.environment_name}-${local.name}-ecs-task-policy" + policy = data.aws_iam_policy_document.task.json +} + +resource "aws_iam_role_policy_attachment" "task" { + role = aws_iam_role.task.name + policy_arn = aws_iam_policy.task.arn +} + +data "aws_iam_policy_document" "task" { + #checkov:skip=CKV_AWS_356:"*" is necessary here, and all of the actions are read-only. + + # https://grafana.com/docs/grafana/latest/datasources/aws-cloudwatch/configure/ + # + # GetMetricData and ListMetrics also authorise the CloudWatch PromQL endpoint + # (https://monitoring..amazonaws.com/api/v1/*) that the Amazon + # Managed Service for Prometheus data source queries OTLP metrics through, see + # https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-PromQL.html#CloudWatch-PromQL-IAM + statement { + sid = "ReadCloudWatchMetrics" + effect = "Allow" + actions = [ + "cloudwatch:DescribeAlarms", + "cloudwatch:DescribeAlarmsForMetric", + "cloudwatch:DescribeAlarmHistory", + "cloudwatch:ListMetrics", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics", + "cloudwatch:GetInsightRuleReport", + "pi:GetResourceMetrics" + ] + resources = ["*"] + } + + statement { + sid = "ReadCloudWatchLogs" + effect = "Allow" + actions = [ + "logs:DescribeLogGroups", + "logs:ListAggregateLogGroupSummaries", + "logs:GetLogGroupFields", + "logs:StartQuery", + "logs:StopQuery", + "logs:GetQueryResults", + "logs:GetLogEvents", + "logs:FilterLogEvents" + ] + resources = ["*"] + } + + statement { + sid = "ReadResourceMetadataForDataSources" + effect = "Allow" + actions = [ + "ec2:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "tag:GetResources", + "oam:ListSinks", + "oam:ListAttachedLinks" + ] + resources = ["*"] + } + + # https://grafana.com/docs/plugins/grafana-x-ray-datasource/latest/configure/ + statement { + sid = "ReadXRayAndApplicationSignals" + effect = "Allow" + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:GetTraceGraph", + "xray:GetGroups", + "xray:GetTimeSeriesServiceStatistics", + "xray:GetInsightSummaries", + "xray:GetInsight", + "xray:GetServiceGraph", + "xray:GetSamplingRules", + "application-signals:ListServices", + "application-signals:ListServiceOperations", + "application-signals:ListServiceDependencies", + "application-signals:ListServiceLevelObjectives", + "application-signals:GetService", + "application-signals:GetServiceLevelObjective" + ] + resources = ["*"] + } +} + +## +# Execution role: what ECS needs to start the task +## +resource "aws_iam_role" "task_execution" { + name = "${var.environment_name}-${local.name}-ecs-task-execution" + description = "Used by ECS to create Grafana tasks" + assume_role_policy = data.aws_iam_policy_document.ecs_tasks_assume_role.json +} + +resource "aws_iam_role_policy_attachment" "task_execution_standard" { + role = aws_iam_role.task_execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +} + +resource "aws_iam_policy" "task_execution_additional" { + name = "${var.environment_name}-${local.name}-ecs-task-execution-additional" + policy = data.aws_iam_policy_document.task_execution_additional.json +} + +resource "aws_iam_role_policy_attachment" "task_execution_additional" { + role = aws_iam_role.task_execution.name + policy_arn = aws_iam_policy.task_execution_additional.arn +} + +data "aws_iam_policy_document" "task_execution_additional" { + statement { + sid = "ReadSecrets" + effect = "Allow" + actions = ["ssm:GetParameters"] + resources = [ + aws_ssm_parameter.database_password.arn, + aws_ssm_parameter.admin_password.arn, + aws_ssm_parameter.secret_key.arn, + aws_ssm_parameter.github_client_id.arn, + aws_ssm_parameter.github_client_secret.arn + ] + } +} diff --git a/infra/modules/grafana/logging.tf b/infra/modules/grafana/logging.tf new file mode 100644 index 000000000..7e504a920 --- /dev/null +++ b/infra/modules/grafana/logging.tf @@ -0,0 +1,27 @@ +locals { + log_group_name = "/aws/ecs/${local.name}" +} + +resource "aws_cloudwatch_log_group" "grafana" { + #checkov:skip=CKV_AWS_338:We're happy with 30 days retention for now + #checkov:skip=CKV_AWS_158:Default AWS SSE is sufficient, no need for CM KMS. + name = local.log_group_name + retention_in_days = 30 +} + +module "cribl_well_known" { + source = "../well-known/cribl" +} + +resource "aws_cloudwatch_log_subscription_filter" "via_cribl_to_splunk" { + count = var.kinesis_subscription_role_arn != "" ? 1 : 0 + + name = "via-cribl-to-splunk" + + log_group_name = aws_cloudwatch_log_group.grafana.name + + filter_pattern = "" + destination_arn = module.cribl_well_known.kinesis_destination_arns["eu-west-2"] + distribution = "ByLogStream" + role_arn = var.kinesis_subscription_role_arn +} diff --git a/infra/modules/grafana/main.tf b/infra/modules/grafana/main.tf new file mode 100644 index 000000000..cb0bd0f2a --- /dev/null +++ b/infra/modules/grafana/main.tf @@ -0,0 +1,9 @@ +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +locals { + name = "grafana" + container_port = 3000 + host_name = "grafana.${var.root_domain}" + ssm_prefix = "/grafana" +} diff --git a/infra/modules/grafana/outputs.tf b/infra/modules/grafana/outputs.tf new file mode 100644 index 000000000..0ea647a78 --- /dev/null +++ b/infra/modules/grafana/outputs.tf @@ -0,0 +1,7 @@ +output "alb_dns_name" { + value = aws_lb.grafana.dns_name +} + +output "alb_zone_id" { + value = aws_lb.grafana.zone_id +} diff --git a/infra/modules/grafana/parameters.tf b/infra/modules/grafana/parameters.tf new file mode 100644 index 000000000..d11fa6ad2 --- /dev/null +++ b/infra/modules/grafana/parameters.tf @@ -0,0 +1,65 @@ +resource "random_password" "database" { + length = 32 + special = false +} + +resource "aws_ssm_parameter" "database_password" { + #checkov:skip=CKV_AWS_337:The parameter is already using the default key + #checkov:skip=CKV2_FORMS_AWS_7:We know the correct value at runtime and should not ignore changes to it + name = "${local.ssm_prefix}/database/password" + description = "Master password for the Grafana database" + type = "SecureString" + value = random_password.database.result +} + +resource "random_password" "admin" { + length = 32 + special = false +} + +resource "aws_ssm_parameter" "admin_password" { + #checkov:skip=CKV_AWS_337:The parameter is already using the default key + #checkov:skip=CKV2_FORMS_AWS_7:We know the correct value at runtime and should not ignore changes to it + name = "${local.ssm_prefix}/admin-password" + description = "Password for the built-in Grafana admin user, for use when GitHub login is unavailable" + type = "SecureString" + value = random_password.admin.result +} + +resource "random_password" "secret_key" { + length = 32 + special = false +} + +resource "aws_ssm_parameter" "secret_key" { + #checkov:skip=CKV_AWS_337:The parameter is already using the default key + #checkov:skip=CKV2_FORMS_AWS_7:We know the correct value at runtime and should not ignore changes to it + name = "${local.ssm_prefix}/secret-key" + description = "Key Grafana uses to encrypt data source credentials at rest" + type = "SecureString" + value = random_password.secret_key.result +} + +resource "aws_ssm_parameter" "github_client_id" { + #checkov:skip=CKV_AWS_337:The parameter is already using the default key + name = "${local.ssm_prefix}/github/client-id" + description = "Client ID of the GitHub OAuth app used to sign in to Grafana" + type = "SecureString" + value = "dummy-value" + + lifecycle { + ignore_changes = [value] + } +} + +resource "aws_ssm_parameter" "github_client_secret" { + #checkov:skip=CKV_AWS_337:The parameter is already using the default key + name = "${local.ssm_prefix}/github/client-secret" + description = "Client secret of the GitHub OAuth app used to sign in to Grafana" + type = "SecureString" + value = "dummy-value" + + lifecycle { + ignore_changes = [value] + } +} diff --git a/infra/modules/grafana/providers.tf b/infra/modules/grafana/providers.tf new file mode 100644 index 000000000..42bbf08ea --- /dev/null +++ b/infra/modules/grafana/providers.tf @@ -0,0 +1,10 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + random = { + source = "hashicorp/random" + } + } +} diff --git a/infra/modules/grafana/rds.tf b/infra/modules/grafana/rds.tf new file mode 100644 index 000000000..3d5f02a53 --- /dev/null +++ b/infra/modules/grafana/rds.tf @@ -0,0 +1,85 @@ +locals { + rds_port = 5432 +} + +resource "aws_db_subnet_group" "grafana" { + name = local.name + description = "${local.name} subnet group" + subnet_ids = var.private_subnet_ids +} + +resource "aws_security_group" "rds" { + name = "${local.name}-rds" + description = "Security group for the Grafana database" + vpc_id = var.vpc_id + + tags = { + Name = "${local.name}-rds" + } +} + +resource "aws_rds_cluster" "grafana" { + #checkov:skip=CKV2_AWS_8:AWS RDS inbuilt backup process is sufficient + #checkov:skip=CKV2_AWS_27:Query logging is not required at this time + #checkov:skip=CKV_AWS_128:IAM auth to be considered: https://trello.com/c/nY2TcBXb/418-consider-rds-iam-auth + #checkov:skip=CKV_AWS_162:Duplicate of CKV_AWS_128 + #checkov:skip=CKV_AWS_324:Log capture is not required at this time + #checkov:skip=CKV_AWS_327:Database is already encrypted with the default key, and we feel this is sufficient + + cluster_identifier = local.name + + database_name = "grafana" + master_username = "grafana" + master_password = random_password.database.result + port = local.rds_port + + engine = "aurora-postgresql" + engine_mode = "provisioned" + # Only used when the cluster is created; AWS then manages minor upgrades + # (see ignore_changes below). Deprecated minor versions cannot be used to + # create a cluster, so bump this to a current one if creation fails. + engine_version = "18.4" + + vpc_security_group_ids = [aws_security_group.rds.id] + db_subnet_group_name = aws_db_subnet_group.grafana.name + + preferred_maintenance_window = var.rds_maintenance_window + + skip_final_snapshot = false + final_snapshot_identifier = "${local.name}-final-snapshot" + copy_tags_to_snapshot = true + storage_encrypted = true + backup_retention_period = 7 + deletion_protection = true + + # Grafana keeps a connection pool open and polls its database, so the + # cluster will only pause while the Grafana service is stopped. It costs + # nothing to allow it to pause when that happens. + serverlessv2_scaling_configuration { + min_capacity = 0 + max_capacity = 1 + seconds_until_auto_pause = var.seconds_until_auto_pause + } + + lifecycle { + # AWS applies minor version upgrades automatically; don't fight it. + # restore_to_point_in_time must remain ignored, see CKV2_FORMS_AWS_3. + ignore_changes = [ + engine_version, + restore_to_point_in_time + ] + } +} + +resource "aws_rds_cluster_instance" "grafana" { + #checkov:skip=CKV_AWS_118:We don't currently have enhanced monitoring + #checkov:skip=CKV_AWS_354:We can use the default kms key for encryption + #checkov:skip=CKV_AWS_353:Performance insights are not required for this database + + cluster_identifier = aws_rds_cluster.grafana.id + engine = aws_rds_cluster.grafana.engine + instance_class = "db.serverless" + identifier = local.name + + auto_minor_version_upgrade = true +} diff --git a/infra/modules/grafana/security-groups.tf b/infra/modules/grafana/security-groups.tf new file mode 100644 index 000000000..60507383a --- /dev/null +++ b/infra/modules/grafana/security-groups.tf @@ -0,0 +1,48 @@ +resource "aws_security_group" "grafana" { + #checkov:skip=CKV2_AWS_5:The security group is attached in ecs.tf + name = local.name + description = "Ingress from the load balancer, egress to the database and the internet" + vpc_id = var.vpc_id +} + +resource "aws_security_group_rule" "ingress_from_alb" { + description = "Permit inbound from the load balancer to the container port" + type = "ingress" + from_port = local.container_port + to_port = local.container_port + protocol = "tcp" + source_security_group_id = aws_security_group.alb.id + security_group_id = aws_security_group.grafana.id +} + +resource "aws_security_group_rule" "egress_to_rds" { + description = "Permit outbound to the Grafana database" + type = "egress" + from_port = local.rds_port + to_port = local.rds_port + protocol = "tcp" + source_security_group_id = aws_security_group.rds.id + security_group_id = aws_security_group.grafana.id +} + +resource "aws_security_group_rule" "rds_ingress_from_grafana" { + description = "Permit inbound to the database from Grafana" + type = "ingress" + from_port = local.rds_port + to_port = local.rds_port + protocol = "tcp" + source_security_group_id = aws_security_group.grafana.id + security_group_id = aws_security_group.rds.id +} + +# GitHub (sign in), grafana.com (plugin install) and the CloudWatch and +# X-Ray APIs, none of which have VPC endpoints in this VPC. +resource "aws_security_group_rule" "egress_to_internet" { + description = "Permits outbound 443 to the internet" + type = "egress" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.grafana.id +} diff --git a/infra/modules/grafana/variables.tf b/infra/modules/grafana/variables.tf new file mode 100644 index 000000000..f237239f7 --- /dev/null +++ b/infra/modules/grafana/variables.tf @@ -0,0 +1,79 @@ +variable "environment_name" { + description = "The name of the environment, used to prefix IAM role and policy names" + type = string +} + +variable "root_domain" { + description = "The root domain of the environment; Grafana is served at grafana." + type = string +} + +variable "vpc_id" { + type = string +} + +variable "vpc_cidr_block" { + type = string +} + +variable "private_subnet_ids" { + type = list(string) +} + +variable "ecs_cluster_arn" { + type = string +} + +variable "public_subnet_ids" { + type = list(string) +} + +variable "alb_access_logs_bucket_name" { + description = "The S3 bucket that receives ALB access logs. Its policy must permit the grafana/ prefix." + type = string +} + +variable "kinesis_subscription_role_arn" { + description = "The ARN of the role used by CloudWatch Logs to ship logs to Cribl/Splunk. Empty string disables shipping." + type = string + default = "" +} + +variable "cpu" { + type = number +} + +variable "memory" { + type = number +} + +variable "image" { + description = "The Grafana container image" + type = string + default = "docker.io/grafana/grafana:13.2.1" +} + +variable "github_allowed_organizations" { + description = "GitHub organisations whose members may sign in" + type = list(string) +} + +variable "github_admin_team" { + description = "GitHub team (org/team) whose members are Grafana server admins" + type = string +} + +variable "github_editor_teams" { + description = "GitHub teams (org/team) whose members are Grafana editors. Members of no listed team are denied." + type = list(string) +} + +variable "seconds_until_auto_pause" { + description = "How long the database must be idle before it pauses" + type = number + default = 3600 +} + +variable "rds_maintenance_window" { + type = string +}