diff --git a/base.yml b/base.yml index 99d6fea0..93d3a589 100644 --- a/base.yml +++ b/base.yml @@ -228,6 +228,7 @@ nav: - docs/operation/creds/aws-ssm.md - docs/operation/creds/conjur.md - docs/operation/creds/credhub.md + - docs/operation/creds/gcp-secret.md - docs/operation/creds/id-token.md - docs/operation/creds/kubernetes.md - docs/operation/creds/vault.md diff --git a/docs/docs/operation/creds/gcp-secret.md b/docs/docs/operation/creds/gcp-secret.md new file mode 100644 index 00000000..c1df5c0d --- /dev/null +++ b/docs/docs/operation/creds/gcp-secret.md @@ -0,0 +1,292 @@ +--- +title: GCP Secret Manager credential manager +--- + +Concourse can be configured to pull credentials +from [GCP Secret Manager](https://cloud.google.com/security/products/secret-manager). + +## Configuration + +To enable this credential manager, configure the following environment variables on the [ +`web` node](../../install/running-web.md): + +```properties +CONCOURSE_GCP_SECRETMANAGER_PROJECT=12345601234 +CONCOURSE_GCP_SECRETMANAGER_CREDENTIALS_JSON="{ 'type' : 'service_account', ... }" +``` + +### Credential File Configuration + +GCP Secret Manager can also be configured to use a Credentials File instead of an inline JSON string by using the +following environment variables: + +```properties +CONCOURSE_GCP_SECRETMANAGER_PROJECT=12345601234 +CONCOURSE_GCP_SECRETMANAGER_CREDENTIALS_FILE="/tmp/credentials.json" +``` + +### Workload Identity Configuration + +GCP Secrets Manager can also be configured to use [Workload Identity]() assigned on the [ +`web` node](../../install/running-web.md). When using Workload Identity, credentials are fetched automatically from ... +and only the Project needs to be configured: + +```properties +CONCOURSE_GCP_SECRETMANAGER_PROJECT=12345601234 +``` + +### IAM Permissions + +The following is an example of an IAM policy that can be used to grant permissions to an IAM user or instance role. + +!!! note + + The `Resource` section can contain a wildcard to a secret or be restricted to an individual secret. + +In order for the health check to work properly (see [Scaling](#scaling)), Concourse needs to have access to the +`__concourse-health-check` secret. + +=== "JSON" + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAccessToSecretManagerParameters", + + "Effect": "Allow", + + "Action": [ + "secretsmanager:ListSecrets" + ], + + "Resource": "*" + }, + { + "Sid": "AllowAccessGetSecret", + + "Effect": "Allow", + + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ], + + "Resource": [ + "arn:aws:secretsmanager:*:*:secret:/concourse/*", + "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" + ] + } + ] + } + ``` + +=== "Terraform / OpenTofu" + + ```hcl + data "aws_iam_policy_document" "secrets_lookup" { + statement { + sid = "AllowAccessToSecretManagerParameters" + + effect = "Allow" + + actions = [ + "secretsmanager:ListSecrets" + ] + + resources = [ + "*", + ] + } + + statement { + sid = "AllowAccessGetSecret" + + effect = "Allow" + + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ] + + resources = [ + "arn:aws:secretsmanager:*:*:secret:/concourse/*", + "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" + ] + } + } + ``` + +If you wish to restrict concourse to only have access to secrets for a specific pipeline, you can replace +`"arn:aws:secretsmanager:*:*:secret:/concourse/*"` in the example above with: + +=== "JSON" + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAccessToSecretManagerParameters", + + "Effect": "Allow", + + "Action": [ + "secretsmanager:ListSecrets" + ], + + "Resource": "*" + }, + { + "Sid": "AllowAccessGetSecret", + + "Effect": "Allow", + + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ], + + "Resource": [ + "arn:aws:secretsmanager:*:*:secret:/concourse/TEAM_NAME/*", + "arn:aws:secretsmanager:*:*:secret:/concourse/TEAM_NAME/PIPELINE_NAME/*", + "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" + ] + } + ] + } + ``` + +=== "Terraform / OpenTofu" + + ```hcl + variable "team_name" { + type = string + default = "my_team" + } + + variable "pipeline_name" { + type = string + default = "my_pipeline" + } + + data "aws_iam_policy_document" "secrets_lookup" { + statement { + sid = "AllowAccessToSecretManagerParameters" + + effect = "Allow" + + actions = [ + "secretsmanager:ListSecrets" + ] + + resources = [ + "*", + ] + } + + statement { + sid = "AllowAccessGetSecret" + + effect = "Allow" + + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ] + + resources = [ + "arn:aws:secretsmanager:*:*:secret:/concourse/${var.team_name}/*", + "arn:aws:secretsmanager:*:*:secret:/concourse/${var.team_name}/${var.pipeline_name}/*", + "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" + ] + } + } + ``` + +where `TEAM_NAME` and `PIPELINE_NAME` are replaced with the team and name of the pipeline in question. + +For more information on how to use IAM roles to restrict access to Secrets Manager, review +the [official documentation](https://docs.aws.amazon.com/secretsmanager/latest/userguide/auth-and-access_identity-based-policies.html). + +### Scaling + +If your cluster has a large workload, in particular if there are many resources, Concourse can generate a lot of traffic +to GCP and subsequently get rate-limited. + +As long as Concourse has permission to get the value of the `__concourse-health-check` secret, you should be able to +measure an error rate by polling the `/api/v1/info/creds` endpoint when authenticated as +a [Concourse Admin](../../auth-and-teams/user-roles.md#concourse-admin). + +Depending on your workflow for updating secrets and your reliability requirements it may be +worth [Caching credentials](caching.md) and/or [Retrying failed fetches](retrying-failed.md) to mitigate +rate-limit-related errors. + +#### Configuring Request Timeout + +In addition to Global Concourse controls, the GCP Secret Manager can also be configured to have a longer timeout using +the following environment variable: + +```properties +CONCOURSE_GCP_SECRETMANAGER_REQUEST_TIMEOUT="20s" +``` + +## Credential Lookup Rules + +When resolving a parameter such as `((foo_param))`, it will look in the following paths, in order: + +* `concourse--TEAM_NAME--PIPELINE_NAME--foo_param` +* `concourse--TEAM_NAME--foo_param` + +If the action is being run in the context of a pipeline (e.g. a `check` or a step in a build of a job), the ATC will +first look in the pipeline path. If it's not found there, it will look in the team path. This allows credentials to be +scoped widely if they're common across many pipelines. + +When executing a one-off task, there is no pipeline: so in this case, only the team path `concourse--TEAM_NAME--foo` is +searched. + +There are several ways to customize the lookup logic: + +1. Add a "shared path", for secrets common to all teams. +2. Change the team- and pipeline-dependent path templates. + +Each of these can be controlled by Concourse command line flags, or environment variables. + +### Configuring a shared path + +A "shared path" can also be configured for credentials that you would like to share across all teams and pipelines, +foregoing the default team/pipeline namespacing. Use with care! + +```properties +CONCOURSE_GCP_SECRETMANAGER_SHARED_SECRET_TEMPLATE=some-shared-path +``` + +This path must exist under the configured path prefix. The above configuration would correspond to +`concourse--some-shared-path` with the default `concourse` prefix. + +### Changing the path templates + +You can choose your own list of templates, which will expand to team- or pipeline-specific paths. By default, the +templates used are: + +```properties +CONCOURSE_GCP_SECRETMANAGER_TEAM_SECRET_TEMPLATE=concourse--{{.Team}}--{{.Secret}} +CONCOURSE_GCP_SECRETMANAGER_PIPELINE_SECRET_TEMPLATE=concourse--{{.Team}}--{{.Pipeline}}--{{.Secret}} +``` + +When secrets are to be looked up, these are evaluated where `{{.Team}}` expands to the current team, `{{.Pipeline}}` to +the current pipeline (if any), and `{{.Secret}}` to the name of the secret. So if the settings are: + +```properties +CONCOURSE_GCP_SECRETMANAGER_TEAM_SECRET_TEMPLATE={{.Team}}--concourse--{{.Secret}} +CONCOURSE_GCP_SECRETMANAGER_PIPELINE_SECRET_TEMPLATE={{.Team}}--concourse--{{.Pipeline}}--{{.Secret}} +CONCOURSE_GCP_SECRETMANAGER_SHARED_SECRET_TEMPLATE=common--{{.Secret}} +``` + +and `((password))` is used in team `myteam` and pipeline `mypipeline`, Concourse will look for the following, in order: + +1. `myteam--concourse--mypipeline--password` +2. `myteam--concourse--password` +3. `common--password` + diff --git a/docs/docs/operation/creds/index.md b/docs/docs/operation/creds/index.md index 1ef9e971..534a1514 100644 --- a/docs/docs/operation/creds/index.md +++ b/docs/docs/operation/creds/index.md @@ -45,6 +45,11 @@ relevant section below for whichever backend you want to use. --- [:octicons-arrow-right-24: Configure](credhub.md) +- :material-google-cloud: GCP Secret Manager + + --- + [:octicons-arrow-right-24: Configure](gcp-secret.md) + - :material-openid: IDToken ---