diff --git a/.github/workflows/modules-docker-build-push.yaml b/.github/workflows/modules-docker-build-push.yaml index 03444d3..bac181a 100644 --- a/.github/workflows/modules-docker-build-push.yaml +++ b/.github/workflows/modules-docker-build-push.yaml @@ -75,7 +75,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx + - name: Set up Docker Buildx 🐳 uses: docker/setup-buildx-action@v3 - name: Login to DockerHub @@ -94,10 +94,10 @@ jobs: file: ${{ inputs.context }}/${{ inputs.dockerfile }} platforms: ${{ inputs.platforms }} push: ${{ github.event_name != 'pull_request' }} - load: ${{ github.event_name == 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} + sbom: true cache-from: type=gha cache-to: type=gha,mode=max build-args: | diff --git a/examples/kind/locals.tf b/examples/kind/locals.tf index 3d6606b..8e1508d 100644 --- a/examples/kind/locals.tf +++ b/examples/kind/locals.tf @@ -5,7 +5,7 @@ resource "random_password" "airflow_fernetKey" { locals { kubernetes_version = "v1.35.0" cluster_name = "kind" - base_domain = format("%s.nip.io", replace(module.traefik.external_ip, ".", "-")) + base_domain = format("%s.nip.io", replace(module.istio.external_ip, ".", "-")) subdomain = "apps" cluster_issuer = module.cert-manager.cluster_issuers.ca enable_service_monitor = false # Can be enabled after the first bootstrap. diff --git a/examples/kind/main.tf b/examples/kind/main.tf index 6c25658..e38ff44 100644 --- a/examples/kind/main.tf +++ b/examples/kind/main.tf @@ -27,7 +27,7 @@ module "argocd_bootstrap" { repositories = [ "git@github.com:GersonRS/modern-gitops-stack-module-argocd.git", "git@github.com:GersonRS/modern-gitops-stack-module-metrics-server.git", - "git@github.com:GersonRS/modern-gitops-stack-module-traefik.git", + "git@github.com:GersonRS/modern-gitops-stack-module-istio.git", "git@github.com:GersonRS/modern-gitops-stack-module-cert-manager.git", "git@github.com:GersonRS/modern-gitops-stack-module-keycloak.git", "git@github.com:GersonRS/modern-gitops-stack-module-postgresql.git", @@ -68,10 +68,14 @@ module "metrics-server" { } } -module "traefik" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-traefik.git//kind?ref=v2.9.0" +module "istio" { + source = "git::https://github.com/GersonRS/modern-gitops-stack-module-istio.git//kind?ref=v1.1.1" - argocd_project = local.cluster_name + cluster_name = local.cluster_name + subdomain = local.subdomain + cluster_issuer = local.cluster_issuer + argocd_project = local.cluster_name + target_revision = "v1.1.1" app_autosync = local.app_autosync enable_service_monitor = local.enable_service_monitor @@ -110,7 +114,7 @@ module "postgresql" { } module "keycloak" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-keycloak.git?ref=develop" + source = "git::https://github.com/GersonRS/modern-gitops-stack-module-keycloak.git?ref=v2.8.0" cluster_name = local.cluster_name base_domain = local.base_domain @@ -118,7 +122,7 @@ module "keycloak" { cluster_issuer = local.cluster_issuer argocd_project = local.cluster_name - target_revision = "develop" + target_revision = "v2.8.0" app_autosync = local.app_autosync database = { @@ -128,14 +132,14 @@ module "keycloak" { } dependency_ids = { - traefik = module.traefik.id + istio = module.istio.id cert-manager = module.cert-manager.id postgresql = module.postgresql.id } } module "oidc" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-keycloak.git//oidc_bootstrap?ref=develop" + source = "git::https://github.com/GersonRS/modern-gitops-stack-module-keycloak.git//oidc_bootstrap?ref=v2.8.0" cluster_name = local.cluster_name base_domain = local.base_domain @@ -149,120 +153,150 @@ module "oidc" { -module "minio" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-minio.git?ref=v2.8.0" +# module "minio" { +# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-minio.git?ref=v2.9.0" +# +# cluster_name = local.cluster_name +# base_domain = local.base_domain +# subdomain = local.subdomain +# cluster_issuer = local.cluster_issuer +# argocd_project = local.cluster_name +# app_autosync = local.app_autosync +# enable_service_monitor = local.enable_service_monitor +# config_minio = local.minio_config +# oidc = module.oidc.oidc +# +# target_revision = "v2.9.0" +# +# dependency_ids = { +# istio = module.istio.id +# cert-manager = module.cert-manager.id +# oidc = module.oidc.id +# } +# } +# --- Modules below commented out - migrating one by one --- + +# module "mlflow" { +# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-mlflow.git?ref=v1.4.0" +# +# cluster_name = local.cluster_name +# base_domain = local.base_domain +# subdomain = local.subdomain +# cluster_issuer = local.cluster_issuer +# argocd_project = local.cluster_name +# app_autosync = local.app_autosync +# enable_service_monitor = local.enable_service_monitor +# +# target_revision = "v1.4.0" +# +# oidc = module.oidc.oidc +# allowed_groups = [ +# "modern-gitops-stack-admins", +# "modern-gitops-stack-data-scientists", +# "modern-gitops-stack-ml-engineers", +# ] +# +# storage = { +# bucket_name = "mlflow" +# endpoint = module.minio.endpoint +# access_key = module.minio.minio_root_user_credentials.username +# secret_access_key = module.minio.minio_root_user_credentials.password +# } +# database = { +# user = module.postgresql.credentials.username +# password = module.postgresql.credentials.password +# database = "mlflow" +# service = module.postgresql.cluster_dns +# } +# dependency_ids = { +# argocd = module.argocd_bootstrap.id +# istio = module.istio.id +# minio = module.minio.id +# postgresql = module.postgresql.id +# oidc = module.oidc.id +# } +# } + +module "strimzi" { + source = "git::https://github.com/GersonRS/modern-gitops-stack-module-strimzi.git?ref=v1.5.0" cluster_name = local.cluster_name base_domain = local.base_domain subdomain = local.subdomain cluster_issuer = local.cluster_issuer argocd_project = local.cluster_name + target_revision = "v1.5.0" app_autosync = local.app_autosync enable_service_monitor = local.enable_service_monitor - config_minio = local.minio_config - oidc = module.oidc.oidc - - dependency_ids = { - traefik = module.traefik.id - cert-manager = module.cert-manager.id - oidc = module.oidc.id + argocd = module.argocd_bootstrap.id } } -module "mlflow" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-mlflow.git?ref=v1.3.0" +module "kafka" { + source = "git::https://github.com/GersonRS/modern-gitops-stack-module-kafka.git?ref=v2.12.0" cluster_name = local.cluster_name base_domain = local.base_domain subdomain = local.subdomain cluster_issuer = local.cluster_issuer argocd_project = local.cluster_name + target_revision = "v2.12.0" app_autosync = local.app_autosync enable_service_monitor = local.enable_service_monitor - - - storage = { - bucket_name = "mlflow" - endpoint = module.minio.endpoint - access_key = module.minio.minio_root_user_credentials.username - secret_access_key = module.minio.minio_root_user_credentials.password - } - database = { - user = module.postgresql.credentials.username - password = module.postgresql.credentials.password - database = "mlflow" - service = module.postgresql.cluster_dns - } dependency_ids = { - argocd = module.argocd_bootstrap.id - traefik = module.traefik.id - minio = module.minio.id - postgresql = module.postgresql.id + argocd = module.argocd_bootstrap.id + strimzi = module.strimzi.id } } -# # # module "strimzi" { -# # # source = "git::https://github.com/GersonRS/modern-gitops-stack-module-strimzi.git?ref=v1.4.3" -# # # cluster_name = local.cluster_name -# # # base_domain = local.base_domain -# # # subdomain = local.subdomain -# # # cluster_issuer = local.cluster_issuer -# # # argocd_project = local.cluster_name -# # # app_autosync = local.app_autosync -# # # enable_service_monitor = local.enable_service_monitor -# # # dependency_ids = { -# # # argocd = module.argocd_bootstrap.id -# # # } -# # # } - -# # # module "kafka" { -# # # source = "git::https://github.com/GersonRS/modern-gitops-stack-module-kafka.git?ref=v2.11.0" -# # # cluster_name = local.cluster_name -# # # base_domain = local.base_domain -# # # subdomain = local.subdomain -# # # cluster_issuer = local.cluster_issuer -# # # argocd_project = local.cluster_name -# # # app_autosync = local.app_autosync -# # # enable_service_monitor = local.enable_service_monitor -# # # dependency_ids = { -# # # argocd = module.argocd_bootstrap.id -# # # traefik = module.traefik.id -# # # strimzi = module.strimzi.id -# # # } -# # # } - -# # # module "cp-schema-registry" { -# # # source = "git::https://github.com/GersonRS/modern-gitops-stack-module-cp-schema-registry.git?ref=v1.3.2" -# # # cluster_name = local.cluster_name -# # # base_domain = local.base_domain -# # # subdomain = local.subdomain -# # # cluster_issuer = local.cluster_issuer -# # # argocd_project = local.cluster_name -# # # app_autosync = local.app_autosync -# # # enable_service_monitor = local.enable_service_monitor -# # # kafka_broker_name = module.kafka.broker_name -# # # dependency_ids = { -# # # argocd = module.argocd_bootstrap.id -# # # kafka = module.kafka.id -# # # } -# # # } +module "cp-schema-registry" { + source = "git::https://github.com/GersonRS/modern-gitops-stack-module-cp-schema-registry.git?ref=v1.5.0" + cluster_name = local.cluster_name + base_domain = local.base_domain + subdomain = local.subdomain + cluster_issuer = local.cluster_issuer + argocd_project = local.cluster_name + target_revision = "v1.5.0" + app_autosync = local.app_autosync + enable_service_monitor = local.enable_service_monitor + kafka_broker_name = module.kafka.broker_name + gateway_name = module.istio.gateway_name + gateway_namespace = module.istio.gateway_namespace + oidc = module.oidc.oidc + allowed_groups = [ + "modern-gitops-stack-admins", + "modern-gitops-stack-data-engineers", + ] + dependency_ids = { + argocd = module.argocd_bootstrap.id + kafka = module.kafka.id + } +} -# # # module "kafka-ui" { -# # # source = "git::https://github.com/GersonRS/modern-gitops-stack-module-kafka-ui.git?ref=v1.3.3" -# # # cluster_name = local.cluster_name -# # # base_domain = local.base_domain -# # # subdomain = local.subdomain -# # # cluster_issuer = local.cluster_issuer -# # # argocd_project = local.cluster_name -# # # app_autosync = local.app_autosync -# # # enable_service_monitor = local.enable_service_monitor -# # # kafka_broker_name = module.kafka.broker_name -# # # dependency_ids = { -# # # argocd = module.argocd_bootstrap.id -# # # kafka = module.kafka.id -# # # cp-schema-registry = module.cp-schema-registry.id -# # # } -# # # } +module "kafka-ui" { + source = "git::https://github.com/GersonRS/modern-gitops-stack-module-kafka-ui.git?ref=v1.5.0" + cluster_name = local.cluster_name + base_domain = local.base_domain + subdomain = local.subdomain + cluster_issuer = local.cluster_issuer + argocd_project = local.cluster_name + target_revision = "v1.5.0" + app_autosync = local.app_autosync + enable_service_monitor = local.enable_service_monitor + kafka_broker_name = module.kafka.broker_name + gateway_name = module.istio.gateway_name + gateway_namespace = module.istio.gateway_namespace + oidc = module.oidc.oidc + allowed_groups = [ + "modern-gitops-stack-admins", + "modern-gitops-stack-data-engineers", + ] + dependency_ids = { + argocd = module.argocd_bootstrap.id + kafka = module.kafka.id + cp-schema-registry = module.cp-schema-registry.id + } +} # # # module "pinot" { # # # source = "git::https://github.com/GersonRS/modern-gitops-stack-module-pinot.git?ref=v1.1.1" @@ -347,97 +381,118 @@ module "mlflow" { # # # } # # # } -module "loki-stack" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-loki-stack.git//kind?ref=v2.7.0" - - argocd_project = local.cluster_name - - app_autosync = local.app_autosync - enable_service_monitor = local.enable_service_monitor - - - logs_storage = { - bucket_name = local.minio_config.buckets.0.name - endpoint = module.minio.endpoint - access_key = local.minio_config.users.0.accessKey - secret_key = local.minio_config.users.0.secretKey - } - - dependency_ids = { - minio = module.minio.id - } -} - -module "thanos" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-thanos.git//kind?ref=v2.7.0" - - cluster_name = local.cluster_name - base_domain = local.base_domain - subdomain = local.subdomain - cluster_issuer = local.cluster_issuer - argocd_project = local.cluster_name - - app_autosync = local.app_autosync - enable_service_monitor = local.enable_service_monitor - - - metrics_storage = { - bucket_name = local.minio_config.buckets.1.name - endpoint = module.minio.endpoint - access_key = local.minio_config.users.1.accessKey - secret_key = local.minio_config.users.1.secretKey - } - - thanos = { - oidc = module.oidc.oidc - } - - dependency_ids = { - argocd = module.argocd_bootstrap.id - traefik = module.traefik.id - cert-manager = module.cert-manager.id - minio = module.minio.id - keycloak = module.keycloak.id - oidc = module.oidc.id - } -} - -module "kube-prometheus-stack" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-kube-prometheus-stack.git//kind?ref=v2.7.0" - - cluster_name = local.cluster_name - base_domain = local.base_domain - subdomain = local.subdomain - cluster_issuer = local.cluster_issuer - argocd_project = local.cluster_name - - app_autosync = local.app_autosync - - - metrics_storage = { - bucket_name = local.minio_config.buckets.1.name - endpoint = module.minio.endpoint - access_key = local.minio_config.users.1.accessKey - secret_key = local.minio_config.users.1.secretKey - } +# module "loki-stack" { +# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-loki-stack.git//kind?ref=v2.8.0" +# +# argocd_project = local.cluster_name +# +# app_autosync = local.app_autosync +# enable_service_monitor = local.enable_service_monitor +# +# target_revision = "v2.8.0" +# +# logs_storage = { +# bucket_name = local.minio_config.buckets.0.name +# endpoint = module.minio.endpoint +# access_key = local.minio_config.users.0.accessKey +# secret_key = local.minio_config.users.0.secretKey +# } +# +# dependency_ids = { +# istio = module.istio.id +# minio = module.minio.id +# } +# } - prometheus = { - oidc = module.oidc.oidc - } - alertmanager = { - oidc = module.oidc.oidc - } - grafana = { - oidc = module.oidc.oidc - } +# module "thanos" { +# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-thanos.git//kind?ref=v2.8.0" +# +# cluster_name = local.cluster_name +# base_domain = local.base_domain +# subdomain = local.subdomain +# cluster_issuer = local.cluster_issuer +# argocd_project = local.cluster_name +# +# app_autosync = local.app_autosync +# enable_service_monitor = local.enable_service_monitor +# +# target_revision = "v2.8.0" +# +# metrics_storage = { +# bucket_name = local.minio_config.buckets.1.name +# endpoint = module.minio.endpoint +# access_key = local.minio_config.users.1.accessKey +# secret_key = local.minio_config.users.1.secretKey +# } +# +# thanos = { +# oidc = module.oidc.oidc +# } +# +# allowed_groups = [ +# "modern-gitops-stack-admins", +# "modern-gitops-stack-editors", +# "modern-gitops-stack-data-engineers", +# "modern-gitops-stack-ml-engineers", +# "modern-gitops-stack-data-scientists", +# ] +# +# dependency_ids = { +# argocd = module.argocd_bootstrap.id +# istio = module.istio.id +# cert-manager = module.cert-manager.id +# minio = module.minio.id +# keycloak = module.keycloak.id +# oidc = module.oidc.id +# } +# } - dependency_ids = { - traefik = module.traefik.id - cert-manager = module.cert-manager.id - minio = module.minio.id - oidc = module.oidc.id - } -} +# module "kube-prometheus-stack" { +# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-kube-prometheus-stack.git//kind?ref=v2.8.0" +# +# cluster_name = local.cluster_name +# base_domain = local.base_domain +# subdomain = local.subdomain +# cluster_issuer = local.cluster_issuer +# argocd_project = local.cluster_name +# +# app_autosync = local.app_autosync +# +# target_revision = "v2.8.0" +# +# metrics_storage = { +# bucket_name = local.minio_config.buckets.1.name +# endpoint = module.minio.endpoint +# access_key = local.minio_config.users.1.accessKey +# secret_key = local.minio_config.users.1.secretKey +# } +# +# prometheus = { +# oidc = module.oidc.oidc +# } +# alertmanager = { +# oidc = module.oidc.oidc +# } +# grafana = { +# oidc = module.oidc.oidc +# } +# +# allowed_groups = [ +# "modern-gitops-stack-admins", +# "modern-gitops-stack-viewers", +# "modern-gitops-stack-editors", +# "modern-gitops-stack-data-engineers", +# "modern-gitops-stack-ml-engineers", +# "modern-gitops-stack-data-scientists", +# ] +# +# dependency_ids = { +# istio = module.istio.id +# cert-manager = module.cert-manager.id +# minio = module.minio.id +# oidc = module.oidc.id +# } +# } # # module "spark" { # # source = "git::https://github.com/GersonRS/modern-gitops-stack-module-spark.git?ref=v1.5.1" @@ -497,46 +552,41 @@ module "kube-prometheus-stack" { # # } # # } -# # module "airflow" { -# # source = "git::https://github.com/GersonRS/modern-gitops-stack-module-airflow.git?ref=v1.6.2" -# # cluster_name = local.cluster_name -# # base_domain = local.base_domain -# # subdomain = local.subdomain -# # cluster_issuer = local.cluster_issuer -# # argocd_project = local.cluster_name -# # app_autosync = local.app_autosync -# # enable_service_monitor = local.enable_service_monitor -# # oidc = module.oidc.oidc -# # fernetKey = base64encode(resource.random_password.airflow_fernetKey.result) -# # storage = { -# # bucket_name = "airflow" -# # endpoint = module.minio.endpoint -# # access_key = module.minio.minio_root_user_credentials.username -# # secret_access_key = module.minio.minio_root_user_credentials.password -# # } -# # database = { -# # database = "airflow" -# # user = module.postgresql.credentials.user -# # password = module.postgresql.credentials.password -# # endpoint = module.postgresql.cluster_dns -# # } -# # # mlflow = { -# # # endpoint = module.mlflow.cluster_dns -# # # } -# # # ray = { -# # # endpoint = module.ray.cluster_dns -# # # } -# # dependency_ids = { -# # argocd = module.argocd_bootstrap.id -# # traefik = module.traefik.id -# # oidc = module.oidc.id -# # minio = module.minio.id -# # postgresql = module.postgresql.id -# # } -# # } +# module "airflow" { +# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-airflow.git?ref=v1.7.0" +# cluster_name = local.cluster_name +# base_domain = local.base_domain +# subdomain = local.subdomain +# cluster_issuer = local.cluster_issuer +# argocd_project = local.cluster_name +# app_autosync = local.app_autosync +# enable_service_monitor = local.enable_service_monitor +# target_revision = "v1.7.0" +# oidc = module.oidc.oidc +# fernetKey = base64encode(resource.random_password.airflow_fernetKey.result) +# storage = { +# bucket_name = local.minio_config.buckets.3.name +# endpoint = module.minio.endpoint +# access_key = local.minio_config.users.3.accessKey +# secret_access_key = local.minio_config.users.3.secretKey +# } +# database = { +# database = "airflow" +# user = module.postgresql.credentials.username +# password = module.postgresql.credentials.password +# endpoint = module.postgresql.cluster_dns +# } +# dependency_ids = { +# argocd = module.argocd_bootstrap.id +# istio = module.istio.id +# oidc = module.oidc.id +# minio = module.minio.id +# postgresql = module.postgresql.id +# } +# } # module "jupyterhub" { -# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-jupyterhub.git?ref=develop" +# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-jupyterhub.git?ref=v1.2.0" # cluster_name = local.cluster_name # base_domain = local.base_domain # subdomain = local.subdomain @@ -544,7 +594,7 @@ module "kube-prometheus-stack" { # argocd_project = local.cluster_name # app_autosync = local.app_autosync # oidc = module.oidc.oidc -# target_revision = "develop" +# target_revision = "v1.2.0" # storage = { # bucket_name = "mlflow" # endpoint = module.minio.endpoint @@ -591,37 +641,38 @@ module "kube-prometheus-stack" { # # } # # } -module "argocd" { - source = "git::https://github.com/GersonRS/modern-gitops-stack-module-argocd.git?ref=v4.1.0" - - base_domain = local.base_domain - cluster_name = local.cluster_name - subdomain = local.subdomain - cluster_issuer = local.cluster_issuer - server_secretkey = module.argocd_bootstrap.argocd_server_secretkey - accounts_pipeline_tokens = module.argocd_bootstrap.argocd_accounts_pipeline_tokens - argocd_project = local.cluster_name - app_autosync = local.app_autosync - - admin_enabled = false - exec_enabled = true - - oidc = { - name = "OIDC" - issuer = module.oidc.oidc.issuer_url - clientID = module.oidc.oidc.client_id - clientSecret = module.oidc.oidc.client_secret - requestedIDTokenClaims = { - groups = { - essential = true - } - } - } - - dependency_ids = { - traefik = module.traefik.id - cert-manager = module.cert-manager.id - oidc = module.oidc.id - kube-prometheus-stack = module.kube-prometheus-stack.id - } -} +# module "argocd" { +# source = "git::https://github.com/GersonRS/modern-gitops-stack-module-argocd.git?ref=v4.2.0" +# +# base_domain = local.base_domain +# cluster_name = local.cluster_name +# subdomain = local.subdomain +# cluster_issuer = local.cluster_issuer +# target_revision = "v4.2.0" +# server_secretkey = module.argocd_bootstrap.argocd_server_secretkey +# accounts_pipeline_tokens = module.argocd_bootstrap.argocd_accounts_pipeline_tokens +# argocd_project = local.cluster_name +# app_autosync = local.app_autosync +# +# admin_enabled = false +# exec_enabled = true +# +# oidc = { +# name = "OIDC" +# issuer = module.oidc.oidc.issuer_url +# clientID = module.oidc.oidc.client_id +# clientSecret = module.oidc.oidc.client_secret +# requestedIDTokenClaims = { +# groups = { +# essential = true +# } +# } +# } +# +# dependency_ids = { +# istio = module.istio.id +# cert-manager = module.cert-manager.id +# oidc = module.oidc.id +# kube-prometheus-stack = module.kube-prometheus-stack.id +# } +# }