Add Grafana, Loki, Alloy containers

data.tf

@@ -39,6 +39,9 @@ data "oci_secrets_secretbundle" "redis_passwords" {
   secret_id = each.value.id
 }
 
+data "oci_secrets_secretbundle" "grafana_secret_key_secret" {
+  secret_id = oci_vault_secret.grafana_secret_key_secret.id
+}
 
 data "oci_core_images" "main" {
   compartment_id = oci_identity_compartment.main.id

dns.tf

@@ -17,7 +17,7 @@ resource "cloudflare_dns_record" "www" {
 }
 
 resource "cloudflare_dns_record" "tunnels" {
-  for_each = toset(["@", "webdav", "vault", "rss", "status", "auth", "wireguard"])
+  for_each = toset(["@", "webdav", "vault", "rss", "status", "auth", "wireguard", "logs"])
   content  = "${cloudflare_zero_trust_tunnel_cloudflared.main.id}.cfargotunnel.com"
   name     = each.value
   proxied  = true

files/authelia.configuration.yml

@@ -92,6 +92,14 @@ notifier:
 identity_providers:
   oidc:
     hmac_secret: '{{ secret "/run/secrets/hmac-secret" }}'
+    claims_policies:
+      # https://www.authelia.com/integration/openid-connect/clients/grafana/#configuration-escape-hatch
+      grafana:
+        id_token:
+          - email
+          - name
+          - groups
+          - preferred_username
     jwks:
       - key_id: "main"
         algorithm: "RS256"
@@ -169,3 +177,23 @@ identity_providers:
         pre_configured_consent_duration: 1 week
         require_pkce: true
         token_endpoint_auth_method: client_secret_post
+      - client_id: '{{ env "GRAFANA_CLIENT_ID" }}'
+        client_name: "Grafana"
+        client_secret: '{{ env "GRAFANA_CLIENT_SECRET" }}'
+        require_pkce: true
+        pkce_challenge_method: "S256"
+        redirect_uris:
+          - 'https://logs.{{ env "SERVER_DOMAIN" }}/login/generic_oauth'
+        scopes:
+          - "openid"
+          - "profile"
+          - "groups"
+          - "email"
+        claims_policy: "grafana"
+        response_types:
+          - "code"
+        grant_types:
+          - "authorization_code"
+        access_token_signed_response_alg: "none"
+        userinfo_signed_response_alg: "none"
+        token_endpoint_auth_method: "client_secret_basic"

files/compose.yaml

@@ -5,6 +5,11 @@ x-default-service: &default-service
   cap_drop: [ALL]
   security_opt: [no-new-privileges]
   pull_policy: weekly
+  logging:
+    driver: syslog
+    options:
+      tag: "{{.Name}}"
+      syslog-facility: "local0"
 
 services:
   vaultwarden:
@@ -69,6 +74,7 @@ services:
       - freshrss
       - authelia
       - wg-easy
+      - grafana
     env_file:
       - gatus.env
     volumes:
@@ -93,6 +99,7 @@ services:
       - freshrss
       - authelia
       - wg-easy
+      - grafana
     volumes:
       - "/mnt/oraclevdb/cloudflared:/user/nonroot/.cloudflared/"
     command: tunnel run
@@ -218,6 +225,64 @@ services:
         uid: "999"
         gid: "999"
 
+  grafana:
+    <<: *default-service
+    container_name: grafana
+    image: grafana/grafana:latest
+    env_file: ./grafana.env
+    user: "472:0" # user grafana, group root
+    volumes:
+      - /mnt/oraclevdb/grafana/:/var/lib/grafana/
+    configs:
+      - source: grafana-ini
+        target: /etc/grafana/grafana.ini
+        uid: "472"
+        gid: "0"
+      - source: grafana-datasources
+        target: /etc/grafana/provisioning/datasources/datasources.yaml
+        uid: "472"
+        gid: "0"
+      - source: grafana-dashboards
+        target: /etc/grafana/provisioning/dashboards/dashboards.yaml
+        uid: "472"
+        gid: "0"
+    secrets:
+      - grafana-database-url
+
+  loki:
+    <<: *default-service
+    container_name: loki
+    image: grafana/loki:latest
+    user: 10001:10001 # loki user
+    volumes:
+      - /mnt/oraclevdb/loki/:/loki/
+    configs:
+      - source: loki-config
+        target: /etc/loki/local-config.yaml
+        uid: "10001"
+        gid: "10001"
+
+  alloy:
+    <<: *default-service
+    container_name: alloy
+    image: grafana/alloy:latest
+    user: 473:473 # alloy user
+    ports:
+      - 514:514/tcp
+    # https://github.com/grafana/alloy/blob/a473efe2b676ba826ec1ba68897b6b1f475bcbbc/Dockerfile#L82
+    command: >
+      run
+      /etc/alloy/config.alloy
+      --server.http.listen-addr=0.0.0.0:12345
+      --storage.path=/var/lib/alloy/data
+    volumes:
+      - /mnt/oraclevdb/alloy/:/var/lib/alloy/data
+    configs:
+      - source: alloy-config
+        target: /etc/alloy/config.alloy
+        uid: "473"
+        gid: "473"
+
 configs:
   rclone-config:
     file: ./rclone.conf
@@ -240,6 +305,16 @@ configs:
     file: ./authelia.configuration.yml
   redis-config:
     file: ./redis.conf
+  grafana-ini:
+    file: ./grafana.ini
+  grafana-datasources:
+    file: ./datasources.yaml
+  grafana-dashboards:
+    file: ./dashboards.yaml
+  loki-config:
+    file: ./loki-config.yaml
+  alloy-config:
+    file: ./config.alloy
 
 secrets:
   vaultwarden-database-url:
@@ -262,3 +337,5 @@ secrets:
     file: "./authelia-redis-password"
   redis-users-acl:
     file: "./redis.users.acl"
+  grafana-database-url:
+    file: "./grafana-database-url"

files/config.alloy

@@ -0,0 +1,36 @@
+logging {
+  format = "logfmt"
+  level  = "info"
+}
+
+
+loki.relabel "syslog" {
+  forward_to = []
+
+  rule {
+    action = "labelmap"
+    regex = "__syslog_(.+)"
+  }
+
+  rule {
+    action = "labelmap"
+    regex  = "message_(.+)"
+  }
+}
+
+loki.source.syslog "syslog_receiver" {
+  listener {
+    address  = "0.0.0.0:514"
+    protocol = "tcp"
+    labels   = { job = "syslog" }
+  }
+  relabel_rules = loki.relabel.syslog.rules
+
+  forward_to = [loki.write.default.receiver]
+}
+
+loki.write "default" {
+	endpoint {
+		url = "http://loki:3100/loki/api/v1/push"
+	}
+}

files/gatus-config.yaml

@@ -14,6 +14,9 @@ security:
 web:
   port: 8080
 
+ui:
+  default-sort-by: group
+
 alerting:
   email:
     enabled: true
@@ -76,8 +79,8 @@ endpoints:
     <<: *status-200
   - name: FreshRSS
     group: core
-    url: "https://rss.${SERVER_DOMAIN}"
-    <<: *status-401
+    url: "https://rss.${SERVER_DOMAIN}/api"
+    <<: *status-200
   - name: Authelia
     group: core
     url: "https://auth.${SERVER_DOMAIN}/api/health"
@@ -90,6 +93,14 @@ endpoints:
     group: core
     url: "https://wireguard.${SERVER_DOMAIN}"
     <<: *status-200
+  - name: Grafana
+    group: core
+    url: "https://logs.${SERVER_DOMAIN}/api/health"
+    <<: *status-200
+    conditions: &grafana-conditions
+      - "[STATUS] == 200"
+      - "[RESPONSE_TIME] < 1000"
+      - "[BODY].database == ok"
   # Internal
   - name: Vaultwarden (Internal)
     group: internal
@@ -106,8 +117,8 @@ endpoints:
     <<: *status-200
   - name: FreshRSS (Internal)
     group: internal
-    url: "http://freshrss:80"
-    <<: *status-401
+    url: "http://freshrss:80/api"
+    <<: *status-200
   - name: Authelia (Internal)
     group: internal
     url: "http://authelia:9091/api/health"
@@ -117,6 +128,11 @@ endpoints:
     group: internal
     url: "http://wg-easy:51821"
     <<: *status-200
+  - name: Grafana (Internal)
+    group: internal
+    url: "http://grafana:3000/api/health"
+    <<: *status-200
+    conditions: *grafana-conditions
   - name: Folding@home
     group: internal
     url: "http://foldingathome:7396"

files/grafana/dashboards.yaml

@@ -0,0 +1,7 @@
+apiVersion: 1
+
+providers:
+  - name: "Redis"
+    type: file
+    options:
+      path: /var/lib/grafana/plugins/redis-datasource/dashboards

files/grafana/datasources.yaml

@@ -0,0 +1,38 @@
+apiVersion: 1
+
+prune: true
+
+datasources:
+  - &mysql-datasource
+    name: MySQL (Grafana)
+    type: mysql
+    url: $_GRAFANA_MYSQL_URL
+    user: $_GRAFANA_MYSQL_USERNAME
+    jsonData:
+      database: grafana
+    secureJsonData:
+      password: $_GRAFANA_MYSQL_PASSWORD
+  - <<: *mysql-datasource
+    name: MySQL (Authelia)
+    jsonData:
+      database: authelia
+  - <<: *mysql-datasource
+    name: MySQL (FreshRSS)
+    jsonData:
+      database: freshrss
+  - name: Redis
+    type: redis-datasource
+    url: redis://redis:6379
+    jsonData:
+      client: standalone
+      acl: true
+      user: $_GRAFANA_REDIS_USERNAME
+    secureJsonData:
+      password: $_GRAFANA_REDIS_PASSWORD
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    jsonData:
+      timeout: 60
+      maxLines: 1000

files/grafana/grafana.ini

@@ -0,0 +1,72 @@
+########################## Grafana Configuration #########################
+
+#################################### Server ##############################
+[server]
+protocol = http
+http_addr = 0.0.0.0
+http_port = 3000
+domain = ${HOSTNAME}
+enforce_domain = false
+root_url = https://logs.%(domain)s/
+serve_from_sub_path = false
+router_logging = false
+static_root_path = public
+enable_gzip = false
+
+#################################### Database ############################
+[database]
+url =  $__file{/run/secrets/grafana-database-url}
+conn_max_lifetime = 14400 # 4 hours (14400 seconds)
+
+#################################### Security ############################
+[security]
+secret_key = ${SECRET_KEY_SECRET}
+disable_initial_admin_creation = true
+brute_force_login_protection_max_attempts = 3
+cookie_secure = true
+
+#################################### Users ###############################
+[users]
+allow_sign_up=false
+allow_org_create=false
+default_theme=system
+
+[auth]
+disable_login_form=true
+
+#################################### Generic OAuth #######################
+[auth.generic_oauth]
+enabled = true
+name = Authelia
+icon = signin
+client_id = ${CLIENT_ID}
+client_secret = ${CLIENT_SECRET}
+auth_url = https://auth.${HOSTNAME}/api/oidc/authorization
+token_url = https://auth.${HOSTNAME}/api/oidc/token
+api_url = https://auth.${HOSTNAME}/api/oidc/userinfo
+auth_style = InHeader
+scopes = openid profile email groups
+empty_scopes = false
+login_attribute_path = preferred_username
+name_attribute_path = name
+role_attribute_path = contains(groups[*], 'super_admin') && 'GrafanaAdmin' || contains(groups[*], 'admin') && 'Admin' || contains(groups[*], 'editor') && 'Editor' || 'Viewer'
+allow_assign_grafana_admin = true
+groups_attribute_path = groups
+use_pkce = true
+
+#################################### SMTP / Emailing #####################
+[smtp]
+enabled=true
+host=${SMTP_HOST}
+user=${SMTP_USERNAME}
+password=${SMTP_PASSWORD}
+from_address=${SMTP_EMAIL_FROM}
+from_name=grafana
+
+#################################### Unified Alerting ####################
+[unified_alerting]
+enabled = true
+
+[date_formats]
+use_browser_locale = true
+default_timezone = browser