From 56cda995ad6e00906bc10a23e70eaa02e3551685 Mon Sep 17 00:00:00 2001 From: Daniel Heimburg <5831617+danielheimburg@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:16:15 +0100 Subject: [PATCH] Add an option to omit the branch label from the workflow metrics Every branch adds its own series to the workflow metrics and they are never removed, so pull request and merge queue branches grow the series count without bound. --metrics.omit-branch-label (GITHUB_METRICS_OMIT_BRANCH) registers the five webhook metrics without the branch label. It is off by default. Signed-off-by: Daniel Heimburg <5831617+danielheimburg@users.noreply.github.com> --- README.md | 3 + internal/server/metrics.go | 144 ++++++++++++------ internal/server/server.go | 2 + internal/server/workflow_metrics_exporter.go | 3 +- .../server/workflow_metrics_exporter_test.go | 63 ++++++++ main.go | 2 + 6 files changed, 167 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 6bfc4a01..679168c8 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ Webhook metrics: | `workflow_execution_time_seconds` | histogram | `org`, `repo`, `branch`, `workflow_name`, `conclusion` | Time that a workflow took to run. | | `workflow_status_count` | counter | `org`, `repo`, `branch`, `status`, `conclusion`, `workflow_name` | Count of the occurrences of different workflow states. | +Each branch adds its own series to these metrics, and the exporter keeps them until it restarts. With many short-lived branches (pull requests, merge queue refs) this can add up to a lot of series. Use `--metrics.omit-branch-label` to drop the `branch` label from all five metrics, so every branch of a workflow counts towards the same series. + ### Example queries Time a job waits for a runner (queued until started) per runner group, useful to compare how fast runner groups pick up jobs: @@ -71,6 +73,7 @@ histogram_quantile(0.95, sum by (le, runner_group) (rate(workflow_job_duration_s | `--gh.github-org` | `GITHUB_ORG` | | GitHub organization to collect the billing metrics for. | | `--gh.github-user` | | | GitHub user to collect the billing metrics for. | | `--gh.billing-poll-seconds` | `BILLING_POLL_SECONDS` | `5` | Frequency at which to poll the billing API. | +| `--metrics.omit-branch-label` | `GITHUB_METRICS_OMIT_BRANCH` | `false` | Omit the `branch` label from the webhook metrics. | | `--log.level` | | `info` | Only log messages with the given severity or above (`debug`, `info`, `warn`, `error`). | | `--log.format` | | `logfmt` | Output format of log messages (`logfmt`, `json`). | diff --git a/internal/server/metrics.go b/internal/server/metrics.go index ef7f1fad..0edbf760 100644 --- a/internal/server/metrics.go +++ b/internal/server/metrics.go @@ -1,45 +1,12 @@ package server -import "github.com/prometheus/client_golang/prometheus" +import ( + "errors" -var ( - workflowJobHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: "workflow_job_duration_seconds", - Help: "Time that a workflow job took to reach a given state.", - Buckets: prometheus.ExponentialBuckets(1, 1.4, 30), - }, - []string{"org", "repo", "branch", "state", "runner_group", "workflow_name", "job_name"}, - ) - - workflowJobDurationCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "workflow_job_duration_seconds_total", - Help: "The total duration of jobs.", - }, - []string{"org", "repo", "branch", "status", "conclusion", "runner_group", "workflow_name", "job_name"}, - ) - - workflowJobStatusCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "workflow_job_status_count", - Help: "Count of workflow job events.", - }, - []string{"org", "repo", "branch", "status", "conclusion", "runner_group", "workflow_name", "job_name"}, - ) - - workflowRunHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: "workflow_execution_time_seconds", - Help: "Time that a workflow took to run.", - Buckets: prometheus.ExponentialBuckets(1, 1.4, 30), - }, - []string{"org", "repo", "branch", "workflow_name", "conclusion"}, - ) - - workflowRunStatusCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "workflow_status_count", - Help: "Count of the occurrences of different workflow states.", - }, - []string{"org", "repo", "branch", "status", "conclusion", "workflow_name"}, - ) + "github.com/prometheus/client_golang/prometheus" +) +var ( totalMinutesUsedActions = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "actions_total_minutes_used_minutes", Help: "Total minutes used for the GitHub Actions.", @@ -71,11 +38,6 @@ var ( func init() { // Register metrics with prometheus - prometheus.MustRegister(workflowJobHistogramVec) - prometheus.MustRegister(workflowJobStatusCounter) - prometheus.MustRegister(workflowJobDurationCounter) - prometheus.MustRegister(workflowRunHistogramVec) - prometheus.MustRegister(workflowRunStatusCounter) prometheus.MustRegister(totalMinutesUsedActions) prometheus.MustRegister(includedMinutesUsedActions) prometheus.MustRegister(totalPaidMinutesActions) @@ -93,26 +55,110 @@ type WorkflowObserver interface { var _ WorkflowObserver = (*PrometheusObserver)(nil) -type PrometheusObserver struct{} +// PrometheusObserver records the workflow metrics in Prometheus. +type PrometheusObserver struct { + omitBranch bool + + workflowJobHistogramVec *prometheus.HistogramVec + workflowJobDurationCounter *prometheus.CounterVec + workflowJobStatusCounter *prometheus.CounterVec + workflowRunHistogramVec *prometheus.HistogramVec + workflowRunStatusCounter *prometheus.CounterVec +} + +// NewPrometheusObserver registers the workflow metrics with reg. When omitBranch is true, +// the metrics have no branch label, so all branches of a workflow share the same series. +// Registering a second observer with the same labels reuses the metrics of the first. +func NewPrometheusObserver(reg prometheus.Registerer, omitBranch bool) *PrometheusObserver { + o := &PrometheusObserver{omitBranch: omitBranch} + + o.workflowJobHistogramVec = registerOrExisting(reg, prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "workflow_job_duration_seconds", + Help: "Time that a workflow job took to reach a given state.", + Buckets: prometheus.ExponentialBuckets(1, 1.4, 30), + }, + o.labelNames("state", "runner_group", "workflow_name", "job_name"), + )) + + o.workflowJobDurationCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "workflow_job_duration_seconds_total", + Help: "The total duration of jobs.", + }, + o.labelNames("status", "conclusion", "runner_group", "workflow_name", "job_name"), + )) + + o.workflowJobStatusCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "workflow_job_status_count", + Help: "Count of workflow job events.", + }, + o.labelNames("status", "conclusion", "runner_group", "workflow_name", "job_name"), + )) + + o.workflowRunHistogramVec = registerOrExisting(reg, prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "workflow_execution_time_seconds", + Help: "Time that a workflow took to run.", + Buckets: prometheus.ExponentialBuckets(1, 1.4, 30), + }, + o.labelNames("workflow_name", "conclusion"), + )) + + o.workflowRunStatusCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "workflow_status_count", + Help: "Count of the occurrences of different workflow states.", + }, + o.labelNames("status", "conclusion", "workflow_name"), + )) + + return o +} + +func registerOrExisting[T prometheus.Collector](reg prometheus.Registerer, c T) T { + if err := reg.Register(c); err != nil { + var are prometheus.AlreadyRegisteredError + if errors.As(err, &are) { + if existing, ok := are.ExistingCollector.(T); ok { + return existing + } + } + panic(err) + } + return c +} + +// labelNames returns the org, repo and (unless omitted) branch labels followed by names. +func (o *PrometheusObserver) labelNames(names ...string) []string { + if o.omitBranch { + return append([]string{"org", "repo"}, names...) + } + return append([]string{"org", "repo", "branch"}, names...) +} + +// labelValues returns the values matching labelNames. +func (o *PrometheusObserver) labelValues(org, repo, branch string, values ...string) []string { + if o.omitBranch { + return append([]string{org, repo}, values...) + } + return append([]string{org, repo, branch}, values...) +} func (o *PrometheusObserver) ObserveWorkflowJobDuration(org, repo, branch, state, runnerGroup, workflowName, jobName string, seconds float64) { - workflowJobHistogramVec.WithLabelValues(org, repo, branch, state, runnerGroup, workflowName, jobName). + o.workflowJobHistogramVec.WithLabelValues(o.labelValues(org, repo, branch, state, runnerGroup, workflowName, jobName)...). Observe(seconds) } func (o *PrometheusObserver) CountWorkflowJobStatus(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string) { - workflowJobStatusCounter.WithLabelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName).Inc() + o.workflowJobStatusCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName)...).Inc() } func (o *PrometheusObserver) CountWorkflowJobDuration(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string, seconds float64) { - workflowJobDurationCounter.WithLabelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName).Add(seconds) + o.workflowJobDurationCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName)...).Add(seconds) } func (o *PrometheusObserver) ObserveWorkflowRunDuration(org, repo, branch, workflowName, conclusion string, seconds float64) { - workflowRunHistogramVec.WithLabelValues(org, repo, branch, workflowName, conclusion). + o.workflowRunHistogramVec.WithLabelValues(o.labelValues(org, repo, branch, workflowName, conclusion)...). Observe(seconds) } func (o *PrometheusObserver) CountWorkflowRunStatus(org, repo, branch, status, conclusion, workflowName string) { - workflowRunStatusCounter.WithLabelValues(org, repo, branch, status, conclusion, workflowName).Inc() + o.workflowRunStatusCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, workflowName)...).Inc() } diff --git a/internal/server/server.go b/internal/server/server.go index 8d22a717..92f5802d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -26,6 +26,8 @@ type Opts struct { GitHubOrg string GitHubUser string BillingAPIPollSeconds int + // OmitBranchLabel registers the workflow metrics without the branch label. + OmitBranchLabel bool } type Server struct { diff --git a/internal/server/workflow_metrics_exporter.go b/internal/server/workflow_metrics_exporter.go index 63b3606f..d7c20849 100644 --- a/internal/server/workflow_metrics_exporter.go +++ b/internal/server/workflow_metrics_exporter.go @@ -17,6 +17,7 @@ import ( "github.com/cpanato/github_actions_exporter/model" "github.com/google/go-github/v92/github" + "github.com/prometheus/client_golang/prometheus" ) // WorkflowMetricsExporter struct to hold some information @@ -31,7 +32,7 @@ func NewWorkflowMetricsExporter(logger *slog.Logger, opts Opts) *WorkflowMetrics return &WorkflowMetricsExporter{ Logger: logger, Opts: opts, - PrometheusObserver: &PrometheusObserver{}, + PrometheusObserver: NewPrometheusObserver(prometheus.DefaultRegisterer, opts.OmitBranchLabel), } } diff --git a/internal/server/workflow_metrics_exporter_test.go b/internal/server/workflow_metrics_exporter_test.go index e47eb969..61b0c4e4 100644 --- a/internal/server/workflow_metrics_exporter_test.go +++ b/internal/server/workflow_metrics_exporter_test.go @@ -17,6 +17,7 @@ import ( "github.com/cpanato/github_actions_exporter/internal/server" "github.com/google/go-github/v92/github" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -748,6 +749,68 @@ type workflowRunStatusCount struct { org, repo, branch, status, conclusion, workflowName string } +func Test_PrometheusObserver_BranchLabel(t *testing.T) { + tests := []struct { + name string + omitBranch bool + wantSeries int + }{ + {name: "with branch label", omitBranch: false, wantSeries: 2}, + {name: "without branch label", omitBranch: true, wantSeries: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Given + reg := prometheus.NewRegistry() + observer := server.NewPrometheusObserver(reg, tt.omitBranch) + + // When + for _, branch := range []string{"main", "feature"} { + observer.ObserveWorkflowJobDuration("org", "repo", branch, "queued", "default", "CI", "test", 1) + observer.CountWorkflowJobStatus("org", "repo", branch, "completed", "success", "default", "CI", "test") + observer.CountWorkflowJobDuration("org", "repo", branch, "completed", "success", "default", "CI", "test", 1) + observer.ObserveWorkflowRunDuration("org", "repo", branch, "CI", "success", 1) + observer.CountWorkflowRunStatus("org", "repo", branch, "completed", "success", "CI") + } + + // Then + families, err := reg.Gather() + require.NoError(t, err) + require.Len(t, families, 5) + for _, family := range families { + require.Len(t, family.GetMetric(), tt.wantSeries, family.GetName()) + for _, metric := range family.GetMetric() { + hasBranch := false + for _, label := range metric.GetLabel() { + if label.GetName() == "branch" { + hasBranch = true + } + } + assert.Equal(t, !tt.omitBranch, hasBranch, family.GetName()) + } + } + }) + } +} + +func Test_NewPrometheusObserver_ReusesRegisteredMetrics(t *testing.T) { + // Given + reg := prometheus.NewRegistry() + first := server.NewPrometheusObserver(reg, false) + + // When + second := server.NewPrometheusObserver(reg, false) + first.CountWorkflowRunStatus("org", "repo", "main", "completed", "success", "CI") + second.CountWorkflowRunStatus("org", "repo", "main", "completed", "success", "CI") + + // Then + families, err := reg.Gather() + require.NoError(t, err) + require.Len(t, families, 1) + assert.InDelta(t, 2, families[0].GetMetric()[0].GetCounter().GetValue(), 1e-9) +} + var _ server.WorkflowObserver = (*TestPrometheusObserver)(nil) type TestPrometheusObserver struct { diff --git a/main.go b/main.go index dc195a56..5836e6d7 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ var ( gitHubOrg = kingpin.Flag("gh.github-org", "GitHub Organization.").Envar("GITHUB_ORG").Default("").String() gitHubUser = kingpin.Flag("gh.github-user", "GitHub User.").Default("").String() gitHubBillingPollingSeconds = kingpin.Flag("gh.billing-poll-seconds", "Frequency at which to poll billing API.").Envar("BILLING_POLL_SECONDS").Default("5").Int() + omitBranchLabel = kingpin.Flag("metrics.omit-branch-label", "Omit the branch label from the workflow metrics, so all branches of a workflow share the same series.").Envar("GITHUB_METRICS_OMIT_BRANCH").Default("false").Bool() ) func init() { @@ -61,6 +62,7 @@ func main() { GitHubUser: *gitHubUser, GitHubOrg: *gitHubOrg, BillingAPIPollSeconds: *gitHubBillingPollingSeconds, + OmitBranchLabel: *omitBranchLabel, }) go func() { err := srv.Serve(context.Background())