diff --git a/README.md b/README.md index 6bfc4a0..679168c 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ Webhook metrics: | `workflow_execution_time_seconds` | histogram | `org`, `repo`, `branch`, `workflow_name`, `conclusion` | Time that a workflow took to run. | | `workflow_status_count` | counter | `org`, `repo`, `branch`, `status`, `conclusion`, `workflow_name` | Count of the occurrences of different workflow states. | +Each branch adds its own series to these metrics, and the exporter keeps them until it restarts. With many short-lived branches (pull requests, merge queue refs) this can add up to a lot of series. Use `--metrics.omit-branch-label` to drop the `branch` label from all five metrics, so every branch of a workflow counts towards the same series. + ### Example queries Time a job waits for a runner (queued until started) per runner group, useful to compare how fast runner groups pick up jobs: @@ -71,6 +73,7 @@ histogram_quantile(0.95, sum by (le, runner_group) (rate(workflow_job_duration_s | `--gh.github-org` | `GITHUB_ORG` | | GitHub organization to collect the billing metrics for. | | `--gh.github-user` | | | GitHub user to collect the billing metrics for. | | `--gh.billing-poll-seconds` | `BILLING_POLL_SECONDS` | `5` | Frequency at which to poll the billing API. | +| `--metrics.omit-branch-label` | `GITHUB_METRICS_OMIT_BRANCH` | `false` | Omit the `branch` label from the webhook metrics. | | `--log.level` | | `info` | Only log messages with the given severity or above (`debug`, `info`, `warn`, `error`). | | `--log.format` | | `logfmt` | Output format of log messages (`logfmt`, `json`). | diff --git a/internal/server/metrics.go b/internal/server/metrics.go index ef7f1fa..0edbf76 100644 --- a/internal/server/metrics.go +++ b/internal/server/metrics.go @@ -1,45 +1,12 @@ package server -import "github.com/prometheus/client_golang/prometheus" +import ( + "errors" -var ( - workflowJobHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: "workflow_job_duration_seconds", - Help: "Time that a workflow job took to reach a given state.", - Buckets: prometheus.ExponentialBuckets(1, 1.4, 30), - }, - []string{"org", "repo", "branch", "state", "runner_group", "workflow_name", "job_name"}, - ) - - workflowJobDurationCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "workflow_job_duration_seconds_total", - Help: "The total duration of jobs.", - }, - []string{"org", "repo", "branch", "status", "conclusion", "runner_group", "workflow_name", "job_name"}, - ) - - workflowJobStatusCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "workflow_job_status_count", - Help: "Count of workflow job events.", - }, - []string{"org", "repo", "branch", "status", "conclusion", "runner_group", "workflow_name", "job_name"}, - ) - - workflowRunHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: "workflow_execution_time_seconds", - Help: "Time that a workflow took to run.", - Buckets: prometheus.ExponentialBuckets(1, 1.4, 30), - }, - []string{"org", "repo", "branch", "workflow_name", "conclusion"}, - ) - - workflowRunStatusCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "workflow_status_count", - Help: "Count of the occurrences of different workflow states.", - }, - []string{"org", "repo", "branch", "status", "conclusion", "workflow_name"}, - ) + "github.com/prometheus/client_golang/prometheus" +) +var ( totalMinutesUsedActions = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "actions_total_minutes_used_minutes", Help: "Total minutes used for the GitHub Actions.", @@ -71,11 +38,6 @@ var ( func init() { // Register metrics with prometheus - prometheus.MustRegister(workflowJobHistogramVec) - prometheus.MustRegister(workflowJobStatusCounter) - prometheus.MustRegister(workflowJobDurationCounter) - prometheus.MustRegister(workflowRunHistogramVec) - prometheus.MustRegister(workflowRunStatusCounter) prometheus.MustRegister(totalMinutesUsedActions) prometheus.MustRegister(includedMinutesUsedActions) prometheus.MustRegister(totalPaidMinutesActions) @@ -93,26 +55,110 @@ type WorkflowObserver interface { var _ WorkflowObserver = (*PrometheusObserver)(nil) -type PrometheusObserver struct{} +// PrometheusObserver records the workflow metrics in Prometheus. +type PrometheusObserver struct { + omitBranch bool + + workflowJobHistogramVec *prometheus.HistogramVec + workflowJobDurationCounter *prometheus.CounterVec + workflowJobStatusCounter *prometheus.CounterVec + workflowRunHistogramVec *prometheus.HistogramVec + workflowRunStatusCounter *prometheus.CounterVec +} + +// NewPrometheusObserver registers the workflow metrics with reg. When omitBranch is true, +// the metrics have no branch label, so all branches of a workflow share the same series. +// Registering a second observer with the same labels reuses the metrics of the first. +func NewPrometheusObserver(reg prometheus.Registerer, omitBranch bool) *PrometheusObserver { + o := &PrometheusObserver{omitBranch: omitBranch} + + o.workflowJobHistogramVec = registerOrExisting(reg, prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "workflow_job_duration_seconds", + Help: "Time that a workflow job took to reach a given state.", + Buckets: prometheus.ExponentialBuckets(1, 1.4, 30), + }, + o.labelNames("state", "runner_group", "workflow_name", "job_name"), + )) + + o.workflowJobDurationCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "workflow_job_duration_seconds_total", + Help: "The total duration of jobs.", + }, + o.labelNames("status", "conclusion", "runner_group", "workflow_name", "job_name"), + )) + + o.workflowJobStatusCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "workflow_job_status_count", + Help: "Count of workflow job events.", + }, + o.labelNames("status", "conclusion", "runner_group", "workflow_name", "job_name"), + )) + + o.workflowRunHistogramVec = registerOrExisting(reg, prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "workflow_execution_time_seconds", + Help: "Time that a workflow took to run.", + Buckets: prometheus.ExponentialBuckets(1, 1.4, 30), + }, + o.labelNames("workflow_name", "conclusion"), + )) + + o.workflowRunStatusCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "workflow_status_count", + Help: "Count of the occurrences of different workflow states.", + }, + o.labelNames("status", "conclusion", "workflow_name"), + )) + + return o +} + +func registerOrExisting[T prometheus.Collector](reg prometheus.Registerer, c T) T { + if err := reg.Register(c); err != nil { + var are prometheus.AlreadyRegisteredError + if errors.As(err, &are) { + if existing, ok := are.ExistingCollector.(T); ok { + return existing + } + } + panic(err) + } + return c +} + +// labelNames returns the org, repo and (unless omitted) branch labels followed by names. +func (o *PrometheusObserver) labelNames(names ...string) []string { + if o.omitBranch { + return append([]string{"org", "repo"}, names...) + } + return append([]string{"org", "repo", "branch"}, names...) +} + +// labelValues returns the values matching labelNames. +func (o *PrometheusObserver) labelValues(org, repo, branch string, values ...string) []string { + if o.omitBranch { + return append([]string{org, repo}, values...) + } + return append([]string{org, repo, branch}, values...) +} func (o *PrometheusObserver) ObserveWorkflowJobDuration(org, repo, branch, state, runnerGroup, workflowName, jobName string, seconds float64) { - workflowJobHistogramVec.WithLabelValues(org, repo, branch, state, runnerGroup, workflowName, jobName). + o.workflowJobHistogramVec.WithLabelValues(o.labelValues(org, repo, branch, state, runnerGroup, workflowName, jobName)...). Observe(seconds) } func (o *PrometheusObserver) CountWorkflowJobStatus(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string) { - workflowJobStatusCounter.WithLabelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName).Inc() + o.workflowJobStatusCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName)...).Inc() } func (o *PrometheusObserver) CountWorkflowJobDuration(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string, seconds float64) { - workflowJobDurationCounter.WithLabelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName).Add(seconds) + o.workflowJobDurationCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName)...).Add(seconds) } func (o *PrometheusObserver) ObserveWorkflowRunDuration(org, repo, branch, workflowName, conclusion string, seconds float64) { - workflowRunHistogramVec.WithLabelValues(org, repo, branch, workflowName, conclusion). + o.workflowRunHistogramVec.WithLabelValues(o.labelValues(org, repo, branch, workflowName, conclusion)...). Observe(seconds) } func (o *PrometheusObserver) CountWorkflowRunStatus(org, repo, branch, status, conclusion, workflowName string) { - workflowRunStatusCounter.WithLabelValues(org, repo, branch, status, conclusion, workflowName).Inc() + o.workflowRunStatusCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, workflowName)...).Inc() } diff --git a/internal/server/server.go b/internal/server/server.go index 8d22a71..92f5802 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -26,6 +26,8 @@ type Opts struct { GitHubOrg string GitHubUser string BillingAPIPollSeconds int + // OmitBranchLabel registers the workflow metrics without the branch label. + OmitBranchLabel bool } type Server struct { diff --git a/internal/server/workflow_metrics_exporter.go b/internal/server/workflow_metrics_exporter.go index 63b3606..d7c2084 100644 --- a/internal/server/workflow_metrics_exporter.go +++ b/internal/server/workflow_metrics_exporter.go @@ -17,6 +17,7 @@ import ( "github.com/cpanato/github_actions_exporter/model" "github.com/google/go-github/v92/github" + "github.com/prometheus/client_golang/prometheus" ) // WorkflowMetricsExporter struct to hold some information @@ -31,7 +32,7 @@ func NewWorkflowMetricsExporter(logger *slog.Logger, opts Opts) *WorkflowMetrics return &WorkflowMetricsExporter{ Logger: logger, Opts: opts, - PrometheusObserver: &PrometheusObserver{}, + PrometheusObserver: NewPrometheusObserver(prometheus.DefaultRegisterer, opts.OmitBranchLabel), } } diff --git a/internal/server/workflow_metrics_exporter_test.go b/internal/server/workflow_metrics_exporter_test.go index e47eb96..61b0c4e 100644 --- a/internal/server/workflow_metrics_exporter_test.go +++ b/internal/server/workflow_metrics_exporter_test.go @@ -17,6 +17,7 @@ import ( "github.com/cpanato/github_actions_exporter/internal/server" "github.com/google/go-github/v92/github" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -748,6 +749,68 @@ type workflowRunStatusCount struct { org, repo, branch, status, conclusion, workflowName string } +func Test_PrometheusObserver_BranchLabel(t *testing.T) { + tests := []struct { + name string + omitBranch bool + wantSeries int + }{ + {name: "with branch label", omitBranch: false, wantSeries: 2}, + {name: "without branch label", omitBranch: true, wantSeries: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Given + reg := prometheus.NewRegistry() + observer := server.NewPrometheusObserver(reg, tt.omitBranch) + + // When + for _, branch := range []string{"main", "feature"} { + observer.ObserveWorkflowJobDuration("org", "repo", branch, "queued", "default", "CI", "test", 1) + observer.CountWorkflowJobStatus("org", "repo", branch, "completed", "success", "default", "CI", "test") + observer.CountWorkflowJobDuration("org", "repo", branch, "completed", "success", "default", "CI", "test", 1) + observer.ObserveWorkflowRunDuration("org", "repo", branch, "CI", "success", 1) + observer.CountWorkflowRunStatus("org", "repo", branch, "completed", "success", "CI") + } + + // Then + families, err := reg.Gather() + require.NoError(t, err) + require.Len(t, families, 5) + for _, family := range families { + require.Len(t, family.GetMetric(), tt.wantSeries, family.GetName()) + for _, metric := range family.GetMetric() { + hasBranch := false + for _, label := range metric.GetLabel() { + if label.GetName() == "branch" { + hasBranch = true + } + } + assert.Equal(t, !tt.omitBranch, hasBranch, family.GetName()) + } + } + }) + } +} + +func Test_NewPrometheusObserver_ReusesRegisteredMetrics(t *testing.T) { + // Given + reg := prometheus.NewRegistry() + first := server.NewPrometheusObserver(reg, false) + + // When + second := server.NewPrometheusObserver(reg, false) + first.CountWorkflowRunStatus("org", "repo", "main", "completed", "success", "CI") + second.CountWorkflowRunStatus("org", "repo", "main", "completed", "success", "CI") + + // Then + families, err := reg.Gather() + require.NoError(t, err) + require.Len(t, families, 1) + assert.InDelta(t, 2, families[0].GetMetric()[0].GetCounter().GetValue(), 1e-9) +} + var _ server.WorkflowObserver = (*TestPrometheusObserver)(nil) type TestPrometheusObserver struct { diff --git a/main.go b/main.go index dc195a5..5836e6d 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ var ( gitHubOrg = kingpin.Flag("gh.github-org", "GitHub Organization.").Envar("GITHUB_ORG").Default("").String() gitHubUser = kingpin.Flag("gh.github-user", "GitHub User.").Default("").String() gitHubBillingPollingSeconds = kingpin.Flag("gh.billing-poll-seconds", "Frequency at which to poll billing API.").Envar("BILLING_POLL_SECONDS").Default("5").Int() + omitBranchLabel = kingpin.Flag("metrics.omit-branch-label", "Omit the branch label from the workflow metrics, so all branches of a workflow share the same series.").Envar("GITHUB_METRICS_OMIT_BRANCH").Default("false").Bool() ) func init() { @@ -61,6 +62,7 @@ func main() { GitHubUser: *gitHubUser, GitHubOrg: *gitHubOrg, BillingAPIPollSeconds: *gitHubBillingPollingSeconds, + OmitBranchLabel: *omitBranchLabel, }) go func() { err := srv.Serve(context.Background())