Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ Webhook metrics:
| `workflow_execution_time_seconds` | histogram | `org`, `repo`, `branch`, `workflow_name`, `conclusion` | Time that a workflow took to run. |
| `workflow_status_count` | counter | `org`, `repo`, `branch`, `status`, `conclusion`, `workflow_name` | Count of the occurrences of different workflow states. |

Each branch adds its own series to these metrics, and the exporter keeps them until it restarts. With many short-lived branches (pull requests, merge queue refs) this can add up to a lot of series. Use `--metrics.omit-branch-label` to drop the `branch` label from all five metrics, so every branch of a workflow counts towards the same series.

### Example queries

Time a job waits for a runner (queued until started) per runner group, useful to compare how fast runner groups pick up jobs:
Expand Down Expand Up @@ -71,6 +73,7 @@ histogram_quantile(0.95, sum by (le, runner_group) (rate(workflow_job_duration_s
| `--gh.github-org` | `GITHUB_ORG` | | GitHub organization to collect the billing metrics for. |
| `--gh.github-user` | | | GitHub user to collect the billing metrics for. |
| `--gh.billing-poll-seconds` | `BILLING_POLL_SECONDS` | `5` | Frequency at which to poll the billing API. |
| `--metrics.omit-branch-label` | `GITHUB_METRICS_OMIT_BRANCH` | `false` | Omit the `branch` label from the webhook metrics. |
| `--log.level` | | `info` | Only log messages with the given severity or above (`debug`, `info`, `warn`, `error`). |
| `--log.format` | | `logfmt` | Output format of log messages (`logfmt`, `json`). |

Expand Down
144 changes: 95 additions & 49 deletions internal/server/metrics.go
Original file line number Diff line number Diff line change
@@ -1,45 +1,12 @@
package server

import "github.com/prometheus/client_golang/prometheus"
import (
"errors"

var (
workflowJobHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "workflow_job_duration_seconds",
Help: "Time that a workflow job took to reach a given state.",
Buckets: prometheus.ExponentialBuckets(1, 1.4, 30),
},
[]string{"org", "repo", "branch", "state", "runner_group", "workflow_name", "job_name"},
)

workflowJobDurationCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_job_duration_seconds_total",
Help: "The total duration of jobs.",
},
[]string{"org", "repo", "branch", "status", "conclusion", "runner_group", "workflow_name", "job_name"},
)

workflowJobStatusCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_job_status_count",
Help: "Count of workflow job events.",
},
[]string{"org", "repo", "branch", "status", "conclusion", "runner_group", "workflow_name", "job_name"},
)

workflowRunHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "workflow_execution_time_seconds",
Help: "Time that a workflow took to run.",
Buckets: prometheus.ExponentialBuckets(1, 1.4, 30),
},
[]string{"org", "repo", "branch", "workflow_name", "conclusion"},
)

workflowRunStatusCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_status_count",
Help: "Count of the occurrences of different workflow states.",
},
[]string{"org", "repo", "branch", "status", "conclusion", "workflow_name"},
)
"github.com/prometheus/client_golang/prometheus"
)

var (
totalMinutesUsedActions = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "actions_total_minutes_used_minutes",
Help: "Total minutes used for the GitHub Actions.",
Expand Down Expand Up @@ -71,11 +38,6 @@ var (

func init() {
// Register metrics with prometheus
prometheus.MustRegister(workflowJobHistogramVec)
prometheus.MustRegister(workflowJobStatusCounter)
prometheus.MustRegister(workflowJobDurationCounter)
prometheus.MustRegister(workflowRunHistogramVec)
prometheus.MustRegister(workflowRunStatusCounter)
prometheus.MustRegister(totalMinutesUsedActions)
prometheus.MustRegister(includedMinutesUsedActions)
prometheus.MustRegister(totalPaidMinutesActions)
Expand All @@ -93,26 +55,110 @@ type WorkflowObserver interface {

var _ WorkflowObserver = (*PrometheusObserver)(nil)

type PrometheusObserver struct{}
// PrometheusObserver records the workflow metrics in Prometheus.
type PrometheusObserver struct {
omitBranch bool

workflowJobHistogramVec *prometheus.HistogramVec
workflowJobDurationCounter *prometheus.CounterVec
workflowJobStatusCounter *prometheus.CounterVec
workflowRunHistogramVec *prometheus.HistogramVec
workflowRunStatusCounter *prometheus.CounterVec
}

// NewPrometheusObserver registers the workflow metrics with reg. When omitBranch is true,
// the metrics have no branch label, so all branches of a workflow share the same series.
// Registering a second observer with the same labels reuses the metrics of the first.
func NewPrometheusObserver(reg prometheus.Registerer, omitBranch bool) *PrometheusObserver {
o := &PrometheusObserver{omitBranch: omitBranch}

o.workflowJobHistogramVec = registerOrExisting(reg, prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "workflow_job_duration_seconds",
Help: "Time that a workflow job took to reach a given state.",
Buckets: prometheus.ExponentialBuckets(1, 1.4, 30),
},
o.labelNames("state", "runner_group", "workflow_name", "job_name"),
))

o.workflowJobDurationCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_job_duration_seconds_total",
Help: "The total duration of jobs.",
},
o.labelNames("status", "conclusion", "runner_group", "workflow_name", "job_name"),
))

o.workflowJobStatusCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_job_status_count",
Help: "Count of workflow job events.",
},
o.labelNames("status", "conclusion", "runner_group", "workflow_name", "job_name"),
))

o.workflowRunHistogramVec = registerOrExisting(reg, prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "workflow_execution_time_seconds",
Help: "Time that a workflow took to run.",
Buckets: prometheus.ExponentialBuckets(1, 1.4, 30),
},
o.labelNames("workflow_name", "conclusion"),
))

o.workflowRunStatusCounter = registerOrExisting(reg, prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_status_count",
Help: "Count of the occurrences of different workflow states.",
},
o.labelNames("status", "conclusion", "workflow_name"),
))

return o
}

func registerOrExisting[T prometheus.Collector](reg prometheus.Registerer, c T) T {
if err := reg.Register(c); err != nil {
var are prometheus.AlreadyRegisteredError
if errors.As(err, &are) {
if existing, ok := are.ExistingCollector.(T); ok {
return existing
}
}
panic(err)
}
return c
}

// labelNames returns the org, repo and (unless omitted) branch labels followed by names.
func (o *PrometheusObserver) labelNames(names ...string) []string {
if o.omitBranch {
return append([]string{"org", "repo"}, names...)
}
return append([]string{"org", "repo", "branch"}, names...)
}

// labelValues returns the values matching labelNames.
func (o *PrometheusObserver) labelValues(org, repo, branch string, values ...string) []string {
if o.omitBranch {
return append([]string{org, repo}, values...)
}
return append([]string{org, repo, branch}, values...)
}

func (o *PrometheusObserver) ObserveWorkflowJobDuration(org, repo, branch, state, runnerGroup, workflowName, jobName string, seconds float64) {
workflowJobHistogramVec.WithLabelValues(org, repo, branch, state, runnerGroup, workflowName, jobName).
o.workflowJobHistogramVec.WithLabelValues(o.labelValues(org, repo, branch, state, runnerGroup, workflowName, jobName)...).
Observe(seconds)
}

func (o *PrometheusObserver) CountWorkflowJobStatus(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string) {
workflowJobStatusCounter.WithLabelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName).Inc()
o.workflowJobStatusCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName)...).Inc()
}

func (o *PrometheusObserver) CountWorkflowJobDuration(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string, seconds float64) {
workflowJobDurationCounter.WithLabelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName).Add(seconds)
o.workflowJobDurationCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName)...).Add(seconds)
}

func (o *PrometheusObserver) ObserveWorkflowRunDuration(org, repo, branch, workflowName, conclusion string, seconds float64) {
workflowRunHistogramVec.WithLabelValues(org, repo, branch, workflowName, conclusion).
o.workflowRunHistogramVec.WithLabelValues(o.labelValues(org, repo, branch, workflowName, conclusion)...).
Observe(seconds)
}

func (o *PrometheusObserver) CountWorkflowRunStatus(org, repo, branch, status, conclusion, workflowName string) {
workflowRunStatusCounter.WithLabelValues(org, repo, branch, status, conclusion, workflowName).Inc()
o.workflowRunStatusCounter.WithLabelValues(o.labelValues(org, repo, branch, status, conclusion, workflowName)...).Inc()
}
2 changes: 2 additions & 0 deletions internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ type Opts struct {
GitHubOrg string
GitHubUser string
BillingAPIPollSeconds int
// OmitBranchLabel registers the workflow metrics without the branch label.
OmitBranchLabel bool
}

type Server struct {
Expand Down
3 changes: 2 additions & 1 deletion internal/server/workflow_metrics_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

"github.com/cpanato/github_actions_exporter/model"
"github.com/google/go-github/v92/github"
"github.com/prometheus/client_golang/prometheus"
)

// WorkflowMetricsExporter struct to hold some information
Expand All @@ -31,7 +32,7 @@ func NewWorkflowMetricsExporter(logger *slog.Logger, opts Opts) *WorkflowMetrics
return &WorkflowMetricsExporter{
Logger: logger,
Opts: opts,
PrometheusObserver: &PrometheusObserver{},
PrometheusObserver: NewPrometheusObserver(prometheus.DefaultRegisterer, opts.OmitBranchLabel),
}
}

Expand Down
63 changes: 63 additions & 0 deletions internal/server/workflow_metrics_exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

"github.com/cpanato/github_actions_exporter/internal/server"
"github.com/google/go-github/v92/github"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -748,6 +749,68 @@ type workflowRunStatusCount struct {
org, repo, branch, status, conclusion, workflowName string
}

func Test_PrometheusObserver_BranchLabel(t *testing.T) {
tests := []struct {
name string
omitBranch bool
wantSeries int
}{
{name: "with branch label", omitBranch: false, wantSeries: 2},
{name: "without branch label", omitBranch: true, wantSeries: 1},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Given
reg := prometheus.NewRegistry()
observer := server.NewPrometheusObserver(reg, tt.omitBranch)

// When
for _, branch := range []string{"main", "feature"} {
observer.ObserveWorkflowJobDuration("org", "repo", branch, "queued", "default", "CI", "test", 1)
observer.CountWorkflowJobStatus("org", "repo", branch, "completed", "success", "default", "CI", "test")
observer.CountWorkflowJobDuration("org", "repo", branch, "completed", "success", "default", "CI", "test", 1)
observer.ObserveWorkflowRunDuration("org", "repo", branch, "CI", "success", 1)
observer.CountWorkflowRunStatus("org", "repo", branch, "completed", "success", "CI")
}

// Then
families, err := reg.Gather()
require.NoError(t, err)
require.Len(t, families, 5)
for _, family := range families {
require.Len(t, family.GetMetric(), tt.wantSeries, family.GetName())
for _, metric := range family.GetMetric() {
hasBranch := false
for _, label := range metric.GetLabel() {
if label.GetName() == "branch" {
hasBranch = true
}
}
assert.Equal(t, !tt.omitBranch, hasBranch, family.GetName())
}
}
})
}
}

func Test_NewPrometheusObserver_ReusesRegisteredMetrics(t *testing.T) {
// Given
reg := prometheus.NewRegistry()
first := server.NewPrometheusObserver(reg, false)

// When
second := server.NewPrometheusObserver(reg, false)
first.CountWorkflowRunStatus("org", "repo", "main", "completed", "success", "CI")
second.CountWorkflowRunStatus("org", "repo", "main", "completed", "success", "CI")

// Then
families, err := reg.Gather()
require.NoError(t, err)
require.Len(t, families, 1)
assert.InDelta(t, 2, families[0].GetMetric()[0].GetCounter().GetValue(), 1e-9)
}

var _ server.WorkflowObserver = (*TestPrometheusObserver)(nil)

type TestPrometheusObserver struct {
Expand Down
2 changes: 2 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ var (
gitHubOrg = kingpin.Flag("gh.github-org", "GitHub Organization.").Envar("GITHUB_ORG").Default("").String()
gitHubUser = kingpin.Flag("gh.github-user", "GitHub User.").Default("").String()
gitHubBillingPollingSeconds = kingpin.Flag("gh.billing-poll-seconds", "Frequency at which to poll billing API.").Envar("BILLING_POLL_SECONDS").Default("5").Int()
omitBranchLabel = kingpin.Flag("metrics.omit-branch-label", "Omit the branch label from the workflow metrics, so all branches of a workflow share the same series.").Envar("GITHUB_METRICS_OMIT_BRANCH").Default("false").Bool()
)

func init() {
Expand Down Expand Up @@ -61,6 +62,7 @@ func main() {
GitHubUser: *gitHubUser,
GitHubOrg: *gitHubOrg,
BillingAPIPollSeconds: *gitHubBillingPollingSeconds,
OmitBranchLabel: *omitBranchLabel,
})
go func() {
err := srv.Serve(context.Background())
Expand Down