Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
name: CI

on:
pull_request:
push:
branches: [main]

permissions:
contents: read

jobs:
check:
# aarch64-linux: what `cv` is deployed on, and built on, by a host whose own CI only evaluates.
runs-on: ubuntu-24.04-arm
timeout-minutes: 60
steps:
- uses: actions/checkout@v7
- uses: cachix/install-nix-action@v31
# Builds `cv`, running its tests, for `module-in-sync`. Without a store cache: add one if
# this takes much more than 20 minutes.
- run: nix flake check -L
2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,6 @@ opt-level = 3
opt-level = 3

[dev-dependencies]
# `test-util`: tests of timeouts pause the clock.
tokio = { version = "1.54.1", features = ["test-util"] }
tower = { version = "0.5.3", features = ["util"] }
24 changes: 24 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
# Golden files that are not Cargo sources.
./src/facts/c
./src/facts/python
./src/facts/rust
];
};
strictDeps = true;
Expand Down Expand Up @@ -119,6 +120,29 @@
diff -u <(${lib.getExe cv} --help | ${flags}) <(< ${./nix/module.nix} ${flags})
touch $out
'';

# The alert rules a host gets from the module, against `nix/alerts.test.yml`.
alerts =
let
host = lib.nixosSystem {
modules = [
self.nixosModules.default
{
nixpkgs.pkgs = pkgs;
services.cv.enable = true;
services.prometheus.enable = true;
}
];
};
rules = lib.head host.config.services.prometheus.ruleFiles;
in
pkgs.runCommand "alerts" { nativeBuildInputs = [ pkgs.prometheus.cli ]; } ''
ln -s ${rules} cv-alerts.json
cp ${./nix/alerts.test.yml} alerts.test.yml
promtool check rules cv-alerts.json
promtool test rules alerts.test.yml
touch $out
'';
}
);
};
Expand Down
47 changes: 47 additions & 0 deletions nix/alerts.test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Run by `nix flake check` against the rules in `module.nix`: what a process that has just started
# does not export. A sample a minute from 0; `_` is none.
rule_files: [cv-alerts.json]
evaluation_interval: 1m

tests:
- name: the first sync after a start, an hour in, hangs
input_series:
- series: cv_syncs_running
values: 0x59 1x300
alert_rule_test:
- alertname: CvSyncStuck
eval_time: 3h59m
- alertname: CvSyncStuck
eval_time: 4h
exp_alerts:
- exp_labels: { severity: warning }
exp_annotations:
summary: Syncs are running, but none has ended for 3 hours.

- name: syncs end while another runs
input_series:
- series: cv_syncs_running
values: 1x300
- series: cv_sync_duration_seconds_count{kind="sync", outcome="ok"}
values: 0+1x300
alert_rule_test:
- alertname: CvSyncStuck
eval_time: 4h

- name: one repository has only failed since the start, another synced
input_series:
- series: cv_repo_sync_failures_consecutive{repository="1"}
values: 1+0x400
- series: cv_repo_sync_failures_consecutive{repository="2"}
values: 0x400
- series: cv_repo_last_sync_success_timestamp_seconds{repository="2"}
values: 0+60x400
alert_rule_test:
- alertname: CvRepoStale
eval_time: 5h59m
- alertname: CvRepoStale
eval_time: 6h
exp_alerts:
- exp_labels: { severity: warning, repository: "1" }
exp_annotations:
summary: Repository 1 has not synced for over 6 hours.
10 changes: 8 additions & 2 deletions nix/module.nix
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,17 @@ let
(alert "CvRepoSyncFailing" "cv_repo_sync_failures_consecutive >= 3" "0m"
"Repository {{ $labels.repository }} failed its last {{ $value }} syncs; see the journal."
)
(alert "CvRepoStale" "time() - cv_repo_last_sync_success_timestamp_seconds > 6 * 3600" "0m"
# A process exports no success until a repository has one, and no duration until a sync
# ends: nothing to compute an age from. So these two rule out recent progress, and `for`
# counts the time.
(alert "CvRepoStale"
"cv_repo_sync_failures_consecutive unless time() - cv_repo_last_sync_success_timestamp_seconds < 900"
"6h"
"Repository {{ $labels.repository }} has not synced for over 6 hours."
)
(alert "CvSyncStuck"
"cv_syncs_running > 0 and on () sum(increase(cv_sync_duration_seconds_count[3h])) == 0" "0m"
"cv_syncs_running > 0 unless on () sum(increase(cv_sync_duration_seconds_count[15m])) > 0"
"3h"
"Syncs are running, but none has ended for 3 hours."
)
(alert "CvResyncFailing" "time() - cv_resync_last_success_timestamp_seconds > 3 * 3600" "0m"
Expand Down
4 changes: 2 additions & 2 deletions src/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ use sqlx::PgPool;
use tracing::{Instrument, info, info_span};

use crate::{
App, Result, accounts, commits, facts, mainline,
App, Result, accounts, commits, describe, facts, mainline,
mirror::{self, Mirror, Refs},
pull_heads, pulls, telemetry,
};
Expand Down Expand Up @@ -81,7 +81,7 @@ async fn step<T>(name: &'static str, work: impl Future<Output = Result<T>>) -> R
let status = source.status_code.as_str();
telemetry::GITHUB_ERRORS.with_label_values(&[status]).inc();
}
format!("{name}: {error}").into()
format!("{name}: {}", describe(&*error)).into()
})
}

Expand Down
4 changes: 4 additions & 0 deletions src/limits.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ pub const WEBHOOK_BODY: usize = 25 << 20;
pub const WEBHOOKS_AT_ONCE: usize = 8;
/// GitHub's own: it gives up on a delivery after 10 seconds.
pub const WEBHOOK_TIME: Duration = Duration::from_secs(10);
/// Sign-ins asking GitHub at once.
pub const SIGN_INS_AT_ONCE: usize = 8;
/// Of a whole sign-in, waiting for a turn included: a person is watching a browser.
pub const SIGN_IN_TIME: Duration = Duration::from_secs(30);
pub const GITHUB_REQUEST: Duration = Duration::from_secs(30);
/// Of one request, after a server or transport error, or a `429`.
pub const GITHUB_RETRIES: usize = 3;
Expand Down
49 changes: 40 additions & 9 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -188,20 +188,27 @@ async fn main() -> Result {
}

fn router(app: Arc<App>) -> Router {
// For what anyone can reach. The timeout covers waiting for a turn.
let bound = |time, at_once| {
ServiceBuilder::new()
.layer(TimeoutLayer::with_status_code(
StatusCode::REQUEST_TIMEOUT,
time,
))
.layer(ConcurrencyLimitLayer::new(at_once))
};
Router::new()
.route("/setup", get(sign_in::setup))
.route("/callback", get(sign_in::callback))
// Anyone can reach it, and a body is read whole before its signature is checked. axum
// accepts 2 MB by default. The timeout covers waiting for a turn.
// It asks GitHub whenever the cookie matches `state`, and the caller picks both.
.route(
"/callback",
get(sign_in::callback).layer(bound(limits::SIGN_IN_TIME, limits::SIGN_INS_AT_ONCE)),
)
// A body is read whole before its signature is checked. axum accepts 2 MB by default.
.route(
"/webhook",
post(sync::webhook).layer(
ServiceBuilder::new()
.layer(TimeoutLayer::with_status_code(
StatusCode::REQUEST_TIMEOUT,
limits::WEBHOOK_TIME,
))
.layer(ConcurrencyLimitLayer::new(limits::WEBHOOKS_AT_ONCE))
bound(limits::WEBHOOK_TIME, limits::WEBHOOKS_AT_ONCE)
.layer(DefaultBodyLimit::max(limits::WEBHOOK_BODY)),
),
)
Expand Down Expand Up @@ -271,6 +278,17 @@ fn column<'r, R, T>(rows: &'r [R], field: impl Fn(&'r R) -> T) -> Vec<T> {
rows.iter().map(field).collect()
}

/// An error as logged. octocrab displays an answer GitHub refused with as `GitHub` alone: this
/// adds the status and GitHub's message.
fn describe(error: &(dyn Error + 'static)) -> String {
match error.downcast_ref() {
Some(octocrab::Error::GitHub { source, .. }) => {
format!("GitHub: {}: {}", source.status_code, source.message)
}
_ => error.to_string(),
}
}

/// Every item of GitHub's list at `path`. Fails past `limit` items.
async fn list<T: DeserializeOwned>(github: &Octocrab, path: &str, limit: usize) -> Result<Vec<T>> {
let mut items = Vec::new();
Expand Down Expand Up @@ -337,6 +355,19 @@ mod tests {
}
}

#[tokio::test]
async fn describe_adds_githubs_status_and_message() {
let github = fixtures::github(|_| async {
let body = serde_json::json!({ "message": "Bad credentials" });
(StatusCode::UNAUTHORIZED, body)
});
let error = page::<()>(&github, "/app/installations", 1).await;
assert_eq!(
describe(&*error.unwrap_err()),
"GitHub: 401 Unauthorized: Bad credentials"
);
}

/// As GitHub signs with the secret `secret`: HMAC-SHA256.
fn sign(body: &str) -> String {
use sha2::{Digest, Sha256};
Expand Down
48 changes: 44 additions & 4 deletions src/sign_in.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Signing people in with GitHub after they install the app.

use std::{fmt::Display, sync::Arc};
use std::{error::Error, sync::Arc};

use axum::{
extract::{Query, State},
Expand All @@ -16,7 +16,7 @@ use serde::Deserialize;
use sha2::{Digest, Sha256};
use tracing::{info, warn};

use crate::{App, limits, list};
use crate::{App, describe, limits, list};

/// `__Host-`: set by this origin only, over HTTPS only.
const VERIFIER_COOKIE: &str = "__Host-pkce";
Expand Down Expand Up @@ -107,7 +107,47 @@ pub async fn callback(
))
}

fn failed(error: impl Display) -> StatusCode {
warn!(%error, "sign-in failed");
fn failed(error: impl Into<Box<dyn Error + Send + Sync>>) -> StatusCode {
warn!(error = %describe(&*error.into()), "sign-in failed");
StatusCode::BAD_GATEWAY
}

#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};

use axum::{body::Body, http::Request};
use tower::ServiceExt;

use super::*;
use crate::fixtures;

/// GitHub never answers. Callbacks that pass the check ask it only so many at once, and all
/// give up, those still waiting for a turn included.
#[tokio::test(start_paused = true)]
async fn callbacks_are_bounded() {
let asked = Arc::new(AtomicUsize::new(0));
let counted = asked.clone();
let github = fixtures::github(move |_| {
counted.fetch_add(1, Ordering::SeqCst);
std::future::pending()
});
let router = crate::router(fixtures::app(github));
let uri = format!("/callback?code=code&state={}", challenge("verifier"));
let callbacks = Vec::from_iter((0..2 * limits::SIGN_INS_AT_ONCE).map(|_| {
let request = Request::get(&uri)
.header(COOKIE, format!("{VERIFIER_COOKIE}=verifier"))
.body(Body::empty())
.unwrap();
tokio::spawn(router.clone().oneshot(request))
}));

// The paused clock moves once every task waits.
tokio::time::sleep(limits::SIGN_IN_TIME / 2).await;
assert_eq!(asked.load(Ordering::SeqCst), limits::SIGN_INS_AT_ONCE);
for callback in callbacks {
let response = callback.await.unwrap().unwrap();
assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT);
}
}
}
6 changes: 4 additions & 2 deletions src/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use octocrab::{
use serde::Deserialize;
use tracing::{Instrument, Span, field::Empty, info, info_span, warn};

use crate::{App, Result, app, limits, list, telemetry};
use crate::{App, Result, app, describe, limits, list, telemetry};

// 1. Triggers

Expand Down Expand Up @@ -113,7 +113,7 @@ pub async fn resync(app: Arc<App>) {
match schedule_all(&app).await {
Ok(true) => telemetry::RESYNC_LAST_SUCCESS.set(telemetry::now()),
Ok(false) => {}
Err(error) => warn!(%error, "resync failed"),
Err(error) => warn!(error = %describe(&*error), "resync failed"),
}
}
}
Expand All @@ -135,6 +135,7 @@ async fn schedule_all(app: &Arc<App>) -> Result<bool> {
match scheduled.await {
Ok(scheduled) => repositories += scheduled,
Err(error) => {
let error = describe(&*error);
warn!(installation = installation.id.0, %error, "resync failed");
failed += 1;
}
Expand Down Expand Up @@ -199,6 +200,7 @@ async fn run(app: Arc<App>, repository: RepositoryId) {
let label = [repository.0.to_string()];
match (synced, kind) {
(Err(error), _) => {
let error = describe(&*error);
warn!(repository = repository.0, kind, %error, "sync failed");
telemetry::REPO_FAILURES.with_label_values(&label).inc();
}
Expand Down