diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e6e30fb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + check: + # aarch64-linux: what `cv` is deployed on, and built on, by a host whose own CI only evaluates. + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - uses: cachix/install-nix-action@v31 + # Builds `cv`, running its tests, for `module-in-sync`. Without a store cache: add one if + # this takes much more than 20 minutes. + - run: nix flake check -L diff --git a/Cargo.toml b/Cargo.toml index bca06cd..0f7b9c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,4 +56,6 @@ opt-level = 3 opt-level = 3 [dev-dependencies] +# `test-util`: tests of timeouts pause the clock. +tokio = { version = "1.54.1", features = ["test-util"] } tower = { version = "0.5.3", features = ["util"] } diff --git a/flake.nix b/flake.nix index 3f995e3..738db55 100644 --- a/flake.nix +++ b/flake.nix @@ -57,6 +57,7 @@ # Golden files that are not Cargo sources. ./src/facts/c ./src/facts/python + ./src/facts/rust ]; }; strictDeps = true; @@ -119,6 +120,29 @@ diff -u <(${lib.getExe cv} --help | ${flags}) <(< ${./nix/module.nix} ${flags}) touch $out ''; + + # The alert rules a host gets from the module, against `nix/alerts.test.yml`. + alerts = + let + host = lib.nixosSystem { + modules = [ + self.nixosModules.default + { + nixpkgs.pkgs = pkgs; + services.cv.enable = true; + services.prometheus.enable = true; + } + ]; + }; + rules = lib.head host.config.services.prometheus.ruleFiles; + in + pkgs.runCommand "alerts" { nativeBuildInputs = [ pkgs.prometheus.cli ]; } '' + ln -s ${rules} cv-alerts.json + cp ${./nix/alerts.test.yml} alerts.test.yml + promtool check rules cv-alerts.json + promtool test rules alerts.test.yml + touch $out + ''; } ); }; diff --git a/nix/alerts.test.yml b/nix/alerts.test.yml new file mode 100644 index 0000000..329db2a --- /dev/null +++ b/nix/alerts.test.yml @@ -0,0 +1,47 @@ +# Run by `nix flake check` against the rules in `module.nix`: what a process that has just started +# does not export. A sample a minute from 0; `_` is none. +rule_files: [cv-alerts.json] +evaluation_interval: 1m + +tests: + - name: the first sync after a start, an hour in, hangs + input_series: + - series: cv_syncs_running + values: 0x59 1x300 + alert_rule_test: + - alertname: CvSyncStuck + eval_time: 3h59m + - alertname: CvSyncStuck + eval_time: 4h + exp_alerts: + - exp_labels: { severity: warning } + exp_annotations: + summary: Syncs are running, but none has ended for 3 hours. + + - name: syncs end while another runs + input_series: + - series: cv_syncs_running + values: 1x300 + - series: cv_sync_duration_seconds_count{kind="sync", outcome="ok"} + values: 0+1x300 + alert_rule_test: + - alertname: CvSyncStuck + eval_time: 4h + + - name: one repository has only failed since the start, another synced + input_series: + - series: cv_repo_sync_failures_consecutive{repository="1"} + values: 1+0x400 + - series: cv_repo_sync_failures_consecutive{repository="2"} + values: 0x400 + - series: cv_repo_last_sync_success_timestamp_seconds{repository="2"} + values: 0+60x400 + alert_rule_test: + - alertname: CvRepoStale + eval_time: 5h59m + - alertname: CvRepoStale + eval_time: 6h + exp_alerts: + - exp_labels: { severity: warning, repository: "1" } + exp_annotations: + summary: Repository 1 has not synced for over 6 hours. diff --git a/nix/module.nix b/nix/module.nix index 6ab871d..7f2d6ad 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -26,11 +26,17 @@ let (alert "CvRepoSyncFailing" "cv_repo_sync_failures_consecutive >= 3" "0m" "Repository {{ $labels.repository }} failed its last {{ $value }} syncs; see the journal." ) - (alert "CvRepoStale" "time() - cv_repo_last_sync_success_timestamp_seconds > 6 * 3600" "0m" + # A process exports no success until a repository has one, and no duration until a sync + # ends: nothing to compute an age from. So these two rule out recent progress, and `for` + # counts the time. + (alert "CvRepoStale" + "cv_repo_sync_failures_consecutive unless time() - cv_repo_last_sync_success_timestamp_seconds < 900" + "6h" "Repository {{ $labels.repository }} has not synced for over 6 hours." ) (alert "CvSyncStuck" - "cv_syncs_running > 0 and on () sum(increase(cv_sync_duration_seconds_count[3h])) == 0" "0m" + "cv_syncs_running > 0 unless on () sum(increase(cv_sync_duration_seconds_count[15m])) > 0" + "3h" "Syncs are running, but none has ended for 3 hours." ) (alert "CvResyncFailing" "time() - cv_resync_last_success_timestamp_seconds > 3 * 3600" "0m" diff --git a/src/app.rs b/src/app.rs index 5f672db..f6f4fad 100644 --- a/src/app.rs +++ b/src/app.rs @@ -22,7 +22,7 @@ use sqlx::PgPool; use tracing::{Instrument, info, info_span}; use crate::{ - App, Result, accounts, commits, facts, mainline, + App, Result, accounts, commits, describe, facts, mainline, mirror::{self, Mirror, Refs}, pull_heads, pulls, telemetry, }; @@ -81,7 +81,7 @@ async fn step(name: &'static str, work: impl Future>) -> R let status = source.status_code.as_str(); telemetry::GITHUB_ERRORS.with_label_values(&[status]).inc(); } - format!("{name}: {error}").into() + format!("{name}: {}", describe(&*error)).into() }) } diff --git a/src/limits.rs b/src/limits.rs index c1be295..180237f 100644 --- a/src/limits.rs +++ b/src/limits.rs @@ -29,6 +29,10 @@ pub const WEBHOOK_BODY: usize = 25 << 20; pub const WEBHOOKS_AT_ONCE: usize = 8; /// GitHub's own: it gives up on a delivery after 10 seconds. pub const WEBHOOK_TIME: Duration = Duration::from_secs(10); +/// Sign-ins asking GitHub at once. +pub const SIGN_INS_AT_ONCE: usize = 8; +/// Of a whole sign-in, waiting for a turn included: a person is watching a browser. +pub const SIGN_IN_TIME: Duration = Duration::from_secs(30); pub const GITHUB_REQUEST: Duration = Duration::from_secs(30); /// Of one request, after a server or transport error, or a `429`. pub const GITHUB_RETRIES: usize = 3; diff --git a/src/main.rs b/src/main.rs index ab20da1..46be1fa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -188,20 +188,27 @@ async fn main() -> Result { } fn router(app: Arc) -> Router { + // For what anyone can reach. The timeout covers waiting for a turn. + let bound = |time, at_once| { + ServiceBuilder::new() + .layer(TimeoutLayer::with_status_code( + StatusCode::REQUEST_TIMEOUT, + time, + )) + .layer(ConcurrencyLimitLayer::new(at_once)) + }; Router::new() .route("/setup", get(sign_in::setup)) - .route("/callback", get(sign_in::callback)) - // Anyone can reach it, and a body is read whole before its signature is checked. axum - // accepts 2 MB by default. The timeout covers waiting for a turn. + // It asks GitHub whenever the cookie matches `state`, and the caller picks both. + .route( + "/callback", + get(sign_in::callback).layer(bound(limits::SIGN_IN_TIME, limits::SIGN_INS_AT_ONCE)), + ) + // A body is read whole before its signature is checked. axum accepts 2 MB by default. .route( "/webhook", post(sync::webhook).layer( - ServiceBuilder::new() - .layer(TimeoutLayer::with_status_code( - StatusCode::REQUEST_TIMEOUT, - limits::WEBHOOK_TIME, - )) - .layer(ConcurrencyLimitLayer::new(limits::WEBHOOKS_AT_ONCE)) + bound(limits::WEBHOOK_TIME, limits::WEBHOOKS_AT_ONCE) .layer(DefaultBodyLimit::max(limits::WEBHOOK_BODY)), ), ) @@ -271,6 +278,17 @@ fn column<'r, R, T>(rows: &'r [R], field: impl Fn(&'r R) -> T) -> Vec { rows.iter().map(field).collect() } +/// An error as logged. octocrab displays an answer GitHub refused with as `GitHub` alone: this +/// adds the status and GitHub's message. +fn describe(error: &(dyn Error + 'static)) -> String { + match error.downcast_ref() { + Some(octocrab::Error::GitHub { source, .. }) => { + format!("GitHub: {}: {}", source.status_code, source.message) + } + _ => error.to_string(), + } +} + /// Every item of GitHub's list at `path`. Fails past `limit` items. async fn list(github: &Octocrab, path: &str, limit: usize) -> Result> { let mut items = Vec::new(); @@ -337,6 +355,19 @@ mod tests { } } + #[tokio::test] + async fn describe_adds_githubs_status_and_message() { + let github = fixtures::github(|_| async { + let body = serde_json::json!({ "message": "Bad credentials" }); + (StatusCode::UNAUTHORIZED, body) + }); + let error = page::<()>(&github, "/app/installations", 1).await; + assert_eq!( + describe(&*error.unwrap_err()), + "GitHub: 401 Unauthorized: Bad credentials" + ); + } + /// As GitHub signs with the secret `secret`: HMAC-SHA256. fn sign(body: &str) -> String { use sha2::{Digest, Sha256}; diff --git a/src/sign_in.rs b/src/sign_in.rs index 64b7239..bcc4e8f 100644 --- a/src/sign_in.rs +++ b/src/sign_in.rs @@ -1,6 +1,6 @@ //! Signing people in with GitHub after they install the app. -use std::{fmt::Display, sync::Arc}; +use std::{error::Error, sync::Arc}; use axum::{ extract::{Query, State}, @@ -16,7 +16,7 @@ use serde::Deserialize; use sha2::{Digest, Sha256}; use tracing::{info, warn}; -use crate::{App, limits, list}; +use crate::{App, describe, limits, list}; /// `__Host-`: set by this origin only, over HTTPS only. const VERIFIER_COOKIE: &str = "__Host-pkce"; @@ -107,7 +107,47 @@ pub async fn callback( )) } -fn failed(error: impl Display) -> StatusCode { - warn!(%error, "sign-in failed"); +fn failed(error: impl Into>) -> StatusCode { + warn!(error = %describe(&*error.into()), "sign-in failed"); StatusCode::BAD_GATEWAY } + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use axum::{body::Body, http::Request}; + use tower::ServiceExt; + + use super::*; + use crate::fixtures; + + /// GitHub never answers. Callbacks that pass the check ask it only so many at once, and all + /// give up, those still waiting for a turn included. + #[tokio::test(start_paused = true)] + async fn callbacks_are_bounded() { + let asked = Arc::new(AtomicUsize::new(0)); + let counted = asked.clone(); + let github = fixtures::github(move |_| { + counted.fetch_add(1, Ordering::SeqCst); + std::future::pending() + }); + let router = crate::router(fixtures::app(github)); + let uri = format!("/callback?code=code&state={}", challenge("verifier")); + let callbacks = Vec::from_iter((0..2 * limits::SIGN_INS_AT_ONCE).map(|_| { + let request = Request::get(&uri) + .header(COOKIE, format!("{VERIFIER_COOKIE}=verifier")) + .body(Body::empty()) + .unwrap(); + tokio::spawn(router.clone().oneshot(request)) + })); + + // The paused clock moves once every task waits. + tokio::time::sleep(limits::SIGN_IN_TIME / 2).await; + assert_eq!(asked.load(Ordering::SeqCst), limits::SIGN_INS_AT_ONCE); + for callback in callbacks { + let response = callback.await.unwrap().unwrap(); + assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT); + } + } +} diff --git a/src/sync.rs b/src/sync.rs index a1212d4..bae93e1 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -29,7 +29,7 @@ use octocrab::{ use serde::Deserialize; use tracing::{Instrument, Span, field::Empty, info, info_span, warn}; -use crate::{App, Result, app, limits, list, telemetry}; +use crate::{App, Result, app, describe, limits, list, telemetry}; // 1. Triggers @@ -113,7 +113,7 @@ pub async fn resync(app: Arc) { match schedule_all(&app).await { Ok(true) => telemetry::RESYNC_LAST_SUCCESS.set(telemetry::now()), Ok(false) => {} - Err(error) => warn!(%error, "resync failed"), + Err(error) => warn!(error = %describe(&*error), "resync failed"), } } } @@ -135,6 +135,7 @@ async fn schedule_all(app: &Arc) -> Result { match scheduled.await { Ok(scheduled) => repositories += scheduled, Err(error) => { + let error = describe(&*error); warn!(installation = installation.id.0, %error, "resync failed"); failed += 1; } @@ -199,6 +200,7 @@ async fn run(app: Arc, repository: RepositoryId) { let label = [repository.0.to_string()]; match (synced, kind) { (Err(error), _) => { + let error = describe(&*error); warn!(repository = repository.0, kind, %error, "sync failed"); telemetry::REPO_FAILURES.with_label_values(&label).inc(); }