From d7eb6c00f2104c0bc006a7c7a1fa78d86506cc53 Mon Sep 17 00:00:00 2001 From: Amp Date: Fri, 7 Aug 2026 23:27:19 +0000 Subject: [PATCH 1/3] Add manual service cron runs Amp-Thread-ID: https://ampcode.com/threads/T-019fd70c-af2d-706c-8870-496c40cf01eb Co-authored-by: Arjun Komath --- docs/services/configuration.mdx | 4 +- web/actions/crons.ts | 34 +++++ .../service/details/crons-section.tsx | 75 ++++++++-- web/lib/inngest/events/service-cron.ts | 7 +- web/lib/inngest/functions/crons.ts | 1 + .../functions/service-cron-workflow.ts | 1 + web/lib/service-crons.ts | 139 +++--------------- web/tests/service-crons.test.ts | 88 +++-------- 8 files changed, 146 insertions(+), 203 deletions(-) create mode 100644 web/actions/crons.ts diff --git a/docs/services/configuration.mdx b/docs/services/configuration.mdx index f01be417..371c5cb1 100644 --- a/docs/services/configuration.mdx +++ b/docs/services/configuration.mdx @@ -41,10 +41,10 @@ Add these service secrets from the web UI: | Secret | Required | Description | | --- | --- | --- | -| `CRON_BASE_URL` | Yes | Public HTTP or HTTPS origin joined with each configured path | +| `CRON_BASE_URL` | Yes | HTTP or HTTPS origin joined with each configured path | | `CRON_SECRET` | No | Sent as `Authorization: Bearer `; requires an HTTPS base URL | -The control plane only connects to public destinations. It blocks private and reserved addresses, does not follow redirects, and gives each request a 10-second deadline. Cron requests do not retry. If the control plane misses multiple intervals, it sends only the latest due occurrence instead of backfilling every missed run. +The destination must be reachable from the control plane. Cron requests do not follow redirects or retry, and each request has a 10-second deadline. If the control plane misses multiple intervals, it sends only the latest due occurrence instead of backfilling every missed run. Missing or invalid configuration produces a skipped run. Redirects and non-2xx responses produce failed runs. The read-only **Crons** section in service configuration shows the latest result. Full history is available in [service logs](/infrastructure/logging) and follows the configured log retention period. diff --git a/web/actions/crons.ts b/web/actions/crons.ts new file mode 100644 index 00000000..eb99c38f --- /dev/null +++ b/web/actions/crons.ts @@ -0,0 +1,34 @@ +"use server"; + +import { and, eq, isNull } from "drizzle-orm"; +import { db } from "@/db"; +import { serviceCrons, services } from "@/db/schema"; +import { requireDeveloperRole } from "@/lib/auth"; +import { inngest } from "@/lib/inngest/client"; +import { inngestEvents } from "@/lib/inngest/events"; + +export async function runServiceCron(cronId: string) { + await requireDeveloperRole(); + const cron = await db + .select({ id: serviceCrons.id, schedule: serviceCrons.schedule }) + .from(serviceCrons) + .innerJoin( + services, + and(eq(serviceCrons.serviceId, services.id), isNull(services.deletedAt)), + ) + .where(eq(serviceCrons.id, cronId)) + .limit(1) + .then((rows) => rows[0]); + if (!cron) throw new Error("Cron job not found"); + + await inngest.send( + inngestEvents.serviceCronExecute.create({ + cronId: cron.id, + schedule: cron.schedule, + scheduledFor: new Date().toISOString(), + source: "manual", + }), + ); + + return { success: true }; +} diff --git a/web/components/service/details/crons-section.tsx b/web/components/service/details/crons-section.tsx index 3d7a34e7..8dd1e287 100644 --- a/web/components/service/details/crons-section.tsx +++ b/web/components/service/details/crons-section.tsx @@ -1,10 +1,21 @@ "use client"; import cronstrue from "cronstrue"; -import { Ban, CheckCircle2, XCircle } from "lucide-react"; -import { memo } from "react"; +import { + AlertTriangle, + Ban, + CheckCircle2, + Loader2, + Play, + XCircle, +} from "lucide-react"; +import { memo, useState } from "react"; +import { toast } from "sonner"; +import { runServiceCron } from "@/actions/crons"; import { LocalDate } from "@/components/core/local-date"; import { ConfigSection } from "@/components/service/details/config-section"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; import { StatusBadge } from "@/components/ui/status-badge"; import type { ServiceCron, ServiceWithDetails as Service } from "@/db/types"; @@ -54,6 +65,21 @@ export const CronsSection = memo(function CronsSection({ service: Service; }) { const crons = service.crons ?? []; + const [runningCronId, setRunningCronId] = useState(null); + + const handleRun = async (cronId: string) => { + setRunningCronId(cronId); + try { + await runServiceCron(cronId); + toast.success("Cron run queued"); + } catch (error) { + toast.error( + error instanceof Error ? error.message : "Failed to queue cron run", + ); + } finally { + setRunningCronId(null); + } + }; return ( + {crons.length > 0 && ( + + + + Cron base URL required + + + Cron jobs will not run until the{" "} + CRON_BASE_URL secret is set. + + + )} + {crons.length === 0 ? (

No cron jobs configured. @@ -76,14 +115,30 @@ export const CronsSection = memo(function CronsSection({

{crons.map((cron) => (
-
-

- {cron.path} -

-

{cron.schedule}

-

- {describeSchedule(cron.schedule)} (UTC) -

+
+
+

+ {cron.path} +

+

{cron.schedule}

+

+ {describeSchedule(cron.schedule)} (UTC) +

+
+
{cron.lastStatus ? ( diff --git a/web/lib/inngest/events/service-cron.ts b/web/lib/inngest/events/service-cron.ts index 49fc3e4a..9cce76d2 100644 --- a/web/lib/inngest/events/service-cron.ts +++ b/web/lib/inngest/events/service-cron.ts @@ -1,5 +1,10 @@ export type ServiceCronEvents = { "service-cron/execute": { - data: { cronId: string; schedule: string; scheduledFor: string }; + data: { + cronId: string; + schedule: string; + scheduledFor: string; + source: "scheduled" | "manual"; + }; }; }; diff --git a/web/lib/inngest/functions/crons.ts b/web/lib/inngest/functions/crons.ts index 89b5a9b7..1489b637 100644 --- a/web/lib/inngest/functions/crons.ts +++ b/web/lib/inngest/functions/crons.ts @@ -252,6 +252,7 @@ export const serviceCronDispatcher = inngest.createFunction( cronId: row.id, schedule: row.schedule, scheduledFor: occurrence.toISOString(), + source: "scheduled", }, }); await db diff --git a/web/lib/inngest/functions/service-cron-workflow.ts b/web/lib/inngest/functions/service-cron-workflow.ts index 3bec2513..fd796139 100644 --- a/web/lib/inngest/functions/service-cron-workflow.ts +++ b/web/lib/inngest/functions/service-cron-workflow.ts @@ -15,6 +15,7 @@ export const serviceCronWorkflow = inngest.createFunction( event.data.cronId, event.data.schedule, new Date(event.data.scheduledFor), + event.data.source, ), ), ); diff --git a/web/lib/service-crons.ts b/web/lib/service-crons.ts index 78d0d8db..68f9c89c 100644 --- a/web/lib/service-crons.ts +++ b/web/lib/service-crons.ts @@ -1,10 +1,7 @@ -import { lookup as dnsLookup } from "node:dns/promises"; import * as http from "node:http"; import * as https from "node:https"; -import { isIP } from "node:net"; import { and, eq, inArray, isNull, lt, or } from "drizzle-orm"; import { CronExpressionParser } from "cron-parser"; -import { Address4, Address6 } from "ip-address"; import { db } from "@/db"; import { secrets, serviceCrons, services } from "@/db/schema"; import { decryptSecret } from "@/lib/crypto"; @@ -13,28 +10,7 @@ import { ingestCronLog, type CronLog } from "@/lib/victoria-logs"; const MAX_ERROR = 500; const EXECUTION_BUDGET_MS = 10_000; -const blockedV4 = [ - "0.0.0.0/8", - "10.0.0.0/8", - "100.64.0.0/10", - "127.0.0.0/8", - "169.254.0.0/16", - "172.16.0.0/12", - "192.0.0.0/24", - "192.0.2.0/24", - "192.88.99.0/24", - "192.168.0.0/16", - "198.18.0.0/15", - "198.51.100.0/24", - "203.0.113.0/24", - "224.0.0.0/4", - "240.0.0.0/4", -].map((value) => new Address4(value)); -const blockedV6 = ["2001::/23", "2001:db8::/32", "3fff::/20"].map( - (value) => new Address6(value), -); -export type ResolvedAddress = { address: string; family: 4 | 6 }; export type CronRequestResult = { status: "succeeded" | "failed"; statusCode: number | null; @@ -72,24 +48,6 @@ export function sanitizeCronError(error: unknown): string { return message.replace(/[\u0000-\u001f\u007f]/g, " ").slice(0, MAX_ERROR); } -export function isGlobalAddress(value: string): boolean { - try { - if (isIP(value) === 4) { - const address = new Address4(value); - return !blockedV4.some((range) => address.isInSubnet(range)); - } - if (isIP(value) === 6) { - const address = new Address6(value); - if (address.is4()) return isGlobalAddress(address.to4().address); - return ( - address.isInSubnet(new Address6("2000::/3")) && - !blockedV6.some((range) => address.isInSubnet(range)) - ); - } - } catch {} - return false; -} - export function parseCronUrl(base: string, path: string): URL { if (!isSafeCronPath(path)) throw new Error("Invalid cron path"); let url: URL; @@ -109,34 +67,6 @@ export function parseCronUrl(base: string, path: string): URL { return new URL(path, `${url.origin}/`); } -export async function resolvePublicAddresses( - hostname: string, - lookup: ( - hostname: string, - options: { all: true; verbatim: true }, - ) => Promise = async (host, options) => - (await dnsLookup(host, options)).map((answer) => ({ - address: answer.address, - family: answer.family as 4 | 6, - })), -): Promise { - const literal = isIP(hostname); - let answers: ResolvedAddress[]; - try { - answers = literal - ? [{ address: hostname, family: literal as 4 | 6 }] - : await lookup(hostname, { all: true, verbatim: true }); - } catch { - throw new Error("DNS lookup failed"); - } - if ( - !answers.length || - answers.some(({ address }) => !isGlobalAddress(address)) - ) - throw new Error("Cron destination is not public"); - return answers; -} - type RequestImpl = typeof http.request; export function validateCronTransport(url: URL, secret?: string): void { if (secret && url.protocol === "http:") @@ -145,7 +75,6 @@ export function validateCronTransport(url: URL, secret?: string): void { export async function performCronGet( url: URL, - addresses: ResolvedAddress[], secret: string | undefined, timeoutMs: number, requestImpl: RequestImpl = url.protocol === "https:" @@ -165,19 +94,11 @@ export async function performCronGet( statusCode: null, error: "Cron request timed out", }); - const requestOptions: http.RequestOptions & { autoSelectFamily: boolean } = - { - method: "GET", - agent: false, - autoSelectFamily: false, - headers: secret ? { Authorization: `Bearer ${secret}` } : undefined, - lookup: (_hostname, options, callback) => { - const selected = addresses[0]; - if (options?.all) callback(null, addresses); - else callback(null, selected.address, selected.family); - }, - ...(url.protocol === "https:" ? { servername: url.hostname } : {}), - }; + const requestOptions: http.RequestOptions = { + method: "GET", + agent: false, + headers: secret ? { Authorization: `Bearer ${secret}` } : undefined, + }; const req = requestImpl(url, requestOptions, (response) => { const code = response.statusCode ?? null; response.destroy(); @@ -219,33 +140,11 @@ export async function performCronGet( req.end(); }); } - -async function withinDeadline( - promise: Promise, - deadline: number, -): Promise { - const remaining = deadline - Date.now(); - if (remaining <= 0) throw new Error("Cron request timed out"); - let timer: ReturnType; - try { - return await Promise.race([ - promise, - new Promise((_, reject) => { - timer = setTimeout( - () => reject(new Error("Cron request timed out")), - remaining, - ); - }), - ]); - } finally { - clearTimeout(timer!); - } -} - export async function executeServiceCron( cronId: string, schedule: string, scheduledFor: Date, + source: "scheduled" | "manual", ) { const deadline = Date.now() + EXECUTION_BUDGET_MS; const row = await db @@ -262,15 +161,20 @@ export async function executeServiceCron( const startedAt = new Date(); const claimed = await db .update(serviceCrons) - .set({ lastAttemptedFor: scheduledFor, lastStartedAt: startedAt }) + .set({ + lastStartedAt: startedAt, + ...(source === "scheduled" ? { lastAttemptedFor: scheduledFor } : {}), + }) .where( and( eq(serviceCrons.id, cronId), eq(serviceCrons.schedule, schedule), - or( - isNull(serviceCrons.lastAttemptedFor), - lt(serviceCrons.lastAttemptedFor, scheduledFor), - ), + source === "scheduled" + ? or( + isNull(serviceCrons.lastAttemptedFor), + lt(serviceCrons.lastAttemptedFor, scheduledFor), + ) + : undefined, ), ) .returning({ id: serviceCrons.id }); @@ -309,21 +213,14 @@ export async function executeServiceCron( const url = parseCronUrl(base.trim(), row.cron.path); validateCronTransport(url, secret); try { - const addresses = await withinDeadline( - resolvePublicAddresses(url.hostname), - deadline, - ); ({ status, statusCode, error } = await performCronGet( url, - addresses, secret, deadline - Date.now(), )); } catch (cause) { - const message = sanitizeCronError(cause); - status = - message === "Cron destination is not public" ? "skipped" : "failed"; - error = message; + status = "failed"; + error = sanitizeCronError(cause); } } catch (cause) { error = sanitizeCronError(cause); diff --git a/web/tests/service-crons.test.ts b/web/tests/service-crons.test.ts index 351f054e..ba144ddf 100644 --- a/web/tests/service-crons.test.ts +++ b/web/tests/service-crons.test.ts @@ -3,16 +3,14 @@ import type { ClientRequest, IncomingMessage, RequestOptions } from "node:http"; import { describe, expect, it, vi } from "vitest"; import { cronEventId, - isGlobalAddress, latestDueOccurrence, nextOccurrenceAfter, parseCronUrl, performCronGet, - resolvePublicAddresses, validateCronTransport, } from "@/lib/service-crons"; -describe("service cron scheduling and SSRF validation", () => { +describe("service cron scheduling and requests", () => { it("returns the first UTC occurrence strictly after the supplied instant", () => { expect( nextOccurrenceAfter( @@ -56,33 +54,6 @@ describe("service cron scheduling and SSRF validation", () => { ); }); - it.each([ - "10.0.0.1", - "127.0.0.1", - "169.254.1.1", - "192.0.2.1", - "192.88.99.1", - "::1", - "fc00::1", - "fe80::1", - "2001::1", - "2001:db8::1", - "3fff::1", - "::ffff:10.0.0.1", - ])("rejects non-global address %s", (address) => - expect(isGlobalAddress(address)).toBe(false), - ); - - it("rejects mixed DNS results", async () => { - const lookup = vi.fn(async () => [ - { address: "8.8.8.8", family: 4 as const }, - { address: "10.0.0.1", family: 4 as const }, - ]); - await expect(resolvePublicAddresses("example.com", lookup)).rejects.toThrow( - "not public", - ); - }); - it("rejects a secret over cleartext HTTP", () => { expect(() => validateCronTransport(new URL("http://example.com/job"), "not-logged"), @@ -92,44 +63,27 @@ describe("service cron scheduling and SSRF validation", () => { ).not.toThrow(); }); - it.each([false, true])( - "pins lookup for all=%s without pooling", - async (all) => { - let options!: RequestOptions & { autoSelectFamily?: boolean }; - const request = fakeRequest(204, (received) => { - options = received; - }); - const result = performCronGet( + it("performs a GET without connection pooling", async () => { + let options!: RequestOptions; + const request = fakeRequest(204, (received) => { + options = received; + }); + await expect( + performCronGet( new URL("https://example.com/job"), - [ - { address: "8.8.8.8", family: 4 }, - { address: "2001:4860:4860::8888", family: 6 }, - ], undefined, 1_000, request, - ); - const callback = vi.fn(); - options.lookup!("example.com", { all }, callback); - if (all) - expect(callback).toHaveBeenCalledWith(null, [ - { address: "8.8.8.8", family: 4 }, - { address: "2001:4860:4860::8888", family: 6 }, - ]); - else expect(callback).toHaveBeenCalledWith(null, "8.8.8.8", 4); - expect(options).toMatchObject({ - agent: false, - autoSelectFamily: false, - servername: "example.com", - }); - expect(await result).toEqual({ - status: "succeeded", - statusCode: 204, - error: null, - }); - expect(request).toHaveBeenCalledTimes(1); - }, - ); + ), + ).resolves.toEqual({ + status: "succeeded", + statusCode: 204, + error: null, + }); + expect(options).toMatchObject({ method: "GET", agent: false }); + expect(options.lookup).toBeUndefined(); + expect(request).toHaveBeenCalledTimes(1); + }); it.each([302, 404])( "classifies HTTP %s at headers without retaining a body", @@ -141,7 +95,6 @@ describe("service cron scheduling and SSRF validation", () => { await expect( performCronGet( new URL("https://example.com/job"), - [{ address: "8.8.8.8", family: 4 }], undefined, 1_000, request, @@ -163,7 +116,6 @@ describe("service cron scheduling and SSRF validation", () => { }) as unknown as typeof import("node:http").request; const result = performCronGet( new URL("https://example.com/job"), - [{ address: "8.8.8.8", family: 4 }], undefined, 25, request, @@ -183,9 +135,7 @@ describe("service cron scheduling and SSRF validation", () => { function fakeRequest( statusCode: number, - onOptions?: ( - options: RequestOptions & { autoSelectFamily?: boolean }, - ) => void, + onOptions?: (options: RequestOptions) => void, onResponseDestroy?: () => void, ) { return vi.fn( From 039833fea5d5e09d19f18a3c7de77e318c713725 Mon Sep 17 00:00:00 2001 From: Amp Date: Sat, 8 Aug 2026 01:00:23 +0000 Subject: [PATCH 2/3] Address service cron review feedback Amp-Thread-ID: https://ampcode.com/threads/T-019fd70c-af2d-706c-8870-496c40cf01eb Co-authored-by: Arjun Komath --- docs/services/configuration.mdx | 2 +- .../service/details/crons-section.tsx | 22 ++++++++++++++----- web/lib/service-crons.ts | 1 + web/lib/victoria-logs.ts | 1 + web/tests/victoria-logs.test.ts | 2 ++ 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/services/configuration.mdx b/docs/services/configuration.mdx index 371c5cb1..c1c44b64 100644 --- a/docs/services/configuration.mdx +++ b/docs/services/configuration.mdx @@ -46,7 +46,7 @@ Add these service secrets from the web UI: The destination must be reachable from the control plane. Cron requests do not follow redirects or retry, and each request has a 10-second deadline. If the control plane misses multiple intervals, it sends only the latest due occurrence instead of backfilling every missed run. -Missing or invalid configuration produces a skipped run. Redirects and non-2xx responses produce failed runs. The read-only **Crons** section in service configuration shows the latest result. Full history is available in [service logs](/infrastructure/logging) and follows the configured log retention period. +Missing or invalid configuration produces a skipped run. Redirects and non-2xx responses produce failed runs. The **Crons** section in service configuration shows the latest result. Full history is available in [service logs](/infrastructure/logging) and follows the configured log retention period. `tc apply` treats `service.crons` as the complete desired list. Removing the field or setting it to an empty list removes all cron definitions. Cron-only changes take effect without creating a deployment revision. diff --git a/web/components/service/details/crons-section.tsx b/web/components/service/details/crons-section.tsx index 8dd1e287..fed41e3d 100644 --- a/web/components/service/details/crons-section.tsx +++ b/web/components/service/details/crons-section.tsx @@ -11,13 +11,19 @@ import { } from "lucide-react"; import { memo, useState } from "react"; import { toast } from "sonner"; +import useSWR from "swr"; import { runServiceCron } from "@/actions/crons"; import { LocalDate } from "@/components/core/local-date"; import { ConfigSection } from "@/components/service/details/config-section"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { StatusBadge } from "@/components/ui/status-badge"; -import type { ServiceCron, ServiceWithDetails as Service } from "@/db/types"; +import type { + Secret, + ServiceCron, + ServiceWithDetails as Service, +} from "@/db/types"; +import { fetcher } from "@/lib/fetcher"; const STATUS_CONFIG = { succeeded: { @@ -65,6 +71,13 @@ export const CronsSection = memo(function CronsSection({ service: Service; }) { const crons = service.crons ?? []; + const { data: secrets } = useSWR[]>( + crons.length > 0 ? `/api/services/${service.id}/secrets` : null, + fetcher, + ); + const hasCronBaseUrl = secrets?.some( + (secret) => secret.key === "CRON_BASE_URL", + ); const [runningCronId, setRunningCronId] = useState(null); const handleRun = async (cronId: string) => { @@ -89,12 +102,11 @@ export const CronsSection = memo(function CronsSection({ >

- Cron jobs are managed in{" "} - techulus.yml. Schedules run in UTC - and are read-only here. + Cron schedules are configured in{" "} + techulus.yml and run in UTC.

- {crons.length > 0 && ( + {crons.length > 0 && secrets !== undefined && !hasCronBaseUrl && ( diff --git a/web/lib/service-crons.ts b/web/lib/service-crons.ts index 68f9c89c..c40f76db 100644 --- a/web/lib/service-crons.ts +++ b/web/lib/service-crons.ts @@ -245,6 +245,7 @@ export async function executeServiceCron( service_id: row.serviceId, cron_id: cronId, path: row.cron.path, + source, scheduled_for: scheduledFor.toISOString(), started_at: startedAt.toISOString(), finished_at: finishedAt.toISOString(), diff --git a/web/lib/victoria-logs.ts b/web/lib/victoria-logs.ts index 482e5f6a..5e4111f3 100644 --- a/web/lib/victoria-logs.ts +++ b/web/lib/victoria-logs.ts @@ -362,6 +362,7 @@ export type CronLog = { service_id: string; cron_id: string; path: string; + source: "scheduled" | "manual"; scheduled_for: string; started_at: string; finished_at: string; diff --git a/web/tests/victoria-logs.test.ts b/web/tests/victoria-logs.test.ts index 070b8a03..fee38e43 100644 --- a/web/tests/victoria-logs.test.ts +++ b/web/tests/victoria-logs.test.ts @@ -400,6 +400,7 @@ describe("VictoriaLogs queries", () => { service_id: "service-1", cron_id: "cron-1", path: "/job", + source: "manual", scheduled_for: "2026-08-06T10:00:00Z", started_at: "2026-08-06T10:00:00Z", finished_at: "2026-08-06T10:00:01Z", @@ -413,6 +414,7 @@ describe("VictoriaLogs queries", () => { expect(String(url)).toBe("http://victoria.test/insert/jsonline"); expect(init?.signal).toBeInstanceOf(AbortSignal); expect(init?.body).toContain('"status":204'); + expect(init?.body).toContain('"source":"manual"'); expect(init?.body).not.toContain("Authorization"); expect(init?.body).not.toContain("CRON_BASE_URL"); }); From 7793706db776a235b0bb67be659057e7b3ec8ab3 Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Sun, 9 Aug 2026 21:56:01 +1000 Subject: [PATCH 3/3] Add cron failure notifications Amp-Thread-ID: https://ampcode.com/threads/T-019fe659-750f-76b7-aeb9-f597a646cf79 Co-authored-by: Amp --- web/components/settings/email-settings.tsx | 14 +++++- web/lib/email/index.ts | 57 ++++++++++++++++++++++ web/lib/inngest/events/notification.ts | 8 +++ web/lib/notifications/index.ts | 9 ++++ web/lib/service-crons.ts | 16 ++++++ web/lib/settings-keys.ts | 1 + web/tests/notifications.test.ts | 56 +++++++++++++++++++-- 7 files changed, 156 insertions(+), 5 deletions(-) diff --git a/web/components/settings/email-settings.tsx b/web/components/settings/email-settings.tsx index 6da5c5f4..3285e296 100644 --- a/web/components/settings/email-settings.tsx +++ b/web/components/settings/email-settings.tsx @@ -19,7 +19,8 @@ type AlertField = | "serverOfflineAlert" | "buildFailure" | "deploymentFailure" - | "deploymentMovedAlert"; + | "deploymentMovedAlert" + | "cronFailure"; type AlertSetting = { field: AlertField; @@ -49,6 +50,11 @@ const ALERT_SETTINGS: AlertSetting[] = [ description: "Receive a notification when offline replicas need manual recovery", }, + { + field: "cronFailure", + label: "Cron Failure Alert", + description: "Receive a notification when a cron run fails", + }, ]; type State = { @@ -56,6 +62,7 @@ type State = { buildFailure: boolean; deploymentFailure: boolean; deploymentMovedAlert: boolean; + cronFailure: boolean; isSavingAlerts: boolean; }; @@ -70,6 +77,7 @@ function createInitialState(props: Props): State { buildFailure: alertsConfig?.buildFailure ?? true, deploymentFailure: alertsConfig?.deploymentFailure ?? true, deploymentMovedAlert: alertsConfig?.deploymentMovedAlert ?? true, + cronFailure: alertsConfig?.cronFailure ?? true, isSavingAlerts: false, }; } @@ -99,6 +107,7 @@ export function EmailSettings({ initialAlertsConfig }: Props) { buildFailure: state.buildFailure, deploymentFailure: state.deploymentFailure, deploymentMovedAlert: state.deploymentMovedAlert, + cronFailure: state.cronFailure, }); toast.success("Alert settings saved"); router.refresh(); @@ -120,7 +129,8 @@ export function EmailSettings({ initialAlertsConfig }: Props) { state.deploymentFailure !== (initialAlertsConfig?.deploymentFailure ?? true) || state.deploymentMovedAlert !== - (initialAlertsConfig?.deploymentMovedAlert ?? true); + (initialAlertsConfig?.deploymentMovedAlert ?? true) || + state.cronFailure !== (initialAlertsConfig?.cronFailure ?? true); return (
diff --git a/web/lib/email/index.ts b/web/lib/email/index.ts index 1fe78ec0..d65b3b58 100644 --- a/web/lib/email/index.ts +++ b/web/lib/email/index.ts @@ -261,6 +261,60 @@ async function sendBuildFailureAlert( }); } +type CronFailureAlertOptions = { + to: string; + serviceId: string; + path: string; + statusCode: number | null; + error: string | null; +}; + +async function sendCronFailureAlert( + options: CronFailureAlertOptions, +): Promise { + const [result] = await db + .select({ + serviceName: services.name, + projectName: projects.name, + projectSlug: projects.slug, + envName: environments.name, + }) + .from(services) + .innerJoin(projects, eq(projects.id, services.projectId)) + .innerJoin(environments, eq(environments.id, services.environmentId)) + .where(eq(services.id, options.serviceId)); + + if (!result) return; + + const baseUrl = getAppBaseUrl(); + const serviceUrl = baseUrl + ? `${baseUrl}/dashboard/projects/${result.projectSlug}/${result.envName}/services/${options.serviceId}` + : undefined; + const details = [ + { label: "Service", value: result.serviceName }, + { label: "Project", value: result.projectName }, + { label: "Cron Path", value: options.path }, + ...(options.statusCode !== null + ? [{ label: "HTTP Status", value: String(options.statusCode) }] + : []), + ...(options.error ? [{ label: "Error", value: options.error }] : []), + ]; + + await sendAlert({ + to: options.to, + subject: `Cron Failed: ${result.serviceName}`, + template: Alert({ + bannerText: "CRON FAILED", + heading: "Cron Failure Alert", + description: `The cron job "${options.path}" for service "${result.serviceName}" in project "${result.projectName}" has failed.`, + details, + buttonText: serviceUrl ? "View Service" : undefined, + buttonUrl: serviceUrl, + baseUrl, + }), + }); +} + type DeploymentFailureAlertOptions = { to: string; serviceId: string; @@ -400,6 +454,9 @@ export async function deliverNotificationEmail( case "build.failed": await sendBuildFailureAlert({ ...event, to }); return; + case "cron.failed": + await sendCronFailureAlert({ ...event, to }); + return; case "deployment.failed": await sendDeploymentFailureAlert({ ...event, to }); } diff --git a/web/lib/inngest/events/notification.ts b/web/lib/inngest/events/notification.ts index 71ba5619..746205dc 100644 --- a/web/lib/inngest/events/notification.ts +++ b/web/lib/inngest/events/notification.ts @@ -29,6 +29,14 @@ export type NotificationEvent = serverId: string | null; failedStage?: string; } + | { + kind: "cron.failed"; + occurrenceId: string; + serviceId: string; + path: string; + statusCode: number | null; + error: string | null; + } | { kind: "member.invited"; occurrenceId: string; diff --git a/web/lib/notifications/index.ts b/web/lib/notifications/index.ts index 737c66e8..f978fc46 100644 --- a/web/lib/notifications/index.ts +++ b/web/lib/notifications/index.ts @@ -37,6 +37,8 @@ export async function notificationEventIsEnabled(event: NotificationEvent) { return config?.buildFailure !== false; case "deployment.failed": return config?.deploymentFailure !== false; + case "cron.failed": + return config?.cronFailure !== false; } } @@ -81,6 +83,13 @@ export async function renderInAppNotification(event: NotificationEvent) { href: `${serviceHref}/builds/${event.buildId}`, }; } + if (event.kind === "cron.failed") { + return { + title: `Cron failed: ${context.serviceName}`, + body: `${event.path}: ${event.error ?? "Cron request failed"}`, + href: serviceHref, + }; + } return { title: `Deployment failed: ${context.serviceName}`, body: event.failedStage diff --git a/web/lib/service-crons.ts b/web/lib/service-crons.ts index c40f76db..9b7b2c90 100644 --- a/web/lib/service-crons.ts +++ b/web/lib/service-crons.ts @@ -5,6 +5,7 @@ import { CronExpressionParser } from "cron-parser"; import { db } from "@/db"; import { secrets, serviceCrons, services } from "@/db/schema"; import { decryptSecret } from "@/lib/crypto"; +import { notify } from "@/lib/notifications"; import { isSafeCronPath, nextOccurrenceAfter } from "@/lib/public-api"; import { ingestCronLog, type CronLog } from "@/lib/victoria-logs"; @@ -256,5 +257,20 @@ export async function executeServiceCron( log_type: "cron", }; await ingestCronLog(log); + if (status === "failed") { + notify({ + kind: "cron.failed", + occurrenceId: cronEventId(cronId, scheduledFor), + serviceId: row.serviceId, + path: row.cron.path, + statusCode, + error, + }).catch((cause) => { + console.error( + "[service-cron] failed to enqueue cron failure notification:", + cause, + ); + }); + } return { stale: false as const, status, statusCode, error }; } diff --git a/web/lib/settings-keys.ts b/web/lib/settings-keys.ts index 1d3fd7b3..687eaaef 100644 --- a/web/lib/settings-keys.ts +++ b/web/lib/settings-keys.ts @@ -89,6 +89,7 @@ export const emailAlertsConfigSchema = z.object({ buildFailure: z.boolean(), deploymentFailure: z.boolean(), deploymentMovedAlert: z.boolean(), + cronFailure: z.boolean(), }); export type EmailAlertsConfig = z.infer; diff --git a/web/tests/notifications.test.ts b/web/tests/notifications.test.ts index e88c3d5e..8450deb6 100644 --- a/web/tests/notifications.test.ts +++ b/web/tests/notifications.test.ts @@ -99,12 +99,49 @@ describe("notification pipeline", () => { ).resolves.toBeNull(); }); + it("renders cron failures with the service deep link", async () => { + mocks.select.mockReturnValueOnce({ + from: vi.fn(() => ({ + innerJoin: vi.fn(() => ({ + innerJoin: vi.fn(() => ({ + where: vi.fn(() => + Promise.resolve([ + { + serviceName: "API", + projectName: "Cloud", + projectSlug: "cloud", + environmentName: "production", + }, + ]), + ), + })), + })), + })), + }); + + await expect( + renderInAppNotification({ + kind: "cron.failed", + occurrenceId: "cron-1", + serviceId: "service-1", + path: "/jobs/nightly", + statusCode: 500, + error: "HTTP status 500", + }), + ).resolves.toEqual({ + title: "Cron failed: API", + body: "/jobs/nightly: HTTP status 500", + href: "/dashboard/projects/cloud/production/services/service-1", + }); + }); + it("maps every operational event to its alert toggle", async () => { mocks.getAlertsConfig.mockResolvedValue({ serverOfflineAlert: false, buildFailure: false, deploymentFailure: false, deploymentMovedAlert: false, + cronFailure: false, }); await expect( @@ -141,6 +178,16 @@ describe("notification pipeline", () => { serverId: "server-1", }), ).resolves.toBe(false); + await expect( + notificationEventIsEnabled({ + kind: "cron.failed", + occurrenceId: "cron-1", + serviceId: "service-1", + path: "/jobs/nightly", + statusCode: 500, + error: "HTTP status 500", + }), + ).resolves.toBe(false); }); it("defaults missing alert settings to enabled", async () => { @@ -148,10 +195,12 @@ describe("notification pipeline", () => { await expect( notificationEventIsEnabled({ - kind: "build.failed", - occurrenceId: "build-1", + kind: "cron.failed", + occurrenceId: "cron-1", serviceId: "service-1", - buildId: "build-1", + path: "/jobs/nightly", + statusCode: null, + error: "Cron request failed", }), ).resolves.toBe(true); }); @@ -162,6 +211,7 @@ describe("notification pipeline", () => { buildFailure: true, deploymentFailure: true, deploymentMovedAlert: true, + cronFailure: true, }); await deliverInAppNotification({