diff --git a/docs/operations/observability.md b/docs/operations/observability.md index 277e8bbd..fbfb44c9 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -104,6 +104,78 @@ journalctl -u anythingmcp-stuck-report -n 50 `bash deploy/cloud/stuck-users-report.test.sh` runs the report against a throwaway database with every migration applied and a throwaway SMTP server; CI runs it on each PR. +## 5. Connector-failure alerts (webhook, opt-in per organization) + +The three pipelines above tell *you* something is wrong once you go looking. This one pushes a notification the moment a connector starts failing, so you find out without watching the logs. + +An admin configures one webhook per organization at `PUT /api/admin/settings/alert-webhook` (`GET`/`DELETE` and `POST .../test` alongside it; ADMIN role required). Nothing fires until it's configured: self-hosted installs that never set one up get no outbound traffic from this feature. + +```json +{ + "url": "https://example.com/hooks/anythingmcp", + "type": "json", + "enabled": true, + "threshold": 5, + "windowMinutes": 10, + "cooldownMinutes": 30 +} +``` + +`type: "slack"` posts a ready-to-render Slack message (`{ "text": "..." }`) instead of the JSON event below; point it at an Incoming Webhook URL. The signing secret is generated server-side on first save (or when `rotateSecret: true` is sent) and returned exactly once in the response; `GET` never returns it. + +### When it fires + +Every tool call against a connector that ends in `ERROR` or `TIMEOUT` counts toward a window of `windowMinutes` (default 10) that starts at the connector's first failure. Once the count for that connector reaches `threshold` (default 5) within the window, exactly one alert is dispatched; a `cooldownMinutes` (default 30) cooldown then suppresses further alerts for that connector even if it keeps failing, so a sustained outage sends one notification, not one per call. + +### Payload (`type: "json"`) + +```json +{ + "event": "connector.failing", + "organizationId": "org_123", + "connector": { "id": "conn_abc", "name": "SAP OData" }, + "failures": 5, + "windowMinutes": 10, + "lastError": { "tool": "sap_get_orders", "message": "503 Service Unavailable" }, + "firstFailureAt": "2026-10-09T10:00:00.000Z", + "lastFailureAt": "2026-10-09T10:04:12.000Z", + "logsUrl": "https:///logs?connectorId=conn_abc" +} +``` + +`lastError.message` is truncated to 300 characters and never includes tool input or output, the same boundary the audit log itself keeps. + +### Headers and signature + +``` +Content-Type: application/json +X-AnythingMCP-Event: connector.failing +X-AnythingMCP-Timestamp: 1760000000 +X-AnythingMCP-Signature: sha256=.")> +``` + +Verify it (Node): + +```js +const crypto = require('crypto'); + +function verify(secret, timestamp, rawBody, signatureHeader) { + const expected = 'sha256=' + crypto + .createHmac('sha256', secret) + .update(`${timestamp}.${rawBody}`) + .digest('hex'); + if (Math.abs(Date.now() / 1000 - Number(timestamp)) > 300) return false; // reject replays + if (signatureHeader.length !== expected.length) return false; + return crypto.timingSafeEqual(Buffer.from(expected), Buffer.from(signatureHeader)); +} +``` + +Use the *raw* request body, not a re-serialized JSON object, since re-serializing can change key order or whitespace and break the comparison. + +### SSRF + +The webhook URL is validated with the same guard (`SSRF_GUARD`, private/loopback/metadata-address blocking) used for every other outbound call this app makes on a user's behalf, both when it's saved and again on every dispatch, and redirects are not followed. On AnythingMCP Cloud the URL must use `https`, and it ignores both the env (`SSRF_ALLOWED_HOSTS`) and the admin-editable SSRF allowlist, which name hosts only the platform may reach. On a self-hosted instance the operator's allowlist applies, so an internal Mattermost or Slack proxy can receive alerts. + ## Correlating across pipelines The same `req.id` UUID appears in: diff --git a/packages/backend/src/alerts/alerts.controller.spec.ts b/packages/backend/src/alerts/alerts.controller.spec.ts new file mode 100644 index 00000000..6ac47fd7 --- /dev/null +++ b/packages/backend/src/alerts/alerts.controller.spec.ts @@ -0,0 +1,92 @@ +import { ForbiddenException } from '@nestjs/common'; +import { Reflector } from '@nestjs/core'; +import { ROLES_KEY, RolesGuard } from '../auth/roles.guard'; +import { AlertsAdminController } from './alerts.controller'; + +function buildController(overrides: Partial> = {}) { + const alerts = { + getConfig: jest.fn().mockResolvedValue({ configured: false }), + saveConfig: jest.fn().mockResolvedValue({}), + deleteConfig: jest.fn().mockResolvedValue(undefined), + testWebhook: jest.fn().mockResolvedValue({ success: true, status: 200 }), + ...overrides, + }; + const controller = new AlertsAdminController(alerts as any); + return { controller, alerts }; +} + +const req = (organizationId = 'org1') => ({ user: { organizationId, role: 'ADMIN' } }); + +describe('AlertsAdminController', () => { + it('requires the ADMIN role', () => { + const reflector = new Reflector(); + const roles = reflector.get(ROLES_KEY, AlertsAdminController); + expect(roles).toEqual(['ADMIN']); + }); + + // RolesGuard's own pass/fail logic has its own spec (auth/roles.guard.spec.ts); + // this exercises it against this controller's actual metadata, so a non-admin + // caller hitting any of these routes is proven to get a 403, not just asserted + // to be configured for one. + it('rejects a non-admin caller with 403', () => { + const reflector = new Reflector(); + const guard = new RolesGuard(reflector); + const context = { + getHandler: () => AlertsAdminController.prototype.getConfig, + getClass: () => AlertsAdminController, + switchToHttp: () => ({ getRequest: () => ({ user: { organizationId: 'org1', role: 'EDITOR' } }) }), + } as any; + + expect(() => guard.canActivate(context)).toThrow(ForbiddenException); + }); + + it('lets an admin caller through', () => { + const reflector = new Reflector(); + const guard = new RolesGuard(reflector); + const context = { + getHandler: () => AlertsAdminController.prototype.getConfig, + getClass: () => AlertsAdminController, + switchToHttp: () => ({ getRequest: () => ({ user: { organizationId: 'org1', role: 'ADMIN' } }) }), + } as any; + + expect(guard.canActivate(context)).toBe(true); + }); + + it('getConfig delegates to the service with the caller\'s organization', async () => { + const { controller, alerts } = buildController(); + await controller.getConfig(req('org-x')); + expect(alerts.getConfig).toHaveBeenCalledWith('org-x'); + }); + + it('saveConfig returns the secret only when the service generated one', async () => { + const { controller, alerts } = buildController({ + saveConfig: jest.fn().mockResolvedValue({ secret: 'plain-secret' }), + }); + const result = await controller.saveConfig(req('org-x'), { + url: 'https://hooks.example.com/x', + type: 'json', + } as any); + expect(alerts.saveConfig).toHaveBeenCalledWith('org-x', expect.objectContaining({ url: 'https://hooks.example.com/x' })); + expect(result).toEqual({ message: 'Alert webhook saved', secret: 'plain-secret' }); + }); + + it('saveConfig omits the secret field when none was generated', async () => { + const { controller } = buildController({ saveConfig: jest.fn().mockResolvedValue({}) }); + const result = await controller.saveConfig(req(), { url: 'https://hooks.example.com/x', type: 'json' } as any); + expect(result).toEqual({ message: 'Alert webhook saved' }); + }); + + it('deleteConfig delegates to the service', async () => { + const { controller, alerts } = buildController(); + const result = await controller.deleteConfig(req('org-x')); + expect(alerts.deleteConfig).toHaveBeenCalledWith('org-x'); + expect(result).toEqual({ message: 'Alert webhook removed' }); + }); + + it('testWebhook delegates to the service and returns its result', async () => { + const { controller, alerts } = buildController(); + const result = await controller.testWebhook(req('org-x')); + expect(alerts.testWebhook).toHaveBeenCalledWith('org-x'); + expect(result).toEqual({ success: true, status: 200 }); + }); +}); diff --git a/packages/backend/src/alerts/alerts.controller.ts b/packages/backend/src/alerts/alerts.controller.ts new file mode 100644 index 00000000..42909300 --- /dev/null +++ b/packages/backend/src/alerts/alerts.controller.ts @@ -0,0 +1,90 @@ +import { Body, Controller, Delete, Get, Post, Put, Req, UseGuards } from '@nestjs/common'; +import { ApiBearerAuth, ApiOperation, ApiProperty, ApiPropertyOptional, ApiTags } from '@nestjs/swagger'; +import { AuthGuard } from '@nestjs/passport'; +import { Throttle } from '@nestjs/throttler'; +import { IsBoolean, IsIn, IsInt, IsOptional, IsString, IsUrl, Max, MaxLength, Min } from 'class-validator'; +import { Roles, RolesGuard } from '../auth/roles.guard'; +import { AlertsService } from './alerts.service'; + +class SaveAlertWebhookDto { + @ApiProperty({ description: 'Webhook URL to POST the signed alert to.', example: 'https://example.com/hooks/anythingmcp' }) + @IsString() + @MaxLength(2048) + @IsUrl({ protocols: ['http', 'https'], require_protocol: true, require_tld: false }) + url: string; + + @ApiProperty({ description: 'Payload shape: a generic JSON event, or a ready-to-post Slack message.', enum: ['json', 'slack'] }) + @IsIn(['json', 'slack']) + type: 'json' | 'slack'; + + @ApiPropertyOptional({ description: 'Whether the alert is active. Default true.' }) + @IsOptional() + @IsBoolean() + enabled?: boolean; + + @ApiPropertyOptional({ description: 'Consecutive failures within the window before an alert fires. Default 5.' }) + @IsOptional() + @IsInt() + @Min(1) + @Max(1000) + threshold?: number; + + @ApiPropertyOptional({ description: 'Window, in minutes, that failures are counted over; it starts at the first failure. Default 10.' }) + @IsOptional() + @IsInt() + @Min(1) + @Max(1440) + windowMinutes?: number; + + @ApiPropertyOptional({ description: 'Minutes to wait after a dispatch before the same connector can alert again. Default 30.' }) + @IsOptional() + @IsInt() + @Min(1) + @Max(10080) + cooldownMinutes?: number; + + @ApiPropertyOptional({ + description: + 'Generate a new signing secret and return it once. Required to see the secret again after the first save.', + }) + @IsOptional() + @IsBoolean() + rotateSecret?: boolean; +} + +@ApiTags('Alerts') +@ApiBearerAuth() +@UseGuards(AuthGuard('jwt'), RolesGuard) +@Roles('ADMIN') +@Controller('api/admin/settings') +export class AlertsAdminController { + constructor(private readonly alerts: AlertsService) {} + + @Get('alert-webhook') + @ApiOperation({ summary: 'Get the connector-failure alert webhook for the current organization (ADMIN)' }) + async getConfig(@Req() req: any) { + return this.alerts.getConfig(req.user.organizationId); + } + + @Put('alert-webhook') + @ApiOperation({ summary: 'Save the connector-failure alert webhook for the current organization (ADMIN)' }) + async saveConfig(@Req() req: any, @Body() dto: SaveAlertWebhookDto) { + const { secret } = await this.alerts.saveConfig(req.user.organizationId, dto); + return { message: 'Alert webhook saved', ...(secret ? { secret } : {}) }; + } + + @Delete('alert-webhook') + @ApiOperation({ summary: 'Remove the connector-failure alert webhook for the current organization (ADMIN)' }) + async deleteConfig(@Req() req: any) { + await this.alerts.deleteConfig(req.user.organizationId); + return { message: 'Alert webhook removed' }; + } + + @Post('alert-webhook/test') + // It reports the status any public URL answers with: keep it a test, not a probe. + @Throttle({ default: { limit: 5, ttl: 60_000 } }) + @ApiOperation({ summary: 'Send a test alert to the configured webhook (ADMIN)' }) + async testWebhook(@Req() req: any) { + return this.alerts.testWebhook(req.user.organizationId); + } +} diff --git a/packages/backend/src/alerts/alerts.module.ts b/packages/backend/src/alerts/alerts.module.ts new file mode 100644 index 00000000..86349e2f --- /dev/null +++ b/packages/backend/src/alerts/alerts.module.ts @@ -0,0 +1,17 @@ +import { Module } from '@nestjs/common'; +import { SettingsModule } from '../settings/settings.module'; +import { AlertsService } from './alerts.service'; +import { AlertsAdminController } from './alerts.controller'; + +// RedisService and PrismaService come from the @Global() RedisModule and +// PrismaModule, so only SettingsModule (for OrgSettingsService) needs to be +// imported here. Keep this list short: AuditModule imports AlertsModule to +// inject AlertsService into AuditService, so anything this module pulled in +// that itself depended on AuditModule would be a cycle. +@Module({ + imports: [SettingsModule], + controllers: [AlertsAdminController], + providers: [AlertsService], + exports: [AlertsService], +}) +export class AlertsModule {} diff --git a/packages/backend/src/alerts/alerts.service.spec.ts b/packages/backend/src/alerts/alerts.service.spec.ts new file mode 100644 index 00000000..5f955fb4 --- /dev/null +++ b/packages/backend/src/alerts/alerts.service.spec.ts @@ -0,0 +1,273 @@ +import { BadRequestException } from '@nestjs/common'; +import { createHmac } from 'crypto'; +import { AlertsService } from './alerts.service'; +import { outboundRequest } from '../common/outbound-http'; + +jest.mock('../common/outbound-http', () => ({ + outboundRequest: jest.fn(), +})); + +const mockedOutboundRequest = outboundRequest as jest.Mock; + +const ENCRYPTION_KEY = 'a'.repeat(48); + +function makeOrgSettings() { + const store = new Map(); + return { + async get(organizationId: string, key: string) { + return store.get(`${organizationId}:${key}`) ?? null; + }, + async getJson(organizationId: string, key: string) { + const raw = store.get(`${organizationId}:${key}`); + return raw ? JSON.parse(raw) : null; + }, + async set(organizationId: string, key: string, value: string) { + store.set(`${organizationId}:${key}`, value); + }, + async setJson(organizationId: string, key: string, value: unknown) { + store.set(`${organizationId}:${key}`, JSON.stringify(value)); + }, + async delete(organizationId: string, key: string) { + store.delete(`${organizationId}:${key}`); + }, + }; +} + +function makePrisma() { + return { + connector: { findFirst: jest.fn().mockResolvedValue({ id: 'conn1', name: 'SAP' }) }, + mcpTool: { findUnique: jest.fn().mockResolvedValue({ name: 'get_orders' }) }, + }; +} + +/** isConnected: false exercises the in-memory fallback path used throughout these tests. */ +function makeDisconnectedRedis() { + return { + isConnected: false, + incr: jest.fn(), + expire: jest.fn(), + get: jest.fn(), + set: jest.fn(), + del: jest.fn(), + ttl: jest.fn(), + }; +} + +function makeConfigService(cloud = false) { + return { + get: (key: string) => + key === 'ENCRYPTION_KEY' + ? ENCRYPTION_KEY + : key === 'FRONTEND_URL' + ? 'https://app.test' + : key === 'DEPLOYMENT_MODE' + ? (cloud ? 'cloud' : 'self-hosted') + : undefined, + }; +} + +function buildService(overrides: { prisma?: any; orgSettings?: any; redis?: any; cloud?: boolean } = {}) { + const prisma = overrides.prisma ?? makePrisma(); + const orgSettings = overrides.orgSettings ?? makeOrgSettings(); + const redis = overrides.redis ?? makeDisconnectedRedis(); + const service = new AlertsService(prisma as any, orgSettings as any, redis as any, makeConfigService(overrides.cloud) as any); + return { service, prisma, orgSettings, redis }; +} + +describe('AlertsService', () => { + beforeEach(() => { + mockedOutboundRequest.mockReset(); + mockedOutboundRequest.mockResolvedValue({ status: 200 }); + }); + + it('dispatches exactly once at the failure threshold, then the cooldown suppresses the next', async () => { + const { service } = buildService(); + await service.saveConfig('org1', { + url: 'https://hooks.example.com/x', + type: 'json', + threshold: 3, + windowMinutes: 10, + cooldownMinutes: 30, + }); + + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 'tool1', error: 'boom' }); + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 'tool1', error: 'boom' }); + expect(mockedOutboundRequest).not.toHaveBeenCalled(); + + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 'tool1', error: 'boom' }); + expect(mockedOutboundRequest).toHaveBeenCalledTimes(1); + + // Past the threshold, still within the cooldown: no second dispatch. + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 'tool1', error: 'boom' }); + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 'tool1', error: 'boom' }); + expect(mockedOutboundRequest).toHaveBeenCalledTimes(1); + }); + + it('keeps per-organization counters isolated even for the same connectorId', async () => { + const { service } = buildService(); + await service.saveConfig('org1', { url: 'https://hooks.example.com/a', type: 'json', threshold: 2 }); + await service.saveConfig('org2', { url: 'https://hooks.example.com/b', type: 'json', threshold: 2 }); + + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 't', error: 'e' }); + await service.recordFailure({ organizationId: 'org2', connectorId: 'conn1', toolId: 't', error: 'e' }); + expect(mockedOutboundRequest).not.toHaveBeenCalled(); + + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 't', error: 'e' }); + expect(mockedOutboundRequest).toHaveBeenCalledTimes(1); + + await service.recordFailure({ organizationId: 'org2', connectorId: 'conn1', toolId: 't', error: 'e' }); + expect(mockedOutboundRequest).toHaveBeenCalledTimes(2); + }); + + it('never throws out of recordFailure when dispatch fails', async () => { + const { service } = buildService(); + await service.saveConfig('org1', { url: 'https://hooks.example.com/x', type: 'json', threshold: 1 }); + mockedOutboundRequest.mockRejectedValue(new Error('network down')); + + await expect( + service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 't', error: 'e' }), + ).resolves.toBeUndefined(); + }); + + it('never throws out of recordFailure when reading the config itself fails', async () => { + const orgSettings = { getJson: jest.fn().mockRejectedValue(new Error('db down')) }; + const { service } = buildService({ orgSettings }); + + await expect( + service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 't', error: 'e' }), + ).resolves.toBeUndefined(); + }); + + it('does nothing when no webhook is configured for the organization', async () => { + const { service } = buildService(); + await service.recordFailure({ organizationId: 'org-without-config', connectorId: 'conn1', toolId: 't', error: 'e' }); + expect(mockedOutboundRequest).not.toHaveBeenCalled(); + }); + + it('signs the dispatched payload with the generated secret', async () => { + const { service } = buildService(); + const { secret } = await service.saveConfig('org1', { + url: 'https://hooks.example.com/x', + type: 'json', + threshold: 1, + }); + expect(secret).toBeDefined(); + + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 'tool1', error: 'boom' }); + + const call = mockedOutboundRequest.mock.calls[0][0]; + const timestamp = call.headers['X-AnythingMCP-Timestamp']; + const expectedSignature = `sha256=${createHmac('sha256', secret!) + .update(`${timestamp}.${call.data}`) + .digest('hex')}`; + expect(call.headers['X-AnythingMCP-Signature']).toBe(expectedSignature); + expect(call.headers['X-AnythingMCP-Event']).toBe('connector.failing'); + }); + + it('getConfig never returns the secret', async () => { + const { service } = buildService(); + await service.saveConfig('org1', { url: 'https://hooks.example.com/x', type: 'json' }); + const config = await service.getConfig('org1'); + expect(config).not.toHaveProperty('secretEnc'); + expect(JSON.stringify(config)).not.toContain('secretEnc'); + }); + + it('only returns a secret from saveConfig on create or when rotateSecret is set', async () => { + const { service } = buildService(); + const first = await service.saveConfig('org1', { url: 'https://hooks.example.com/x', type: 'json' }); + expect(first.secret).toBeDefined(); + + const second = await service.saveConfig('org1', { url: 'https://hooks.example.com/x', type: 'json', threshold: 10 }); + expect(second.secret).toBeUndefined(); + + const rotated = await service.saveConfig('org1', { url: 'https://hooks.example.com/x', type: 'json', rotateSecret: true }); + expect(rotated.secret).toBeDefined(); + expect(rotated.secret).not.toBe(first.secret); + }); + + it('reads the webhook config once a minute per organization, not on every failure', async () => { + const orgSettings = makeOrgSettings(); + const getJson = jest.spyOn(orgSettings, 'getJson'); + const { service } = buildService({ orgSettings }); + for (let i = 0; i < 5; i++) { + await service.recordFailure({ organizationId: 'org-none', connectorId: 'conn1', toolId: 't', error: 'e' }); + } + expect(getJson).toHaveBeenCalledTimes(1); + + // Saving a webhook takes effect at once, not after the cache expires. + await service.saveConfig('org-none', { url: 'https://hooks.example.com/x', type: 'json', threshold: 1 }); + await service.recordFailure({ organizationId: 'org-none', connectorId: 'conn1', toolId: 't', error: 'e' }); + expect(mockedOutboundRequest).toHaveBeenCalledTimes(1); + }); + + it('escapes the connector name and the vendor error in a Slack message', async () => { + const prisma = makePrisma(); + prisma.connector.findFirst.mockResolvedValue({ id: 'conn1', name: 'Shop ' }); + const { service } = buildService({ prisma }); + await service.saveConfig('org1', { url: 'https://hooks.slack.com/x', type: 'slack', threshold: 1 }); + await service.recordFailure({ + organizationId: 'org1', + connectorId: 'conn1', + toolId: 't', + error: 'see ', + }); + const text = JSON.parse(mockedOutboundRequest.mock.calls[0][0].data).text as string; + expect(text).toContain('Shop <!channel>'); + expect(text).toContain('<https://evil.example|Reset password>'); + expect(text).not.toContain(''); + }); + + it('only looks up the connector inside the organization that failed', async () => { + const { service, prisma } = buildService(); + await service.saveConfig('org1', { url: 'https://hooks.example.com/x', type: 'json', threshold: 1 }); + await service.recordFailure({ organizationId: 'org1', connectorId: 'conn1', toolId: 't', error: 'e' }); + expect(prisma.connector.findFirst).toHaveBeenCalledWith( + expect.objectContaining({ where: { id: 'conn1', organizationId: 'org1' } }), + ); + }); + + describe('SSRF: the webhook target must not inherit the connector allowlist', () => { + const originalGuard = process.env.SSRF_GUARD; + const originalAllowed = process.env.SSRF_ALLOWED_HOSTS; + + beforeEach(() => { + process.env.SSRF_GUARD = 'enabled'; + }); + afterEach(() => { + if (originalGuard === undefined) delete process.env.SSRF_GUARD; + else process.env.SSRF_GUARD = originalGuard; + if (originalAllowed === undefined) delete process.env.SSRF_ALLOWED_HOSTS; + else process.env.SSRF_ALLOWED_HOSTS = originalAllowed; + }); + + it('on Cloud, rejects a private IP even when it is on the SSRF allowlist', async () => { + process.env.SSRF_ALLOWED_HOSTS = '127.0.0.1'; + const { service } = buildService({ cloud: true }); + await expect( + service.saveConfig('org1', { url: 'https://127.0.0.1/hook', type: 'json' }), + ).rejects.toThrow(/not a public IP/); + }); + + it('on Cloud, requires https', async () => { + const { service } = buildService({ cloud: true }); + await expect( + service.saveConfig('org1', { url: 'http://hooks.example.com/hook', type: 'json' }), + ).rejects.toThrow(/must use https/); + }); + + it('self-hosted, keeps the operator allowlist so an internal chat server can receive alerts', async () => { + process.env.SSRF_ALLOWED_HOSTS = '127.0.0.1'; + const { service } = buildService(); + await expect( + service.saveConfig('org1', { url: 'http://127.0.0.1/hook', type: 'json' }), + ).resolves.toBeDefined(); + }); + + it('rejects the cloud metadata address', async () => { + const { service } = buildService(); + await expect( + service.saveConfig('org1', { url: 'http://169.254.169.254/hook', type: 'json' }), + ).rejects.toBeInstanceOf(BadRequestException); + }); + }); +}); diff --git a/packages/backend/src/alerts/alerts.service.ts b/packages/backend/src/alerts/alerts.service.ts new file mode 100644 index 00000000..79e7e859 --- /dev/null +++ b/packages/backend/src/alerts/alerts.service.ts @@ -0,0 +1,438 @@ +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { createHmac, randomBytes } from 'crypto'; +import { PrismaService } from '../common/prisma.service'; +import { RedisService } from '../common/redis.service'; +import { OrgSettingsService } from '../settings/org-settings.service'; +import { encrypt, decrypt } from '../common/crypto/encryption.util'; +import { assertSafeOutboundUrl, SsrfBlockedError } from '../common/ssrf.util'; +import { outboundRequest } from '../common/outbound-http'; +import { getRequiredSecret } from '../common/secrets.util'; +import { + ALERT_WEBHOOK_SETTINGS_KEY, + AlertWebhookConfig, + AlertWebhookPublicConfig, + DEFAULT_COOLDOWN_MINUTES, + DEFAULT_THRESHOLD, + DEFAULT_WINDOW_MINUTES, +} from './alerts.types'; + +export interface SaveAlertWebhookDto { + url: string; + type: 'json' | 'slack'; + enabled?: boolean; + threshold?: number; + windowMinutes?: number; + cooldownMinutes?: number; + rotateSecret?: boolean; +} + +export interface RecordFailureInput { + organizationId: string; + connectorId: string; + toolId: string; + error?: string; +} + +export interface WebhookDispatchResult { + success: boolean; + status?: number; + message?: string; +} + +const MAX_ERROR_CHARS = 300; +/** HTTP request budget for a single webhook attempt. */ +const DISPATCH_TIMEOUT_MS = 5000; +/** One retry on top of the first attempt — enough to ride out a blip without piling up requests. */ +const DISPATCH_ATTEMPTS = 2; +/** + * How long a webhook config (or its absence) is reused for failure counting. + * recordFailure runs on every failed tool call of every organization, nearly + * all of which have no webhook: without this, each failure cost a DB read. + */ +const CONFIG_CACHE_MS = 60_000; +const CONFIG_CACHE_MAX = 10_000; + +/** + * Detects a connector that keeps failing and dispatches a signed webhook + * (or Slack message) so an admin finds out without having to watch the logs. + * + * Counting lives in Redis (shared across backend instances) with an + * in-memory fallback for self-hosted installs that haven't configured it — + * see `incrWithExpiry`/`isCooldownActive`/`setCooldown` below. + */ +@Injectable() +export class AlertsService { + private readonly logger = new Logger(AlertsService.name); + private readonly encryptionKey: string; + + // In-memory fallback, used only while RedisService.isConnected is false. + // Not shared across processes — fine for a single self-hosted instance, + // and still correct (just not synchronized) across a horizontally scaled + // Cloud deployment, where Redis is expected to be configured anyway. + private readonly memCounters = new Map(); + private readonly memCooldowns = new Map(); + private readonly memFirstFailure = new Map(); + private readonly configCache = new Map(); + + constructor( + private readonly prisma: PrismaService, + private readonly orgSettings: OrgSettingsService, + private readonly redis: RedisService, + private readonly configService: ConfigService, + ) { + this.encryptionKey = getRequiredSecret( + 'ENCRYPTION_KEY', + this.configService.get('ENCRYPTION_KEY'), + ); + } + + // ── Admin-facing config ──────────────────────────────────────────────── + + async getConfig(organizationId: string): Promise { + const config = await this.orgSettings.getJson( + organizationId, + ALERT_WEBHOOK_SETTINGS_KEY, + ); + if (!config?.url) return { configured: false }; + const { secretEnc: _secretEnc, ...publicConfig } = config; + return publicConfig; + } + + /** + * Save the webhook config. Returns the plaintext secret once, only when a + * new one was generated (first save, or `rotateSecret: true`) — the same + * "shown once" pattern as an API key, since it is never stored in plain + * text and `getConfig` never returns it again. + */ + async saveConfig( + organizationId: string, + dto: SaveAlertWebhookDto, + ): Promise<{ secret?: string }> { + this.assertHttpsOnCloud(dto.url); + try { + // On Cloud the allowlists name hosts that only the platform may reach, + // so a webhook URL supplied here must not inherit them (see + // VetHostOptions in ssrf.util.ts). A self-hosted operator's allowlist is + // their own, which is how an internal Mattermost or Slack proxy works. + await assertSafeOutboundUrl(dto.url, process.env, { skipAllowlists: this.isCloud() }); + } catch (err) { + if (err instanceof SsrfBlockedError) { + throw new BadRequestException(err.message); + } + throw err; + } + + const existing = await this.orgSettings.getJson( + organizationId, + ALERT_WEBHOOK_SETTINGS_KEY, + ); + + let secretEnc = existing?.secretEnc; + let plaintextSecret: string | undefined; + if (!secretEnc || dto.rotateSecret) { + plaintextSecret = randomBytes(32).toString('hex'); + secretEnc = encrypt(plaintextSecret, this.encryptionKey, organizationId); + } + + const config: AlertWebhookConfig = { + url: dto.url, + type: dto.type, + enabled: dto.enabled ?? true, + threshold: dto.threshold || DEFAULT_THRESHOLD, + windowMinutes: dto.windowMinutes || DEFAULT_WINDOW_MINUTES, + cooldownMinutes: dto.cooldownMinutes || DEFAULT_COOLDOWN_MINUTES, + secretEnc, + }; + await this.orgSettings.setJson(organizationId, ALERT_WEBHOOK_SETTINGS_KEY, config); + this.configCache.delete(organizationId); + + return plaintextSecret ? { secret: plaintextSecret } : {}; + } + + async deleteConfig(organizationId: string): Promise { + await this.orgSettings.delete(organizationId, ALERT_WEBHOOK_SETTINGS_KEY); + this.configCache.delete(organizationId); + } + + async testWebhook(organizationId: string): Promise { + const config = await this.orgSettings.getJson( + organizationId, + ALERT_WEBHOOK_SETTINGS_KEY, + ); + if (!config?.url) { + throw new BadRequestException('No alert webhook is configured for this organization'); + } + const now = new Date().toISOString(); + return this.dispatch(organizationId, config, { + event: 'connector.test', + organizationId, + connector: { id: 'test', name: 'Test connector' }, + failures: 1, + windowMinutes: config.windowMinutes, + lastError: { tool: 'test_tool', message: 'This is a test alert from AnythingMCP.' }, + firstFailureAt: now, + lastFailureAt: now, + logsUrl: this.buildLogsUrl(), + }); + } + + // ── Failure detection (called from AuditService.logInvocation) ───────── + + /** + * Fire-and-forget: called for every non-SUCCESS invocation. Never throws — + * any failure here must not affect the MCP tool-call path that triggered + * it, so every exit point beyond the initial argument check is wrapped. + */ + async recordFailure(input: RecordFailureInput): Promise { + try { + const config = await this.cachedConfig(input.organizationId); + if (!config?.enabled || !config.url) return; + + const threshold = config.threshold || DEFAULT_THRESHOLD; + const windowSeconds = (config.windowMinutes || DEFAULT_WINDOW_MINUTES) * 60; + const cooldownSeconds = (config.cooldownMinutes || DEFAULT_COOLDOWN_MINUTES) * 60; + const scope = `${input.organizationId}:${input.connectorId}`; + const failKey = `alerts:fail:${scope}`; + const cooldownKey = `alerts:cooldown:${scope}`; + const firstFailureKey = `alerts:firstfail:${scope}`; + + const now = new Date(); + const count = await this.incrWithExpiry(failKey, windowSeconds, threshold); + if (count === 1) { + await this.setKeyWithExpiry(firstFailureKey, now.toISOString(), windowSeconds); + } + + // Only the call that crosses the line dispatches. A later failure in + // the same window pushes the count past `threshold` without ever + // landing on it again, so it is naturally skipped until the window's + // TTL lapses and a fresh cycle starts. + if (count !== threshold) return; + if (await this.isCooldownActive(cooldownKey)) return; + // INCR is atomic, so only one call sees exactly `threshold`. The + // cooldown is set before sending so a slow dispatch cannot overlap the + // next window's alert. + await this.setCooldown(cooldownKey, cooldownSeconds); + + const [connector, tool] = await Promise.all([ + this.prisma.connector.findFirst({ + where: { id: input.connectorId, organizationId: input.organizationId }, + select: { id: true, name: true }, + }), + this.prisma.mcpTool.findUnique({ + where: { id: input.toolId }, + select: { name: true }, + }), + ]); + if (!connector) return; + + const firstFailureAt = (await this.getKey(firstFailureKey)) ?? now.toISOString(); + + const result = await this.dispatch(input.organizationId, config, { + event: 'connector.failing', + organizationId: input.organizationId, + connector: { id: connector.id, name: connector.name }, + failures: count, + windowMinutes: config.windowMinutes || DEFAULT_WINDOW_MINUTES, + lastError: { + tool: tool?.name ?? input.toolId, + message: truncate(input.error, MAX_ERROR_CHARS), + }, + firstFailureAt, + lastFailureAt: now.toISOString(), + logsUrl: this.buildLogsUrl(connector.id), + }); + if (!result.success) { + // Never log the URL: a Slack webhook URL is itself a credential. + this.logger.warn( + `Connector-failure alert for org ${input.organizationId} was not delivered: ${result.status ?? ''} ${result.message ?? ''}`.trim(), + ); + } + } catch (err: any) { + this.logger.warn(`Failed to process connector-failure alert: ${err?.message || err}`); + } + } + + // ── Dispatch ───────────────────────────────────────────────────────────── + + private async dispatch( + organizationId: string, + config: AlertWebhookConfig, + payload: Record, + ): Promise { + let secret: string; + try { + secret = decrypt(config.secretEnc, this.encryptionKey, organizationId); + } catch (err: any) { + this.logger.warn(`Could not decrypt alert webhook secret for org ${organizationId}: ${err.message}`); + return { success: false, message: 'Stored webhook secret could not be decrypted' }; + } + + const body = + config.type === 'slack' + ? JSON.stringify({ text: slackText(payload) }) + : JSON.stringify(payload); + const timestamp = Math.floor(Date.now() / 1000).toString(); + const signature = createHmac('sha256', secret) + .update(`${timestamp}.${body}`) + .digest('hex'); + const headers = { + 'Content-Type': 'application/json', + 'X-AnythingMCP-Event': String(payload.event), + 'X-AnythingMCP-Timestamp': timestamp, + 'X-AnythingMCP-Signature': `sha256=${signature}`, + }; + + try { + this.assertHttpsOnCloud(config.url); + } catch (err: any) { + return { success: false, message: err.message }; + } + const skipAllowlists = this.isCloud(); + for (let attempt = 1; attempt <= DISPATCH_ATTEMPTS; attempt++) { + try { + // The outbound adapter re-checks every hop at send time (policy or + // DNS can change after the save), with the same allowlist rule as + // saveConfig. + const res = await outboundRequest( + { + url: config.url, + method: 'POST', + data: body, + headers, + timeout: DISPATCH_TIMEOUT_MS, + validateStatus: () => true, + }, + { skipAllowlists, maxRedirects: 0 }, + ); + if (res.status >= 200 && res.status < 300) { + return { success: true, status: res.status }; + } + if (attempt === DISPATCH_ATTEMPTS || res.status < 500) { + return { success: false, status: res.status, message: `Webhook responded with ${res.status}` }; + } + } catch (err: any) { + if (err instanceof SsrfBlockedError || attempt === DISPATCH_ATTEMPTS) { + return { success: false, message: err.message }; + } + } + } + return { success: false, message: 'Webhook dispatch failed' }; + } + + private isCloud(): boolean { + return this.configService.get('DEPLOYMENT_MODE') === 'cloud'; + } + + /** Signed alerts carry error text: on Cloud they only go out over TLS. */ + private assertHttpsOnCloud(url: string): void { + let protocol: string; + try { + protocol = new URL(url).protocol; + } catch { + throw new BadRequestException('The webhook URL is not a valid URL'); + } + if (this.isCloud() && protocol !== 'https:') { + throw new BadRequestException('On AnythingMCP Cloud the webhook URL must use https'); + } + } + + private async cachedConfig(organizationId: string): Promise { + const now = Date.now(); + const hit = this.configCache.get(organizationId); + if (hit && hit.expiresAt > now) return hit.config; + const config = + (await this.orgSettings.getJson(organizationId, ALERT_WEBHOOK_SETTINGS_KEY)) ?? null; + if (this.configCache.size >= CONFIG_CACHE_MAX) this.configCache.clear(); + this.configCache.set(organizationId, { config, expiresAt: now + CONFIG_CACHE_MS }); + return config; + } + + private buildLogsUrl(connectorId?: string): string { + const base = + (this.configService.get('FRONTEND_URL') || 'http://localhost:3000').replace(/\/+$/, ''); + return connectorId ? `${base}/logs?connectorId=${connectorId}` : `${base}/logs`; + } + + // ── Counters: Redis when connected, in-memory fallback otherwise ─────── + + private async incrWithExpiry(key: string, ttlSeconds: number, threshold: number): Promise { + if (this.redis.isConnected) { + const count = await this.redis.incr(key); + if (count === 1) { + await this.redis.expire(key, ttlSeconds); + } else if (count > threshold && count % 20 === 0 && (await this.redis.ttl(key)) === -1) { + // INCR and EXPIRE are two round trips: if the EXPIRE was lost, the + // key would count past the threshold forever and never alert again. + await this.redis.expire(key, ttlSeconds); + } + return count; + } + if (this.memCounters.size > 1000) { + const now = Date.now(); + for (const [k, v] of this.memCounters) if (v.expiresAt <= now) this.memCounters.delete(k); + } + const now = Date.now(); + const entry = this.memCounters.get(key); + if (!entry || entry.expiresAt <= now) { + this.memCounters.set(key, { count: 1, expiresAt: now + ttlSeconds * 1000 }); + return 1; + } + entry.count += 1; + return entry.count; + } + + private async setKeyWithExpiry(key: string, value: string, ttlSeconds: number): Promise { + if (this.redis.isConnected) { + await this.redis.set(key, value, ttlSeconds); + return; + } + this.memFirstFailure.set(key, { value, expiresAt: Date.now() + ttlSeconds * 1000 }); + } + + private async getKey(key: string): Promise { + if (this.redis.isConnected) return this.redis.get(key); + const entry = this.memFirstFailure.get(key); + if (!entry || entry.expiresAt <= Date.now()) return null; + return entry.value; + } + + private async isCooldownActive(key: string): Promise { + if (this.redis.isConnected) return (await this.redis.get(key)) !== null; + const expiresAt = this.memCooldowns.get(key); + return !!expiresAt && expiresAt > Date.now(); + } + + private async setCooldown(key: string, ttlSeconds: number): Promise { + if (this.redis.isConnected) { + await this.redis.set(key, '1', ttlSeconds); + return; + } + this.memCooldowns.set(key, Date.now() + ttlSeconds * 1000); + } +} + +function truncate(text: string | undefined, maxChars: number): string { + if (!text) return ''; + return text.length > maxChars ? `${text.slice(0, maxChars)}…` : text; +} + +function slackText(payload: Record): string { + if (payload.event === 'connector.test') { + return ':white_check_mark: AnythingMCP test alert: your webhook is configured correctly.'; + } + const connector = payload.connector as { name?: string } | undefined; + const lastError = payload.lastError as { tool?: string; message?: string } | undefined; + // The connector name and the vendor's error text are not ours: escaped, so + // an upstream body cannot ping or disguise a link in the org's Slack. + return ( + `:rotating_light: *${slackEscape(connector?.name ?? 'A connector')}* has failed ${payload.failures} times ` + + `in the last ${payload.windowMinutes} minutes.\n` + + `Last error (${slackEscape(lastError?.tool ?? 'unknown tool')}): ${slackEscape(lastError?.message ?? 'n/a')}\n` + + `<${payload.logsUrl}|View logs>` + ); +} + +function slackEscape(text: string): string { + return text.replace(/&/g, '&').replace(//g, '>'); +} diff --git a/packages/backend/src/alerts/alerts.types.ts b/packages/backend/src/alerts/alerts.types.ts new file mode 100644 index 00000000..6c6a9ab1 --- /dev/null +++ b/packages/backend/src/alerts/alerts.types.ts @@ -0,0 +1,21 @@ +export type AlertWebhookType = 'json' | 'slack'; + +/** Stored at OrgSettings key `alert_webhook`. `secretEnc` is never sent to a client. */ +export interface AlertWebhookConfig { + url: string; + type: AlertWebhookType; + enabled: boolean; + threshold: number; + windowMinutes: number; + cooldownMinutes: number; + secretEnc: string; +} + +/** `GET`/`PUT` response shape: everything except the secret. */ +export type AlertWebhookPublicConfig = Omit; + +export const DEFAULT_THRESHOLD = 5; +export const DEFAULT_WINDOW_MINUTES = 10; +export const DEFAULT_COOLDOWN_MINUTES = 30; + +export const ALERT_WEBHOOK_SETTINGS_KEY = 'alert_webhook'; diff --git a/packages/backend/src/audit/audit.module.ts b/packages/backend/src/audit/audit.module.ts index 13f1bb0e..0f5565e2 100644 --- a/packages/backend/src/audit/audit.module.ts +++ b/packages/backend/src/audit/audit.module.ts @@ -6,9 +6,11 @@ import { SecurityEventService } from './security-event.service'; import { ProductEventService } from './product-event.service'; import { ProductEventController } from './product-event.controller'; import { SignupAttributionController } from './signup-attribution.controller'; +import { AlertsModule } from '../alerts/alerts.module'; @Global() @Module({ + imports: [AlertsModule], controllers: [AuditController, ProductEventController, SignupAttributionController], providers: [AuditService, SecurityEventService, ProductEventService, AdsActivationService], exports: [AuditService, SecurityEventService, ProductEventService], diff --git a/packages/backend/src/audit/audit.service.spec.ts b/packages/backend/src/audit/audit.service.spec.ts index 6d174150..1cd295e2 100644 --- a/packages/backend/src/audit/audit.service.spec.ts +++ b/packages/backend/src/audit/audit.service.spec.ts @@ -171,6 +171,73 @@ describe('AuditService', () => { }); }); + describe('connector-failure alerts', () => { + it('notifies on every failing call, even ones the repeat-dedup check collapses into an existing row', async () => { + const alerts = { recordFailure: jest.fn().mockResolvedValue(undefined) }; + const svc = new AuditService(mockPrisma, undefined, alerts as any); + // Dedup only engages once create() returns a row id to count repeats + // against — same setup as the 'repeated failures' block above. + let n = 0; + mockPrisma.toolInvocation.create.mockImplementation(async () => ({ id: `row-${++n}` })); + + const fail = () => + svc.logInvocation({ + toolId: 'tool-1', + connectorId: 'conn-1', + organizationId: 'org-1', + input: {}, + status: 'ERROR', + error: '429 Too Many Requests', + }); + + await fail(); + await fail(); + await fail(); + // Only one row was written (see 'repeated failures' above), but all + // three attempts are real failures and must all reach the alert hook. + expect(mockPrisma.toolInvocation.create).toHaveBeenCalledTimes(1); + expect(alerts.recordFailure).toHaveBeenCalledTimes(3); + expect(alerts.recordFailure).toHaveBeenCalledWith({ + organizationId: 'org-1', + connectorId: 'conn-1', + toolId: 'tool-1', + error: '429 Too Many Requests', + }); + + await svc.onModuleDestroy(); + }); + + it('does not notify on success, or when organizationId/connectorId is missing', async () => { + const alerts = { recordFailure: jest.fn().mockResolvedValue(undefined) }; + const svc = new AuditService(mockPrisma, undefined, alerts as any); + + await svc.logInvocation({ toolId: 't', organizationId: 'org-1', connectorId: 'conn-1', input: {}, status: 'SUCCESS' }); + await svc.logInvocation({ toolId: 't', connectorId: 'conn-1', input: {}, status: 'ERROR', error: 'e' }); // no org + await svc.logInvocation({ toolId: 't', organizationId: 'org-1', input: {}, status: 'ERROR', error: 'e' }); // no connector + + expect(alerts.recordFailure).not.toHaveBeenCalled(); + await svc.onModuleDestroy(); + }); + + it('never lets a failing alert dispatch affect logInvocation', async () => { + const alerts = { recordFailure: jest.fn().mockRejectedValue(new Error('webhook down')) }; + const svc = new AuditService(mockPrisma, undefined, alerts as any); + + await expect( + svc.logInvocation({ + toolId: 't', + organizationId: 'org-1', + connectorId: 'conn-1', + input: {}, + status: 'ERROR', + error: 'e', + }), + ).resolves.toBeUndefined(); + + await svc.onModuleDestroy(); + }); + }); + describe('payload budget', () => { it('keeps only a short excerpt once an organisation has logged its hourly budget', async () => { const big = { rows: 'x'.repeat(20_000) }; diff --git a/packages/backend/src/audit/audit.service.ts b/packages/backend/src/audit/audit.service.ts index 7e5a5cc6..fdf479a2 100644 --- a/packages/backend/src/audit/audit.service.ts +++ b/packages/backend/src/audit/audit.service.ts @@ -28,6 +28,7 @@ const REPEAT_KEYS_MAX = 10_000; import { PrismaService } from '../common/prisma.service'; import { InvocationStatus, Prisma } from '../generated/prisma/client'; import { AdsActivationService } from './ads-activation.service'; +import { AlertsService } from '../alerts/alerts.service'; @Injectable() export class AuditService implements OnModuleDestroy { @@ -41,6 +42,7 @@ export class AuditService implements OnModuleDestroy { constructor( private readonly prisma: PrismaService, @Optional() private readonly adsActivation?: AdsActivationService, + @Optional() private readonly alerts?: AlertsService, ) { this.flushTimer = setInterval(() => void this.flushRepeats(false), 30_000); this.flushTimer.unref?.(); @@ -79,6 +81,24 @@ export class AuditService implements OnModuleDestroy { error?: string; clientInfo?: string; }): Promise { + // Every failing call, not just the ones that end up as a fresh row: the + // repeat-dedup check just below collapses an identical, fast-repeating + // failure into a counter on the first row instead of writing it again, + // so a hook placed after that check (or after the DB write) would miss + // most of a real outage. Fire-and-forget; recordFailure never throws, + // and this catch is defense in depth so it can never affect the call + // that triggered it either way. + if (data.status !== 'SUCCESS' && data.organizationId && data.connectorId) { + this.alerts + ?.recordFailure({ + organizationId: data.organizationId, + connectorId: data.connectorId, + toolId: data.toolId, + error: data.error, + }) + .catch((err: any) => this.logger.warn(`Alert dispatch failed: ${err?.message || err}`)); + } + const repeatKey = data.status === 'SUCCESS' ? null : repeatSignature(data); if (repeatKey && this.countRepeat(repeatKey)) return; diff --git a/packages/backend/src/common/outbound-http.ts b/packages/backend/src/common/outbound-http.ts index 28841144..f6603ec7 100644 --- a/packages/backend/src/common/outbound-http.ts +++ b/packages/backend/src/common/outbound-http.ts @@ -80,12 +80,25 @@ export interface OutboundRequestOptions { * another origin that would send the body again (307/308) is refused. */ credentialsInBody?: boolean; + /** + * Ignore the env/DB SSRF allowlists for this call and every redirect hop + * (see {@link VetHostOptions} in `ssrf.util.ts`). Use this when the target + * is a URL a workspace admin supplies for something other than a + * connector — an alert webhook, for instance — so it never inherits trust + * an admin granted a connector for an unrelated reason. + */ + skipAllowlists?: boolean; } let sharedAgents: { httpAgent: http.Agent; httpsAgent: https.Agent } | null = null; +let sharedStrictAgents: { httpAgent: http.Agent; httpsAgent: https.Agent } | null = null; -function guardedAgents() { +function guardedAgents(options?: OutboundRequestOptions) { // The lookup reads process.env on every call, so one pair serves all. + if (options?.skipAllowlists) { + sharedStrictAgents ??= createSsrfGuardedAgents(process.env, { skipAllowlists: true }); + return sharedStrictAgents; + } sharedAgents ??= createSsrfGuardedAgents(); return sharedAgents; } @@ -100,7 +113,7 @@ let sharedAdapter: AxiosAdapter | null = null; export function outboundAxiosOptions( options?: OutboundRequestOptions, ): Pick { - const { httpAgent, httpsAgent } = guardedAgents(); + const { httpAgent, httpsAgent } = guardedAgents(options); const adapter = options ? createOutboundAdapter(options) : (sharedAdapter ??= createOutboundAdapter({})); @@ -143,7 +156,7 @@ function createOutboundAdapter(options: OutboundRequestOptions): AxiosAdapter { }; for (let redirects = 0; ; redirects++) { - await assertSafeOutboundUrl(url); + await assertSafeOutboundUrl(url, process.env, { skipAllowlists: options.skipAllowlists }); if (deadline) { const left = deadline - Date.now(); if (left <= 0) { diff --git a/packages/backend/src/common/ssrf.util.ts b/packages/backend/src/common/ssrf.util.ts index 0294a8d4..4533e7f5 100644 --- a/packages/backend/src/common/ssrf.util.ts +++ b/packages/backend/src/common/ssrf.util.ts @@ -40,6 +40,23 @@ interface SsrfPolicy { allowedHosts: string[]; } +/** + * `skipAllowlists: true` ignores both the env (`SSRF_ALLOWED_HOSTS`) and + * DB-backed (`setDbAllowedHostsProvider`) allowlists, falling straight + * through to the literal-IP / loopback / DNS-resolution checks. + * + * Those allowlists exist so an admin can let a *connector* reach a specific + * internal host for a reason that has nothing to do with any other feature. + * A caller that sends a user-supplied URL to a *different* destination (e.g. + * an outbound alert webhook) must not inherit that trust: a workspace could + * otherwise point its webhook at a host the admin allowlisted for something + * else entirely. Use this for any outbound call whose target the allowlist + * was never meant to vouch for. + */ +export interface VetHostOptions { + skipAllowlists?: boolean; +} + function readPolicy(env: NodeJS.ProcessEnv = process.env): SsrfPolicy { // The guard performs real DNS resolution and would make most unit tests // depend on the network. Disable it under jest unless the test explicitly @@ -185,6 +202,7 @@ function ipv6Groups(ip: string): number[] | null { export async function assertSafeOutboundUrl( url: string, env: NodeJS.ProcessEnv = process.env, + opts: VetHostOptions = {}, ): Promise { const policy = readPolicy(env); if (!policy.enabled) return; @@ -203,7 +221,7 @@ export async function assertSafeOutboundUrl( } // An IPv6 literal comes back bracketed ("[::1]"); the checks want the address. - await assertSafeOutboundHost(parsed.hostname.replace(/^\[|\]$/g, ''), env); + await assertSafeOutboundHost(parsed.hostname.replace(/^\[|\]$/g, ''), env, opts); } /** @@ -218,10 +236,11 @@ export async function assertSafeOutboundUrl( export async function assertSafeOutboundHost( hostname: string, env: NodeJS.ProcessEnv = process.env, + opts: VetHostOptions = {}, ): Promise { const policy = readPolicy(env); if (!policy.enabled) return; - await vetHost(hostname, policy); + await vetHost(hostname, policy, opts); } /** @@ -232,23 +251,26 @@ export async function assertSafeOutboundHost( async function vetHost( hostname: string, policy: SsrfPolicy, + opts: VetHostOptions = {}, ): Promise { if (!hostname) { throw new SsrfBlockedError('SSRF guard: empty hostname'); } - // Env-driven allowlist (synchronous). - if (hostMatchesAllowlist(hostname, policy.allowedHosts)) return null; + if (!opts.skipAllowlists) { + // Env-driven allowlist (synchronous). + if (hostMatchesAllowlist(hostname, policy.allowedHosts)) return null; - // DB-driven allowlist (admin-configured, async). The provider caches - // internally so this is effectively a Map lookup after the first call. - if (dbAllowedHostsProvider) { - try { - const dbHosts = await dbAllowedHostsProvider(); - if (hostMatchesAllowlist(hostname, dbHosts)) return null; - } catch { - // Provider failure: fall through to IP-based checks rather than - // hard-failing every outbound call. + // DB-driven allowlist (admin-configured, async). The provider caches + // internally so this is effectively a Map lookup after the first call. + if (dbAllowedHostsProvider) { + try { + const dbHosts = await dbAllowedHostsProvider(); + if (hostMatchesAllowlist(hostname, dbHosts)) return null; + } catch { + // Provider failure: fall through to IP-based checks rather than + // hard-failing every outbound call. + } } } @@ -319,6 +341,7 @@ async function vetHost( */ export function ssrfGuardedLookup( env: NodeJS.ProcessEnv = process.env, + opts: VetHostOptions = {}, ): LookupFunction { return (hostname, options, callback) => { const policy = readPolicy(env); @@ -326,7 +349,7 @@ export function ssrfGuardedLookup( // when it is in use the proxy resolves the target, not us. const vetted = policy.enabled && !envProxyHosts(env).has(hostname.toLowerCase()) - ? vetHost(hostname, policy) + ? vetHost(hostname, policy, opts) : Promise.resolve(null); vetted .then((addrs) => addrs ?? dns.lookup(hostname, { all: true })) @@ -373,11 +396,14 @@ function envProxyHosts(env: NodeJS.ProcessEnv): Set { * Pass both to axios (`httpAgent`, `httpsAgent`) together with `proxy: false`: * an env proxy would make the agent resolve the proxy, not the target. */ -export function createSsrfGuardedAgents(env: NodeJS.ProcessEnv = process.env): { +export function createSsrfGuardedAgents( + env: NodeJS.ProcessEnv = process.env, + opts: VetHostOptions = {}, +): { httpAgent: http.Agent; httpsAgent: https.Agent; } { - const lookup = ssrfGuardedLookup(env); + const lookup = ssrfGuardedLookup(env, opts); // Same socket reuse as Node's global agents, which these replace. const options = { lookup, keepAlive: true, scheduling: 'lifo' as const, timeout: 5000 }; return {