Send an alert webhook when a connector keeps failing

docs/operations/observability.md

@@ -104,6 +104,78 @@ journalctl -u anythingmcp-stuck-report -n 50
 
 `bash deploy/cloud/stuck-users-report.test.sh` runs the report against a throwaway database with every migration applied and a throwaway SMTP server; CI runs it on each PR.
 
+## 5. Connector-failure alerts (webhook, opt-in per organization)
+
+The three pipelines above tell *you* something is wrong once you go looking. This one pushes a notification the moment a connector starts failing, so you find out without watching the logs.
+
+An admin configures one webhook per organization at `PUT /api/admin/settings/alert-webhook` (`GET`/`DELETE` and `POST .../test` alongside it; ADMIN role required). Nothing fires until it's configured: self-hosted installs that never set one up get no outbound traffic from this feature.
+
+```json
+{
+  "url": "https://example.com/hooks/anythingmcp",
+  "type": "json",
+  "enabled": true,
+  "threshold": 5,
+  "windowMinutes": 10,
+  "cooldownMinutes": 30
+}
+```
+
+`type: "slack"` posts a ready-to-render Slack message (`{ "text": "..." }`) instead of the JSON event below; point it at an Incoming Webhook URL. The signing secret is generated server-side on first save (or when `rotateSecret: true` is sent) and returned exactly once in the response; `GET` never returns it.
+
+### When it fires
+
+Every tool call against a connector that ends in `ERROR` or `TIMEOUT` counts toward a window of `windowMinutes` (default 10) that starts at the connector's first failure. Once the count for that connector reaches `threshold` (default 5) within the window, exactly one alert is dispatched; a `cooldownMinutes` (default 30) cooldown then suppresses further alerts for that connector even if it keeps failing, so a sustained outage sends one notification, not one per call.
+
+### Payload (`type: "json"`)
+
+```json
+{
+  "event": "connector.failing",
+  "organizationId": "org_123",
+  "connector": { "id": "conn_abc", "name": "SAP OData" },
+  "failures": 5,
+  "windowMinutes": 10,
+  "lastError": { "tool": "sap_get_orders", "message": "503 Service Unavailable" },
+  "firstFailureAt": "2026-10-09T10:00:00.000Z",
+  "lastFailureAt": "2026-10-09T10:04:12.000Z",
+  "logsUrl": "https://<your-instance>/logs?connectorId=conn_abc"
+}
+```
+
+`lastError.message` is truncated to 300 characters and never includes tool input or output, the same boundary the audit log itself keeps.
+
+### Headers and signature
+
+```
+Content-Type: application/json
+X-AnythingMCP-Event: connector.failing
+X-AnythingMCP-Timestamp: 1760000000
+X-AnythingMCP-Signature: sha256=<hex HMAC-SHA256(secret, "<timestamp>.<raw body>")>
+```
+
+Verify it (Node):
+
+```js
+const crypto = require('crypto');
+
+function verify(secret, timestamp, rawBody, signatureHeader) {
+  const expected = 'sha256=' + crypto
+    .createHmac('sha256', secret)
+    .update(`${timestamp}.${rawBody}`)
+    .digest('hex');
+  if (Math.abs(Date.now() / 1000 - Number(timestamp)) > 300) return false; // reject replays
+  if (signatureHeader.length !== expected.length) return false;
+  return crypto.timingSafeEqual(Buffer.from(expected), Buffer.from(signatureHeader));
+}
+```
+
+Use the *raw* request body, not a re-serialized JSON object, since re-serializing can change key order or whitespace and break the comparison.
+
+### SSRF
+
+The webhook URL is validated with the same guard (`SSRF_GUARD`, private/loopback/metadata-address blocking) used for every other outbound call this app makes on a user's behalf, both when it's saved and again on every dispatch, and redirects are not followed. On AnythingMCP Cloud the URL must use `https`, and it ignores both the env (`SSRF_ALLOWED_HOSTS`) and the admin-editable SSRF allowlist, which name hosts only the platform may reach. On a self-hosted instance the operator's allowlist applies, so an internal Mattermost or Slack proxy can receive alerts.
+
 ## Correlating across pipelines
 
 The same `req.id` UUID appears in:

packages/backend/src/alerts/alerts.controller.spec.ts

@@ -0,0 +1,92 @@
+import { ForbiddenException } from '@nestjs/common';
+import { Reflector } from '@nestjs/core';
+import { ROLES_KEY, RolesGuard } from '../auth/roles.guard';
+import { AlertsAdminController } from './alerts.controller';
+
+function buildController(overrides: Partial<Record<string, jest.Mock>> = {}) {
+  const alerts = {
+    getConfig: jest.fn().mockResolvedValue({ configured: false }),
+    saveConfig: jest.fn().mockResolvedValue({}),
+    deleteConfig: jest.fn().mockResolvedValue(undefined),
+    testWebhook: jest.fn().mockResolvedValue({ success: true, status: 200 }),
+    ...overrides,
+  };
+  const controller = new AlertsAdminController(alerts as any);
+  return { controller, alerts };
+}
+
+const req = (organizationId = 'org1') => ({ user: { organizationId, role: 'ADMIN' } });
+
+describe('AlertsAdminController', () => {
+  it('requires the ADMIN role', () => {
+    const reflector = new Reflector();
+    const roles = reflector.get<string[]>(ROLES_KEY, AlertsAdminController);
+    expect(roles).toEqual(['ADMIN']);
+  });
+
+  // RolesGuard's own pass/fail logic has its own spec (auth/roles.guard.spec.ts);
+  // this exercises it against this controller's actual metadata, so a non-admin
+  // caller hitting any of these routes is proven to get a 403, not just asserted
+  // to be configured for one.
+  it('rejects a non-admin caller with 403', () => {
+    const reflector = new Reflector();
+    const guard = new RolesGuard(reflector);
+    const context = {
+      getHandler: () => AlertsAdminController.prototype.getConfig,
+      getClass: () => AlertsAdminController,
+      switchToHttp: () => ({ getRequest: () => ({ user: { organizationId: 'org1', role: 'EDITOR' } }) }),
+    } as any;
+
+    expect(() => guard.canActivate(context)).toThrow(ForbiddenException);
+  });
+
+  it('lets an admin caller through', () => {
+    const reflector = new Reflector();
+    const guard = new RolesGuard(reflector);
+    const context = {
+      getHandler: () => AlertsAdminController.prototype.getConfig,
+      getClass: () => AlertsAdminController,
+      switchToHttp: () => ({ getRequest: () => ({ user: { organizationId: 'org1', role: 'ADMIN' } }) }),
+    } as any;
+
+    expect(guard.canActivate(context)).toBe(true);
+  });
+
+  it('getConfig delegates to the service with the caller\'s organization', async () => {
+    const { controller, alerts } = buildController();
+    await controller.getConfig(req('org-x'));
+    expect(alerts.getConfig).toHaveBeenCalledWith('org-x');
+  });
+
+  it('saveConfig returns the secret only when the service generated one', async () => {
+    const { controller, alerts } = buildController({
+      saveConfig: jest.fn().mockResolvedValue({ secret: 'plain-secret' }),
+    });
+    const result = await controller.saveConfig(req('org-x'), {
+      url: 'https://hooks.example.com/x',
+      type: 'json',
+    } as any);
+    expect(alerts.saveConfig).toHaveBeenCalledWith('org-x', expect.objectContaining({ url: 'https://hooks.example.com/x' }));
+    expect(result).toEqual({ message: 'Alert webhook saved', secret: 'plain-secret' });
+  });
+
+  it('saveConfig omits the secret field when none was generated', async () => {
+    const { controller } = buildController({ saveConfig: jest.fn().mockResolvedValue({}) });
+    const result = await controller.saveConfig(req(), { url: 'https://hooks.example.com/x', type: 'json' } as any);
+    expect(result).toEqual({ message: 'Alert webhook saved' });
+  });
+
+  it('deleteConfig delegates to the service', async () => {
+    const { controller, alerts } = buildController();
+    const result = await controller.deleteConfig(req('org-x'));
+    expect(alerts.deleteConfig).toHaveBeenCalledWith('org-x');
+    expect(result).toEqual({ message: 'Alert webhook removed' });
+  });
+
+  it('testWebhook delegates to the service and returns its result', async () => {
+    const { controller, alerts } = buildController();
+    const result = await controller.testWebhook(req('org-x'));
+    expect(alerts.testWebhook).toHaveBeenCalledWith('org-x');
+    expect(result).toEqual({ success: true, status: 200 });
+  });
+});

packages/backend/src/alerts/alerts.controller.ts

@@ -0,0 +1,90 @@
+import { Body, Controller, Delete, Get, Post, Put, Req, UseGuards } from '@nestjs/common';
+import { ApiBearerAuth, ApiOperation, ApiProperty, ApiPropertyOptional, ApiTags } from '@nestjs/swagger';
+import { AuthGuard } from '@nestjs/passport';
+import { Throttle } from '@nestjs/throttler';
+import { IsBoolean, IsIn, IsInt, IsOptional, IsString, IsUrl, Max, MaxLength, Min } from 'class-validator';
+import { Roles, RolesGuard } from '../auth/roles.guard';
+import { AlertsService } from './alerts.service';
+
+class SaveAlertWebhookDto {
+  @ApiProperty({ description: 'Webhook URL to POST the signed alert to.', example: 'https://example.com/hooks/anythingmcp' })
+  @IsString()
+  @MaxLength(2048)
+  @IsUrl({ protocols: ['http', 'https'], require_protocol: true, require_tld: false })
+  url: string;
+
+  @ApiProperty({ description: 'Payload shape: a generic JSON event, or a ready-to-post Slack message.', enum: ['json', 'slack'] })
+  @IsIn(['json', 'slack'])
+  type: 'json' | 'slack';
+
+  @ApiPropertyOptional({ description: 'Whether the alert is active. Default true.' })
+  @IsOptional()
+  @IsBoolean()
+  enabled?: boolean;
+
+  @ApiPropertyOptional({ description: 'Consecutive failures within the window before an alert fires. Default 5.' })
+  @IsOptional()
+  @IsInt()
+  @Min(1)
+  @Max(1000)
+  threshold?: number;
+
+  @ApiPropertyOptional({ description: 'Window, in minutes, that failures are counted over; it starts at the first failure. Default 10.' })
+  @IsOptional()
+  @IsInt()
+  @Min(1)
+  @Max(1440)
+  windowMinutes?: number;
+
+  @ApiPropertyOptional({ description: 'Minutes to wait after a dispatch before the same connector can alert again. Default 30.' })
+  @IsOptional()
+  @IsInt()
+  @Min(1)
+  @Max(10080)
+  cooldownMinutes?: number;
+
+  @ApiPropertyOptional({
+    description:
+      'Generate a new signing secret and return it once. Required to see the secret again after the first save.',
+  })
+  @IsOptional()
+  @IsBoolean()
+  rotateSecret?: boolean;
+}
+
+@ApiTags('Alerts')
+@ApiBearerAuth()
+@UseGuards(AuthGuard('jwt'), RolesGuard)
+@Roles('ADMIN')
+@Controller('api/admin/settings')
+export class AlertsAdminController {
+  constructor(private readonly alerts: AlertsService) {}
+
+  @Get('alert-webhook')
+  @ApiOperation({ summary: 'Get the connector-failure alert webhook for the current organization (ADMIN)' })
+  async getConfig(@Req() req: any) {
+    return this.alerts.getConfig(req.user.organizationId);
+  }
+
+  @Put('alert-webhook')
+  @ApiOperation({ summary: 'Save the connector-failure alert webhook for the current organization (ADMIN)' })
+  async saveConfig(@Req() req: any, @Body() dto: SaveAlertWebhookDto) {
+    const { secret } = await this.alerts.saveConfig(req.user.organizationId, dto);
+    return { message: 'Alert webhook saved', ...(secret ? { secret } : {}) };
+  }
+
+  @Delete('alert-webhook')
+  @ApiOperation({ summary: 'Remove the connector-failure alert webhook for the current organization (ADMIN)' })
+  async deleteConfig(@Req() req: any) {
+    await this.alerts.deleteConfig(req.user.organizationId);
+    return { message: 'Alert webhook removed' };
+  }
+
+  @Post('alert-webhook/test')
+  // It reports the status any public URL answers with: keep it a test, not a probe.
+  @Throttle({ default: { limit: 5, ttl: 60_000 } })
+  @ApiOperation({ summary: 'Send a test alert to the configured webhook (ADMIN)' })
+  async testWebhook(@Req() req: any) {
+    return this.alerts.testWebhook(req.user.organizationId);
+  }
+}

packages/backend/src/alerts/alerts.module.ts

@@ -0,0 +1,17 @@
+import { Module } from '@nestjs/common';
+import { SettingsModule } from '../settings/settings.module';
+import { AlertsService } from './alerts.service';
+import { AlertsAdminController } from './alerts.controller';
+
+// RedisService and PrismaService come from the @Global() RedisModule and
+// PrismaModule, so only SettingsModule (for OrgSettingsService) needs to be
+// imported here. Keep this list short: AuditModule imports AlertsModule to
+// inject AlertsService into AuditService, so anything this module pulled in
+// that itself depended on AuditModule would be a cycle.
+@Module({
+  imports: [SettingsModule],
+  controllers: [AlertsAdminController],
+  providers: [AlertsService],
+  exports: [AlertsService],
+})
+export class AlertsModule {}