Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/operations/observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ Every HTTP request emits one JSON line at completion (or earlier on error), tagg

- `req.id` — UUID, present on the response as `X-Request-Id`. Quote it in bug reports.
- `userId` / `orgId` / `authMethod` — propagated from the authenticated request.
- `req.method`, `req.url`, `res.statusCode`, `responseTime`.
- `req.method`, `req.url` (path only, no query string), `res.statusCode`, `responseTime`.
- `req.mcpMethod` (the `Mcp-Method` header, on MCP requests), `req.userAgent`, `req.ip` (first `X-Forwarded-For` hop, else the socket address).

Sensitive fields are stripped at the pino layer:
Request and response headers are not logged (`packages/backend/src/common/log-serializers.ts` keeps only the fields above), so no credential header can reach a line. As a second line of defence, pino also strips:
- headers: `authorization`, `cookie`, `x-api-key`, `set-cookie`.
- bodies: any `password`, `token`, `refreshToken`, `accessToken`, `apiKey`, `secret` field.

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
-- =============================================================================
-- Migration: kg_value_seen rows belong to their connector
-- =============================================================================
-- kg_value_seen.connector_id had no foreign key, so deleting a connector left
-- its value hashes behind (102 deleted connectors, 57.5k rows on cloud by
-- 21 Oct 2026). The observational ingest kept pairing fresh values with them
-- and tried to create kg_nodes for the deleted connector, which Postgres
-- refused on kg_nodes_connector_id_fkey some 21.8k times a day.
--
-- Delete the orphans, then add the foreign key with ON DELETE CASCADE plus an
-- index on connector_id, so the cascade does not scan the whole table every
-- time a connector is deleted.
--
-- Locking: the migration runs as one transaction. On cloud (2.7 M rows,
-- 1.5 GB) the orphan scan takes under a second and the index build about
-- four. CREATE INDEX blocks writes to kg_value_seen only (the KG ingest runs
-- in the background and retries on the next call). The foreign key is added
-- NOT VALID, which is instant, so "connectors" is locked against writes only
-- for the short second delete and VALIDATE that follow. The second delete
-- removes rows of a connector deleted between the first one and the moment
-- the constraint started to apply.
-- =============================================================================

DELETE FROM "kg_value_seen" v
WHERE NOT EXISTS (SELECT 1 FROM "connectors" c WHERE c."id" = v."connector_id");

CREATE INDEX "kg_value_seen_connector_id_idx" ON "kg_value_seen"("connector_id");

ALTER TABLE "kg_value_seen"
ADD CONSTRAINT "kg_value_seen_connector_id_fkey"
FOREIGN KEY ("connector_id") REFERENCES "connectors"("id")
ON DELETE CASCADE ON UPDATE CASCADE NOT VALID;

DELETE FROM "kg_value_seen" v
WHERE NOT EXISTS (SELECT 1 FROM "connectors" c WHERE c."id" = v."connector_id");

ALTER TABLE "kg_value_seen" VALIDATE CONSTRAINT "kg_value_seen_connector_id_fkey";
6 changes: 6 additions & 0 deletions packages/backend/prisma/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,7 @@ model Connector {
kgNodes KgNode[]
kgState KgConnectorState?
kgSkills KgSkillSuggestion[]
kgValuesSeen KgValueSeen[]

@@index([organizationId])
@@map("connectors")
Expand Down Expand Up @@ -1460,6 +1461,10 @@ model KgValueSeen {
organizationId String @map("organization_id")
organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
connectorId String @map("connector_id")
// Deleting a connector deletes its values. Without the relation they stayed
// behind and every correlate pass tried to create nodes for the deleted
// connector (21.8k foreign-key errors a day on cloud by 21 Oct 2026).
connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade)
valueHash String @map("value_hash")
entity String
field String
Expand All @@ -1472,6 +1477,7 @@ model KgValueSeen {
@@unique([organizationId, connectorId, valueHash, entity, field, direction], map: "kg_value_seen_occurrence_key")
@@index([organizationId, valueHash])
@@index([seenAt])
@@index([connectorId])
@@map("kg_value_seen")
}

Expand Down
16 changes: 14 additions & 2 deletions packages/backend/src/auth/email-verified.guard.spec.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { ForbiddenException } from '@nestjs/common';
import { EmailVerifiedGuard } from './email-verified.guard';
import { McpEmailVerificationRequiredException } from './mcp-email-verification';

/**
* Global guards run before the route's AuthGuard('jwt'), so req.user is unset
Expand All @@ -8,7 +9,9 @@ import { EmailVerifiedGuard } from './email-verified.guard';
describe('EmailVerifiedGuard', () => {
function makeGuard(opts: { cloud?: boolean; verified?: boolean }) {
const prisma = {
user: { findUnique: jest.fn(async () => ({ emailVerified: opts.verified ?? false })) },
user: {
findUnique: jest.fn(async () => ({ emailVerified: opts.verified ?? false, email: 'jane.doe@example.com' })),
},
};
const auth = {
verifyToken: jest.fn((t: string) => {
Expand All @@ -35,8 +38,17 @@ describe('EmailVerifiedGuard', () => {
it('tells an AI client on /mcp what the person has to do', async () => {
const { guard } = makeGuard({ verified: false });
const refusal = guard.canActivate(ctx({ path: '/mcp/srv1', headers: { authorization: 'Bearer good' } }));
await expect(refusal).rejects.toBeInstanceOf(McpEmailVerificationRequiredException);
await expect(refusal).rejects.toBeInstanceOf(ForbiddenException);
await expect(refusal).rejects.toThrow(/confirm the email address.*sign in at https:\/\//);
await expect(refusal).rejects.toThrow(/not been verified.*ja\*\*\*@example\.com.*Sign in at https:\/\/\S+\/login/);
await expect(refusal).rejects.not.toThrow(/jane\.doe/);
});

it('keeps the plain refusal off /mcp', async () => {
const { guard } = makeGuard({ verified: false });
const refusal = guard.canActivate(ctx({ path: '/api/connectors', headers: { authorization: 'Bearer good' } }));
await expect(refusal).rejects.not.toBeInstanceOf(McpEmailVerificationRequiredException);
await expect(refusal).rejects.toThrow('Email verification required');
});

it('lets a verified user through', async () => {
Expand Down
21 changes: 12 additions & 9 deletions packages/backend/src/auth/email-verified.guard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ import { ConfigService } from '@nestjs/config';
import { PrismaService } from '../common/prisma.service';
import { AuthService } from './auth.service';
import { resolveUserIdFromTokenPayload } from './resolve-user-id.util';
import {
emailVerificationNotice,
McpEmailVerificationRequiredException,
} from './mcp-email-verification';

/**
* In cloud mode, users must verify their email before they can access any
Expand Down Expand Up @@ -80,21 +84,20 @@ export class EmailVerifiedGuard implements CanActivate {

const dbUser = await this.prisma.user.findUnique({
where: { id: userId },
select: { emailVerified: true },
select: { emailVerified: true, email: true },
});
// Unknown user: not ours to judge (see the class comment).
if (!dbUser) return true;
if (!dbUser.emailVerified) {
if (path.startsWith('/mcp')) {
this.logger.warn(`Refused ${path}: user ${userId} has not verified their email`);
// An AI client shows this text to the person, who otherwise only sees
// the connection fail: on 9-10 Oct 2026 seven people who had connected
// Claude retried up to 122 times without knowing what was missing.
const dashboard = (this.configService.get<string>('FRONTEND_URL') || 'https://cloud.anythingmcp.com').replace(/\/+$/, '');
throw new ForbiddenException(
'Email verification required: confirm the email address of your AnythingMCP account ' +
`with the link we sent when you signed up, or sign in at ${dashboard} to get a new ` +
'link. Then try again; nothing else needs to change.',
// The person otherwise only sees the connection fail: on 9-10 Oct
// 2026 seven people who had connected Claude retried up to 122 times
// without knowing what was missing. McpEmailVerificationFilter turns
// this into an MCP answer the client shows and the model can read.
const dashboard = this.configService.get<string>('FRONTEND_URL') || 'https://cloud.anythingmcp.com';
throw new McpEmailVerificationRequiredException(
emailVerificationNotice(dashboard, dbUser.email),
);
}
throw new ForbiddenException('Email verification required');
Expand Down
161 changes: 161 additions & 0 deletions packages/backend/src/auth/mcp-email-verification.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
import { ForbiddenException, Logger } from '@nestjs/common';
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
import express from 'express';
import type { AddressInfo } from 'net';
import type { Server } from 'http';
import {
emailVerificationNotice,
maskEmail,
McpEmailVerificationFilter,
McpEmailVerificationRequiredException,
VERIFY_EMAIL_TOOL,
} from './mcp-email-verification';

describe('maskEmail', () => {
it('keeps enough to recognise the address and no more', () => {
expect(maskEmail('jane.doe@example.com')).toBe('ja***@example.com');
expect(maskEmail('jo@example.com')).toBe('j***@example.com');
expect(maskEmail('j@example.com')).toBe('j***@example.com');
});

it('gives nothing for what is not an address', () => {
expect(maskEmail(undefined)).toBeUndefined();
expect(maskEmail('no-at-sign')).toBeUndefined();
expect(maskEmail('@example.com')).toBeUndefined();
});
});

describe('emailVerificationNotice', () => {
it('says what is missing, where the email went, and how to get a new code', () => {
const text = emailVerificationNotice('https://cloud.anythingmcp.com/', 'jane.doe@example.com');
expect(text).toMatch(/email address has not been verified/);
expect(text).toContain('ja***@example.com');
expect(text).toContain('https://cloud.anythingmcp.com/login');
expect(text).not.toContain('jane.doe');
});
});

/**
* The guard's refusal, as an MCP client receives it. A plain 403 reads as a
* broken connection; this answers the protocol so the notice reaches the
* person and the model, with nothing of the workspace in it.
*/
describe('McpEmailVerificationFilter', () => {
const NOTICE = emailVerificationNotice('https://cloud.example.com', 'jane.doe@example.com');
let server: Server;
let base: string;
const savedJson = process.env.MCP_STREAMABLE_JSON_RESPONSE;

beforeAll(async () => {
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
const filter = new McpEmailVerificationFilter();
const app = express();
app.use(express.json());
// Stands in for Nest: the guard throws, the filter gets the exception.
app.all('/mcp', (req, res) => {
const host = { switchToHttp: () => ({ getRequest: () => req, getResponse: () => res }) } as any;
filter.catch(new McpEmailVerificationRequiredException(NOTICE), host);
});
await new Promise<void>((resolve) => {
server = app.listen(0, '127.0.0.1', () => resolve());
});
base = `http://127.0.0.1:${(server.address() as AddressInfo).port}/mcp`;
});

afterAll(async () => {
process.env.MCP_STREAMABLE_JSON_RESPONSE = savedJson;
jest.restoreAllMocks();
await new Promise((resolve) => server.close(resolve));
});

const rpc = (method: string, params: Record<string, unknown> = {}, id = 1) =>
fetch(base, {
method: 'POST',
headers: {
'content-type': 'application/json',
accept: 'application/json, text/event-stream',
'mcp-protocol-version': '2025-06-18',
},
body: JSON.stringify({ jsonrpc: '2.0', id, method, params }),
});

/** The JSON-RPC message of a JSON or SSE-framed answer. */
const message = async (res: Response) => {
const text = await res.text();
if ((res.headers.get('content-type') ?? '').includes('text/event-stream')) {
const data = text.split('\n').find((l) => l.startsWith('data: '));
return JSON.parse(data!.slice(6));
}
return JSON.parse(text);
};

const initialize = () =>
rpc('initialize', {
protocolVersion: '2025-06-18',
capabilities: {},
clientInfo: { name: 'test', version: '1' },
});

for (const json of [true, false]) {
describe(json ? 'with JSON responses' : 'with SSE responses', () => {
beforeAll(() => {
process.env.MCP_STREAMABLE_JSON_RESPONSE = json ? 'true' : 'false';
});

it('lets the client connect and carries the notice in the instructions', async () => {
const res = await initialize();
expect(res.status).toBe(200);
const msg = await message(res);
expect(msg.result.instructions).toBe(NOTICE);
expect(msg.result.serverInfo.name).toBe('AnythingMCP');
});

it('lists only the verification tool', async () => {
const msg = await message(await rpc('tools/list'));
expect(msg.result.tools.map((t: { name: string }) => t.name)).toEqual([VERIFY_EMAIL_TOOL]);
});

it('answers any tool call with the notice as an error the model reads', async () => {
const msg = await message(await rpc('tools/call', { name: 'crm_list_customers', arguments: {} }));
const text = JSON.stringify(msg);
expect(text).toContain('ja***@example.com');
expect(text).not.toContain('jane.doe');
const call = await message(await rpc('tools/call', { name: VERIFY_EMAIL_TOOL, arguments: {} }));
expect(call.result.isError).toBe(true);
expect(call.result.content[0].text).toBe(NOTICE);
});
});
}

it('works for the current SDK client, whatever protocol revision it negotiates', async () => {
delete process.env.MCP_STREAMABLE_JSON_RESPONSE;
const client = new Client({ name: 'spec', version: '1.0.0' });
await client.connect(new StreamableHTTPClientTransport(new URL(base)));
try {
expect(client.getInstructions()).toBe(NOTICE);
expect((await client.listTools()).tools.map((t) => t.name)).toEqual([VERIFY_EMAIL_TOOL]);
const result: any = await client.callTool({ name: 'crm_list_customers', arguments: {} });
expect(result.isError).toBe(true);
expect(result.content[0].text).toBe(NOTICE);
} finally {
await client.close();
}
});

it('refuses a GET and a listen stream with the notice as a JSON-RPC error', async () => {
const get = await fetch(base, { headers: { accept: 'text/event-stream' } });
expect(get.status).toBe(403);
expect((await get.json()).error.message).toBe(NOTICE);

const listen = await rpc('subscriptions/listen');
expect(listen.status).toBe(403);
expect((await listen.json()).error).toEqual({ code: -32001, message: NOTICE });
});

it('is still a 403 with the notice wherever the filter is not in play', () => {
const err = new McpEmailVerificationRequiredException(NOTICE);
expect(err).toBeInstanceOf(ForbiddenException);
expect(err.getStatus()).toBe(403);
expect(err.message).toBe(NOTICE);
});
});
Loading
Loading