From bf40571f0ea2945701476ed0153f1b7441e529ee Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 13 Sep 2026 10:34:56 +0000 Subject: [PATCH] mail: defang hostnames Forward Email refuses before they cost the account (0.37.0) Forward Email runs every hostname in an outbound message past Cloudflare's Family DNS and refuses the whole message when one is blocked (554 5.6.0 "Link hostname of bittorrented.com was detected by Cloudflare's Family DNS to contain adult-related content, phishing, and/or malware"). A bare mention counts, and a few refusals in a rolling window suspend the account's outbound queue until their support lifts it by hand, which is what happened to profullstack.com on 2026-09-13: the nightly nichedb report names our own torrent site. So before a message leaves through a forwardemail.net SMTP host, every hostname-shaped token in the subject, text and html is resolved at 1.1.1.3, and one that answers 0.0.0.0 is defanged in place (bittorrented[.]com). Addresses are left alone, at most 50 distinct hosts are looked up, and a host that does not resolve at all goes as written. stderr says which hosts were touched; --json carries them as `defanged`. Other providers are not checked, and Resend is not affected. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UpE2YNWeoYxtu2B8oL2LPS --- bin/mail.ts | 8 ++- package.json | 2 +- src/mail.ts | 129 +++++++++++++++++++++++++++++++++++++++++++--- test/mail.test.ts | 71 +++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 9 deletions(-) diff --git a/bin/mail.ts b/bin/mail.ts index 27b4227..93be903 100755 --- a/bin/mail.ts +++ b/bin/mail.ts @@ -40,6 +40,7 @@ import { buildReply, chooseTransport, composeRaw, + defangHost, folderFor, formatAccounts, formatFolders, @@ -717,6 +718,11 @@ async function main(argv: string[]): Promise { `mail: ${result.fellBackFrom.transport} failed (${result.fellBackFrom.error}); sent via ${result.transport}\n`, ); } + if (result.defanged && result.defanged.length > 0) { + process.stderr.write( + `mail: defanged ${result.defanged.length} hostname${result.defanged.length === 1 ? '' : 's'} Forward Email refuses (Cloudflare Family DNS): ${result.defanged.map(defangHost).join(', ')}\n`, + ); + } // SMTP servers file their own Sent copy; Resend never does. let filed: string | null = null; if (result.transport === 'resend') { @@ -729,7 +735,7 @@ async function main(argv: string[]): Promise { const uid = needUids(rest, 'reply')[0]!; await withMailbox(account, (box) => box.flag(folder, [uid], ['\\Answered'], [])).catch(() => undefined); } - if (isJson) json({ transport: result.transport, id: result.id, to: outgoing.to, subject: outgoing.subject, filed }); + if (isJson) json({ transport: result.transport, id: result.id, to: outgoing.to, subject: outgoing.subject, filed, defanged: result.defanged ?? [] }); else { out(`sent via ${result.transport} to ${outgoing.to.join(', ')}: ${outgoing.subject}${result.id ? ` (${result.id})` : ''}`); if (filed) out(`copy filed in ${filed}`); diff --git a/package.json b/package.json index 02a7043..12fa086 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/cli-tools", - "version": "0.36.1", + "version": "0.37.0", "private": true, "description": "Local command-line tools, in TypeScript, exposed on PATH.", "type": "module", diff --git a/src/mail.ts b/src/mail.ts index abde4f2..d852f35 100644 --- a/src/mail.ts +++ b/src/mail.ts @@ -26,7 +26,7 @@ * {@link Mailbox} interface so tests never open a socket. */ -import { resolveMx } from 'node:dns/promises'; +import { Resolver, resolveMx } from 'node:dns/promises'; import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -1419,6 +1419,107 @@ export async function composeRaw(outgoing: Outgoing): Promise { return composer.compile().build(); } +// --------------------------------------------------------------------------- +// Hostnames Forward Email refuses +// --------------------------------------------------------------------------- + +/** + * Forward Email runs every hostname in an outbound message past Cloudflare's + * Family DNS and refuses the whole message when one is blocked: + * + * 554 5.6.0 Link hostname of bittorrented.com was detected by Cloudflare's + * Family DNS to contain adult-related content, phishing, and/or malware. + * + * A bare mention counts, not just a link, and a few refusals inside a rolling + * window suspend the account's outbound queue until their support lifts it by + * hand. Our own torrent site in a report title was enough to do it. + * + * So before a message leaves through Forward Email, every hostname in it is + * resolved at 1.1.1.3, and one that answers 0.0.0.0 (the block answer) is + * defanged in place: `bittorrented.com` becomes `bittorrented[.]com`, which a + * reader understands and the filter does not see. Addresses are left alone; + * the check is on link hostnames, and a defanged address would bounce. + */ + +export const FAMILY_DNS = '1.1.1.3'; + +/** SMTP hosts known to run the Family DNS check on what they relay. */ +const FAMILY_DNS_FILTERED_SMTP = /(^|\.)forwardemail\.net$/i; + +export function smtpFiltersHostnames(account: Pick): boolean { + return FAMILY_DNS_FILTERED_SMTP.test(account.smtp.host); +} + +/** Answers true when Family DNS blocks the host. */ +export type HostBlockCheck = (host: string) => Promise; + +/** Hostname-shaped tokens: labels, then a letters-only top level. Not preceded by `@` (an address) or a dot (already inside a longer name). */ +const HOSTNAME_RE = /(?(); + for (const match of text.matchAll(HOSTNAME_RE)) out.add(match[1]!.toLowerCase()); + return [...out]; +} + +/** The resolver check, against 1.1.1.3 by default. A host it cannot resolve at all is not blocked, it is unknown, and goes as written. */ +export function familyDnsBlockCheck(server: string = FAMILY_DNS): HostBlockCheck { + const resolver = new Resolver({ timeout: 3000, tries: 1 }); + resolver.setServers([server]); + return async (host) => { + try { + const addresses = await resolver.resolve4(host); + return addresses.includes('0.0.0.0'); + } catch { + return false; + } + }; +} + +export function defangHost(host: string): string { + return host.replace(/\./g, '[.]'); +} + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** `hosts` defanged wherever they appear as a whole name; longest first, so `www.x.com` is done before `x.com` finds its tail. */ +export function defangHosts(text: string, hosts: string[]): string { + let out = text; + for (const host of [...hosts].sort((a, b) => b.length - a.length)) { + const re = new RegExp(`(? { + const hosts = hostnamesIn([outgoing.subject, outgoing.text, outgoing.html ?? ''].join('\n')).slice(0, HOST_LOOKUP_CAP); + const verdicts = await Promise.all(hosts.map(async (host) => [host, await isBlocked(host)] as const)); + const blocked = verdicts.filter(([, yes]) => yes).map(([host]) => host).sort(); + if (blocked.length === 0) return { outgoing, defanged: [] }; + return { + outgoing: { + ...outgoing, + subject: defangHosts(outgoing.subject, blocked), + text: defangHosts(outgoing.text, blocked), + ...(outgoing.html ? { html: defangHosts(outgoing.html, blocked) } : {}), + }, + defanged: blocked, + }; +} + // --------------------------------------------------------------------------- // Sending // --------------------------------------------------------------------------- @@ -1492,6 +1593,8 @@ export interface SendResult { id: string | null; /** Set when the first transport failed and the second carried it. */ fellBackFrom?: { transport: Transport; error: string }; + /** Hostnames Forward Email would have refused, defanged before the send. */ + defanged?: string[]; } export type SmtpSender = (account: Account, outgoing: Outgoing) => Promise; @@ -1591,27 +1694,39 @@ export async function sendMail( via?: Transport; smtp?: SmtpSender; resend?: ResendSender; + /** The Family DNS check, for a test; the real resolver otherwise. */ + isBlocked?: HostBlockCheck; } = {}, ): Promise { const choice = chooseTransport(account, options.resendKey, options.via); const smtp = options.smtp ?? sendViaSmtp; const resend = options.resend ?? resendSender(); + // Through Forward Email, a blocked hostname is defanged first or the message + // never leaves; the fallback (if taken) carries the same defanged copy. + let message = outgoing; + let defanged: string[] = []; + if (choice.transport === 'smtp' && smtpFiltersHostnames(account)) { + ({ outgoing: message, defanged } = await defangBlockedHosts(outgoing, options.isBlocked ?? familyDnsBlockCheck())); + } + const tail = defanged.length > 0 ? { defanged } : {}; + const attempt = async (transport: Transport): Promise => - transport === 'smtp' ? smtp(account, outgoing) : resend(options.resendKey!, outgoing); + transport === 'smtp' ? smtp(account, message) : resend(options.resendKey!, message); try { - return { transport: choice.transport, id: await attempt(choice.transport) }; + return { transport: choice.transport, id: await attempt(choice.transport), ...tail }; } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (!choice.fallback || !isTransportFailure(message)) { - throw error instanceof MailError ? error : new MailError(`${choice.transport}: ${message}`); + const reason = error instanceof Error ? error.message : String(error); + if (!choice.fallback || !isTransportFailure(reason)) { + throw error instanceof MailError ? error : new MailError(`${choice.transport}: ${reason}`); } const id = await attempt(choice.fallback); return { transport: choice.fallback, id, - fellBackFrom: { transport: choice.transport, error: message }, + fellBackFrom: { transport: choice.transport, error: reason }, + ...tail, }; } } diff --git a/test/mail.test.ts b/test/mail.test.ts index 8b12dc2..681fbac 100644 --- a/test/mail.test.ts +++ b/test/mail.test.ts @@ -17,6 +17,9 @@ import { buildReply, chooseTransport, composeRaw, + defangBlockedHosts, + defangHost, + defangHosts, folderFor, formatAccounts, formatAddresses, @@ -25,6 +28,7 @@ import { formatProviders, fromHeader, guessProvider, + hostnamesIn, isProviderName, isTransportFailure, loadConfig, @@ -45,6 +49,7 @@ import { selectAccount, selectAccounts, sendMail, + smtpFiltersHostnames, senderName, splitAddresses, stripHtml, @@ -552,6 +557,72 @@ describe('sendMail', () => { ).rejects.toThrow(/ETIMEDOUT/); }); + // Forward Email refuses a message that so much as mentions a hostname + // Cloudflare's Family DNS blocks, and a few refusals suspend the account. + describe('hostnames Forward Email refuses', () => { + const blocked = async (host: string) => host.endsWith('bittorrented.com'); + + it('finds the hostname-shaped tokens in a body, once each, addresses excluded', () => { + expect( + hostnamesIn( + 'New on https://www.bittorrented.com/blog and bittorrented.com/dht; mail bot@bittorrented.com. Also nichedb.dev, v1.2.3 and e.g. this.', + ), + ).toEqual(['www.bittorrented.com', 'bittorrented.com', 'nichedb.dev']); + }); + + it('defangs every dot of a blocked host, longest name first, and leaves the rest', () => { + expect(defangHost('www.bittorrented.com')).toBe('www[.]bittorrented[.]com'); + expect(defangHosts('see www.bittorrented.com and bittorrented.com, not bittorrented.community', ['bittorrented.com', 'www.bittorrented.com'])).toBe( + 'see www[.]bittorrented[.]com and bittorrented[.]com, not bittorrented.community', + ); + }); + + it('rewrites subject, text and html, and says which hosts it touched', async () => { + const result = await defangBlockedHosts( + { ...outgoing, subject: 'bittorrented.com is down', text: 'https://bittorrented.com/x and nichedb.dev\n', html: 'x' }, + blocked, + ); + expect(result.defanged).toEqual(['bittorrented.com']); + expect(result.outgoing.subject).toBe('bittorrented[.]com is down'); + expect(result.outgoing.text).toBe('https://bittorrented[.]com/x and nichedb.dev\n'); + expect(result.outgoing.html).toBe('x'); + }); + + it('hands back the same message when nothing is blocked', async () => { + const result = await defangBlockedHosts({ ...outgoing, text: 'nichedb.dev only\n' }, async () => false); + expect(result.defanged).toEqual([]); + expect(result.outgoing).toBe(outgoing === result.outgoing ? outgoing : result.outgoing); + expect(result.outgoing.text).toBe('nichedb.dev only\n'); + }); + + it('applies only through Forward Email SMTP, and before the send', async () => { + const fe = account({ smtp: { host: 'smtp.forwardemail.net', port: 465, secure: true } }); + const seen: string[] = []; + const result = await sendMail(fe, { ...outgoing, text: 'read bittorrented.com today\n' }, { + smtp: async (_account, message) => { + seen.push(message.text); + return 'm-1'; + }, + isBlocked: blocked, + }); + expect(seen).toEqual(['read bittorrented[.]com today\n']); + expect(result).toEqual({ transport: 'smtp', id: 'm-1', defanged: ['bittorrented.com'] }); + + expect(smtpFiltersHostnames(fe)).toBe(true); + expect(smtpFiltersHostnames(account())).toBe(false); + let lookups = 0; + const other = await sendMail(account(), { ...outgoing, text: 'read bittorrented.com today\n' }, { + smtp: async (_account, message) => message.text, + isBlocked: async () => { + lookups += 1; + return true; + }, + }); + expect(lookups).toBe(0); + expect(other).toEqual({ transport: 'smtp', id: 'read bittorrented.com today\n' }); + }); + }); + it('classifies pipe failures apart from message refusals', () => { expect(isTransportFailure('Invalid login: 535 5.7.8 Authentication failed')).toBe(true); expect(isTransportFailure('getaddrinfo ENOTFOUND smtp.example.com')).toBe(true);