diff --git a/bin/mail.ts b/bin/mail.ts index 27b4227..93be903 100755 --- a/bin/mail.ts +++ b/bin/mail.ts @@ -40,6 +40,7 @@ import { buildReply, chooseTransport, composeRaw, + defangHost, folderFor, formatAccounts, formatFolders, @@ -717,6 +718,11 @@ async function main(argv: string[]): Promise { `mail: ${result.fellBackFrom.transport} failed (${result.fellBackFrom.error}); sent via ${result.transport}\n`, ); } + if (result.defanged && result.defanged.length > 0) { + process.stderr.write( + `mail: defanged ${result.defanged.length} hostname${result.defanged.length === 1 ? '' : 's'} Forward Email refuses (Cloudflare Family DNS): ${result.defanged.map(defangHost).join(', ')}\n`, + ); + } // SMTP servers file their own Sent copy; Resend never does. let filed: string | null = null; if (result.transport === 'resend') { @@ -729,7 +735,7 @@ async function main(argv: string[]): Promise { const uid = needUids(rest, 'reply')[0]!; await withMailbox(account, (box) => box.flag(folder, [uid], ['\\Answered'], [])).catch(() => undefined); } - if (isJson) json({ transport: result.transport, id: result.id, to: outgoing.to, subject: outgoing.subject, filed }); + if (isJson) json({ transport: result.transport, id: result.id, to: outgoing.to, subject: outgoing.subject, filed, defanged: result.defanged ?? [] }); else { out(`sent via ${result.transport} to ${outgoing.to.join(', ')}: ${outgoing.subject}${result.id ? ` (${result.id})` : ''}`); if (filed) out(`copy filed in ${filed}`); diff --git a/package.json b/package.json index 02a7043..12fa086 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/cli-tools", - "version": "0.36.1", + "version": "0.37.0", "private": true, "description": "Local command-line tools, in TypeScript, exposed on PATH.", "type": "module", diff --git a/src/mail.ts b/src/mail.ts index abde4f2..d852f35 100644 --- a/src/mail.ts +++ b/src/mail.ts @@ -26,7 +26,7 @@ * {@link Mailbox} interface so tests never open a socket. */ -import { resolveMx } from 'node:dns/promises'; +import { Resolver, resolveMx } from 'node:dns/promises'; import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -1419,6 +1419,107 @@ export async function composeRaw(outgoing: Outgoing): Promise { return composer.compile().build(); } +// --------------------------------------------------------------------------- +// Hostnames Forward Email refuses +// --------------------------------------------------------------------------- + +/** + * Forward Email runs every hostname in an outbound message past Cloudflare's + * Family DNS and refuses the whole message when one is blocked: + * + * 554 5.6.0 Link hostname of bittorrented.com was detected by Cloudflare's + * Family DNS to contain adult-related content, phishing, and/or malware. + * + * A bare mention counts, not just a link, and a few refusals inside a rolling + * window suspend the account's outbound queue until their support lifts it by + * hand. Our own torrent site in a report title was enough to do it. + * + * So before a message leaves through Forward Email, every hostname in it is + * resolved at 1.1.1.3, and one that answers 0.0.0.0 (the block answer) is + * defanged in place: `bittorrented.com` becomes `bittorrented[.]com`, which a + * reader understands and the filter does not see. Addresses are left alone; + * the check is on link hostnames, and a defanged address would bounce. + */ + +export const FAMILY_DNS = '1.1.1.3'; + +/** SMTP hosts known to run the Family DNS check on what they relay. */ +const FAMILY_DNS_FILTERED_SMTP = /(^|\.)forwardemail\.net$/i; + +export function smtpFiltersHostnames(account: Pick): boolean { + return FAMILY_DNS_FILTERED_SMTP.test(account.smtp.host); +} + +/** Answers true when Family DNS blocks the host. */ +export type HostBlockCheck = (host: string) => Promise; + +/** Hostname-shaped tokens: labels, then a letters-only top level. Not preceded by `@` (an address) or a dot (already inside a longer name). */ +const HOSTNAME_RE = /(?(); + for (const match of text.matchAll(HOSTNAME_RE)) out.add(match[1]!.toLowerCase()); + return [...out]; +} + +/** The resolver check, against 1.1.1.3 by default. A host it cannot resolve at all is not blocked, it is unknown, and goes as written. */ +export function familyDnsBlockCheck(server: string = FAMILY_DNS): HostBlockCheck { + const resolver = new Resolver({ timeout: 3000, tries: 1 }); + resolver.setServers([server]); + return async (host) => { + try { + const addresses = await resolver.resolve4(host); + return addresses.includes('0.0.0.0'); + } catch { + return false; + } + }; +} + +export function defangHost(host: string): string { + return host.replace(/\./g, '[.]'); +} + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** `hosts` defanged wherever they appear as a whole name; longest first, so `www.x.com` is done before `x.com` finds its tail. */ +export function defangHosts(text: string, hosts: string[]): string { + let out = text; + for (const host of [...hosts].sort((a, b) => b.length - a.length)) { + const re = new RegExp(`(? { + const hosts = hostnamesIn([outgoing.subject, outgoing.text, outgoing.html ?? ''].join('\n')).slice(0, HOST_LOOKUP_CAP); + const verdicts = await Promise.all(hosts.map(async (host) => [host, await isBlocked(host)] as const)); + const blocked = verdicts.filter(([, yes]) => yes).map(([host]) => host).sort(); + if (blocked.length === 0) return { outgoing, defanged: [] }; + return { + outgoing: { + ...outgoing, + subject: defangHosts(outgoing.subject, blocked), + text: defangHosts(outgoing.text, blocked), + ...(outgoing.html ? { html: defangHosts(outgoing.html, blocked) } : {}), + }, + defanged: blocked, + }; +} + // --------------------------------------------------------------------------- // Sending // --------------------------------------------------------------------------- @@ -1492,6 +1593,8 @@ export interface SendResult { id: string | null; /** Set when the first transport failed and the second carried it. */ fellBackFrom?: { transport: Transport; error: string }; + /** Hostnames Forward Email would have refused, defanged before the send. */ + defanged?: string[]; } export type SmtpSender = (account: Account, outgoing: Outgoing) => Promise; @@ -1591,27 +1694,39 @@ export async function sendMail( via?: Transport; smtp?: SmtpSender; resend?: ResendSender; + /** The Family DNS check, for a test; the real resolver otherwise. */ + isBlocked?: HostBlockCheck; } = {}, ): Promise { const choice = chooseTransport(account, options.resendKey, options.via); const smtp = options.smtp ?? sendViaSmtp; const resend = options.resend ?? resendSender(); + // Through Forward Email, a blocked hostname is defanged first or the message + // never leaves; the fallback (if taken) carries the same defanged copy. + let message = outgoing; + let defanged: string[] = []; + if (choice.transport === 'smtp' && smtpFiltersHostnames(account)) { + ({ outgoing: message, defanged } = await defangBlockedHosts(outgoing, options.isBlocked ?? familyDnsBlockCheck())); + } + const tail = defanged.length > 0 ? { defanged } : {}; + const attempt = async (transport: Transport): Promise => - transport === 'smtp' ? smtp(account, outgoing) : resend(options.resendKey!, outgoing); + transport === 'smtp' ? smtp(account, message) : resend(options.resendKey!, message); try { - return { transport: choice.transport, id: await attempt(choice.transport) }; + return { transport: choice.transport, id: await attempt(choice.transport), ...tail }; } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (!choice.fallback || !isTransportFailure(message)) { - throw error instanceof MailError ? error : new MailError(`${choice.transport}: ${message}`); + const reason = error instanceof Error ? error.message : String(error); + if (!choice.fallback || !isTransportFailure(reason)) { + throw error instanceof MailError ? error : new MailError(`${choice.transport}: ${reason}`); } const id = await attempt(choice.fallback); return { transport: choice.fallback, id, - fellBackFrom: { transport: choice.transport, error: message }, + fellBackFrom: { transport: choice.transport, error: reason }, + ...tail, }; } } diff --git a/test/mail.test.ts b/test/mail.test.ts index 8b12dc2..681fbac 100644 --- a/test/mail.test.ts +++ b/test/mail.test.ts @@ -17,6 +17,9 @@ import { buildReply, chooseTransport, composeRaw, + defangBlockedHosts, + defangHost, + defangHosts, folderFor, formatAccounts, formatAddresses, @@ -25,6 +28,7 @@ import { formatProviders, fromHeader, guessProvider, + hostnamesIn, isProviderName, isTransportFailure, loadConfig, @@ -45,6 +49,7 @@ import { selectAccount, selectAccounts, sendMail, + smtpFiltersHostnames, senderName, splitAddresses, stripHtml, @@ -552,6 +557,72 @@ describe('sendMail', () => { ).rejects.toThrow(/ETIMEDOUT/); }); + // Forward Email refuses a message that so much as mentions a hostname + // Cloudflare's Family DNS blocks, and a few refusals suspend the account. + describe('hostnames Forward Email refuses', () => { + const blocked = async (host: string) => host.endsWith('bittorrented.com'); + + it('finds the hostname-shaped tokens in a body, once each, addresses excluded', () => { + expect( + hostnamesIn( + 'New on https://www.bittorrented.com/blog and bittorrented.com/dht; mail bot@bittorrented.com. Also nichedb.dev, v1.2.3 and e.g. this.', + ), + ).toEqual(['www.bittorrented.com', 'bittorrented.com', 'nichedb.dev']); + }); + + it('defangs every dot of a blocked host, longest name first, and leaves the rest', () => { + expect(defangHost('www.bittorrented.com')).toBe('www[.]bittorrented[.]com'); + expect(defangHosts('see www.bittorrented.com and bittorrented.com, not bittorrented.community', ['bittorrented.com', 'www.bittorrented.com'])).toBe( + 'see www[.]bittorrented[.]com and bittorrented[.]com, not bittorrented.community', + ); + }); + + it('rewrites subject, text and html, and says which hosts it touched', async () => { + const result = await defangBlockedHosts( + { ...outgoing, subject: 'bittorrented.com is down', text: 'https://bittorrented.com/x and nichedb.dev\n', html: 'x' }, + blocked, + ); + expect(result.defanged).toEqual(['bittorrented.com']); + expect(result.outgoing.subject).toBe('bittorrented[.]com is down'); + expect(result.outgoing.text).toBe('https://bittorrented[.]com/x and nichedb.dev\n'); + expect(result.outgoing.html).toBe('x'); + }); + + it('hands back the same message when nothing is blocked', async () => { + const result = await defangBlockedHosts({ ...outgoing, text: 'nichedb.dev only\n' }, async () => false); + expect(result.defanged).toEqual([]); + expect(result.outgoing).toBe(outgoing === result.outgoing ? outgoing : result.outgoing); + expect(result.outgoing.text).toBe('nichedb.dev only\n'); + }); + + it('applies only through Forward Email SMTP, and before the send', async () => { + const fe = account({ smtp: { host: 'smtp.forwardemail.net', port: 465, secure: true } }); + const seen: string[] = []; + const result = await sendMail(fe, { ...outgoing, text: 'read bittorrented.com today\n' }, { + smtp: async (_account, message) => { + seen.push(message.text); + return 'm-1'; + }, + isBlocked: blocked, + }); + expect(seen).toEqual(['read bittorrented[.]com today\n']); + expect(result).toEqual({ transport: 'smtp', id: 'm-1', defanged: ['bittorrented.com'] }); + + expect(smtpFiltersHostnames(fe)).toBe(true); + expect(smtpFiltersHostnames(account())).toBe(false); + let lookups = 0; + const other = await sendMail(account(), { ...outgoing, text: 'read bittorrented.com today\n' }, { + smtp: async (_account, message) => message.text, + isBlocked: async () => { + lookups += 1; + return true; + }, + }); + expect(lookups).toBe(0); + expect(other).toEqual({ transport: 'smtp', id: 'read bittorrented.com today\n' }); + }); + }); + it('classifies pipe failures apart from message refusals', () => { expect(isTransportFailure('Invalid login: 535 5.7.8 Authentication failed')).toBe(true); expect(isTransportFailure('getaddrinfo ENOTFOUND smtp.example.com')).toBe(true);