diff --git a/api/package.json b/api/package.json index 7455c89d..b702de27 100644 --- a/api/package.json +++ b/api/package.json @@ -20,9 +20,9 @@ "@types/ws": "^8.5.12" }, "dependencies": { - "@data-fair/lib-express": "~1.26.0", - "@data-fair/lib-node": "~2.13.3", - "@data-fair/lib-node-registry": "0.7.1", + "@data-fair/lib-express": "~1.28.1", + "@data-fair/lib-node": "~2.15.0", + "@data-fair/lib-node-registry": "0.7.2", "@data-fair/lib-utils": "^1.13.1", "@data-fair/lib-validation": "^1.0.2", "@data-fair/processings-shared": "*", diff --git a/api/src/processings/router.ts b/api/src/processings/router.ts index f3063dba..912dfa46 100644 --- a/api/src/processings/router.ts +++ b/api/src/processings/router.ts @@ -15,6 +15,7 @@ import { reqOrigin, session } from '@data-fair/lib-express/index.js' import { ensureArtefact } from '@data-fair/lib-node-registry' import { httpError } from '@data-fair/lib-utils/http-errors.js' import { axiosBuilder } from '@data-fair/lib-node/axios.js' +import { privateHttpAgent, privateHttpsAgent } from '@data-fair/lib-node/http-agents.js' import { createNext } from '@data-fair/processings-shared/runs.ts' import { importPluginModule } from '@data-fair/processings-shared/plugin-load.ts' import { applyProcessing, deleteProcessing } from '../runs/service.ts' @@ -112,6 +113,9 @@ async function ensurePluginAndReadSchema (processing: Pick) { const ax = axiosBuilder({ baseURL: config.privateRegistryUrl, + // the registry is a service of our own infrastructure + httpAgent: privateHttpAgent, + httpsAgent: privateHttpsAgent, headers: { 'x-secret-key': config.secretKeys.registry, 'x-account': JSON.stringify(registryAccount(processing.owner)) diff --git a/dev/init-env.sh b/dev/init-env.sh index da2a1946..6ae3aa37 100755 --- a/dev/init-env.sh +++ b/dev/init-env.sh @@ -27,4 +27,8 @@ EVENTS_PORT=$((RANDOM_NB + 31)) OAV_PORT=$((RANDOM_NB + 32)) DF_PORT=$((RANDOM_NB + 33)) REGISTRY_PORT=$((RANDOM_NB + 34)) + +# all dev services listen on the loopback, the SSRF protection of the default http agents +# would refuse them (cf @data-fair/lib-node http-agents) +SSRF_PUBLIC_IPS=127.0.0.1,::1 EOF diff --git a/package-lock.json b/package-lock.json index ca32b491..00b964b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "devDependencies": { "@commitlint/config-conventional": "^21.2.3", "@data-fair/lib-common-types": "^1.23.0", - "@data-fair/lib-node": "~2.13.3", + "@data-fair/lib-node": "~2.15.0", "@data-fair/lib-utils": "^1.13.1", "@playwright/test": "^1.52.0", "@types/config": "^3.3.5", @@ -44,9 +44,9 @@ }, "api": { "dependencies": { - "@data-fair/lib-express": "~1.26.0", - "@data-fair/lib-node": "~2.13.3", - "@data-fair/lib-node-registry": "0.7.1", + "@data-fair/lib-express": "~1.28.1", + "@data-fair/lib-node": "~2.15.0", + "@data-fair/lib-node-registry": "0.7.2", "@data-fair/lib-utils": "^1.13.1", "@data-fair/lib-validation": "^1.0.2", "@data-fair/processings-shared": "*", @@ -504,9 +504,9 @@ } }, "node_modules/@data-fair/lib-express": { - "version": "1.26.0", - "resolved": "https://registry.npmjs.org/@data-fair/lib-express/-/lib-express-1.26.0.tgz", - "integrity": "sha512-sSPl09vhgaMfAIqZXVcbcCWT6T8G07R/5BBWqunVG1M9PeEWA0R2gShikQmNJM4kKE6Jqe2e0qmXaqzqcNPflg==", + "version": "1.28.1", + "resolved": "https://registry.npmjs.org/@data-fair/lib-express/-/lib-express-1.28.1.tgz", + "integrity": "sha512-WVNwEBOPJgwb/1Z/IIJ+Byl5/8mX3jBBTzXWb+c/TvYktkUXexNShyCtaCRy1zcYkAQqCuvGrQ1srg6ur5ohUw==", "license": "MIT", "dependencies": { "@data-fair/lib-common-types": "^1.7.1", @@ -517,7 +517,7 @@ "serialize-javascript": "^7.0.0" }, "peerDependencies": { - "@data-fair/lib-node": "^2.13.1", + "@data-fair/lib-node": "^2.15.0", "express": "5", "ws": "8" }, @@ -546,9 +546,9 @@ } }, "node_modules/@data-fair/lib-node": { - "version": "2.13.3", - "resolved": "https://registry.npmjs.org/@data-fair/lib-node/-/lib-node-2.13.3.tgz", - "integrity": "sha512-djpe8B1Ri2TOVWehAXfIdNzHdZEVa/z7Bw9LOiDm6/S8UifPQGepKZCIhNVeKj4AwQ4e706dsyB+wGUOy3ehAw==", + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/@data-fair/lib-node/-/lib-node-2.15.0.tgz", + "integrity": "sha512-eNnjYry7NxbViNCbADz76daKSW7/zxMffTvYMxtnnXbNbZcR6rCjq05BzxLZf8hYIoqgYTMfbBR6jUh5tu5HbA==", "license": "MIT", "dependencies": { "@data-fair/lib-common-types": "^1.8.4", @@ -558,6 +558,7 @@ "axios": "^1.7.7", "cacheable-lookup": "^7.0.0", "cookie": "^0.7.2", + "ipaddr.js": "^2.5.0", "jsonwebtoken": "^9.0.2", "jwks-rsa": "^3.1.0", "semver": "^7.6.3" @@ -584,9 +585,9 @@ } }, "node_modules/@data-fair/lib-node-registry": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/@data-fair/lib-node-registry/-/lib-node-registry-0.7.1.tgz", - "integrity": "sha512-bdS2AnVOXYVpQ7gFtGupSd3fb8DvKifQGqihuCcIvRqr2HsFEau0am1SYvLWH7DRCQutSEqVLKfuTyx0iegTHA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@data-fair/lib-node-registry/-/lib-node-registry-0.7.2.tgz", + "integrity": "sha512-yIQ/oApKk/T3WA8no5nvIWOyGggQVQFszz0tVesHfia9KZYpTC1BKTeVwnH8PWJFYKKuZudwZ9K7YWzOGqX5ug==", "license": "MIT", "dependencies": { "@types/resolve-path": "^1.4.3", @@ -595,7 +596,7 @@ "tar-stream": "^3.1.0" }, "peerDependencies": { - "@data-fair/lib-node": ">=2.8.0" + "@data-fair/lib-node": ">=2.15.0" } }, "node_modules/@data-fair/lib-types-builder": { @@ -6798,6 +6799,15 @@ "node": ">= 0.4" } }, + "node_modules/ipaddr.js": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.5.0.tgz", + "integrity": "sha512-aq+t5NAc+cS6rZQQVWC2x98CPqGtKKTMDd4Gaodv0wShnItdKg/51djkGJ1hqH+Oy0ivDftCbSLCQob8zso01w==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, "node_modules/is-array-buffer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", @@ -12185,7 +12195,7 @@ "shared": { "name": "@data-fair/processings-shared", "dependencies": { - "@data-fair/lib-node": "~2.13.3", + "@data-fair/lib-node": "~2.15.0", "@data-fair/lib-utils": "^1.13.1", "croner": "^9.1.0", "dayjs": "^1.11.13", @@ -12242,13 +12252,13 @@ "vuetify": "^4.0.3" }, "devDependencies": { - "@data-fair/lib-express": "~1.26.0" + "@data-fair/lib-express": "~1.28.1" } }, "worker": { "dependencies": { - "@data-fair/lib-node": "~2.13.3", - "@data-fair/lib-node-registry": "0.7.1", + "@data-fair/lib-node": "~2.15.0", + "@data-fair/lib-node-registry": "0.7.2", "@data-fair/lib-validation": "^1.0.2", "@data-fair/processings-shared": "*", "@mdi/js": "^7.4.47", @@ -12265,7 +12275,7 @@ "tree-kill": "^1.2.2" }, "devDependencies": { - "@data-fair/lib-express": "~1.26.0", + "@data-fair/lib-express": "~1.28.1", "@types/nodemailer": "^8.0.1", "@types/resolve-path": "^1.4.3" } diff --git a/package.json b/package.json index 92b4dded..4edbc3fe 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "devDependencies": { "@commitlint/config-conventional": "^21.2.3", "@data-fair/lib-common-types": "^1.23.0", - "@data-fair/lib-node": "~2.13.3", + "@data-fair/lib-node": "~2.15.0", "@data-fair/lib-utils": "^1.13.1", "@playwright/test": "^1.52.0", "@types/config": "^3.3.5", diff --git a/shared/package.json b/shared/package.json index 7ff2d4ff..fb944080 100644 --- a/shared/package.json +++ b/shared/package.json @@ -2,7 +2,7 @@ "name": "@data-fair/processings-shared", "type": "module", "dependencies": { - "@data-fair/lib-node": "~2.13.3", + "@data-fair/lib-node": "~2.15.0", "@data-fair/lib-utils": "^1.13.1", "croner": "^9.1.0", "dayjs": "^1.11.13", diff --git a/tests/features/worker-utils/data-fair-url.unit.spec.ts b/tests/features/worker-utils/data-fair-url.unit.spec.ts new file mode 100644 index 00000000..19bec4b4 --- /dev/null +++ b/tests/features/worker-utils/data-fair-url.unit.spec.ts @@ -0,0 +1,42 @@ +import { test, expect } from '@playwright/test' +import { resolveDataFairUrl } from '../../../worker/src/task/data-fair-url.ts' + +test.describe('resolveDataFairUrl', () => { + const dataFairUrl = 'https://df.example.com/data-fair' + + test('matches URLs inside the data-fair base path', () => { + expect(resolveDataFairUrl('https://df.example.com/data-fair', dataFairUrl)).toBe('https://df.example.com/data-fair') + expect(resolveDataFairUrl('https://df.example.com/data-fair/api/v1/datasets?q=a', dataFairUrl)).toBe('https://df.example.com/data-fair/api/v1/datasets?q=a') + }) + + test('rejects lookalike URLs that a string prefix would accept', () => { + const rootDataFairUrl = 'https://df.example.com' + expect(resolveDataFairUrl('https://df.example.com.evil.com/api/v1/datasets', rootDataFairUrl)).toBeUndefined() + expect(resolveDataFairUrl('https://df.example.com@evil.com/api/v1/datasets', rootDataFairUrl)).toBeUndefined() + expect(resolveDataFairUrl('https://df.example.com:8443/api/v1/datasets', rootDataFairUrl)).toBeUndefined() + expect(resolveDataFairUrl('https://df.example.com/data-fair-other/api', dataFairUrl)).toBeUndefined() + expect(resolveDataFairUrl('https://df.example.com/data-fair/../other', dataFairUrl)).toBeUndefined() + expect(resolveDataFairUrl('http://df.example.com/data-fair/api', dataFairUrl)).toBeUndefined() + expect(resolveDataFairUrl('https://df.example.com/data-fair@evil.example/x', dataFairUrl)).toBeUndefined() + }) + + test('never lets a path inject an authority in the private URL', () => { + // a string replace of the public base by the private one would give http://data-fair:8080@evil.example/x + expect(resolveDataFairUrl('https://df.example.com/@evil.example/x', 'https://df.example.com', 'http://data-fair:8080')) + .toBe('http://data-fair:8080/@evil.example/x') + expect(new URL(resolveDataFairUrl('https://df.example.com/data-fair/@169.254.169.254/latest', dataFairUrl, 'http://data-fair:8080') as string).host) + .toBe('data-fair:8080') + }) + + test('matches every path of a data-fair served at the root of its domain', () => { + expect(resolveDataFairUrl('https://df.example.com/api/v1/datasets', 'https://df.example.com/')).toBe('https://df.example.com/api/v1/datasets') + }) + + test('rewrites to the private data-fair URL', () => { + expect(resolveDataFairUrl('https://df.example.com/data-fair/api/v1/datasets?q=a', dataFairUrl, 'http://data-fair:8080')) + .toBe('http://data-fair:8080/api/v1/datasets?q=a') + expect(resolveDataFairUrl('https://df.example.com/data-fair/api', dataFairUrl, 'http://data-fair:8080/data-fair/')) + .toBe('http://data-fair:8080/data-fair/api') + expect(resolveDataFairUrl('https://other.com/data-fair/api', dataFairUrl, 'http://data-fair:8080')).toBeUndefined() + }) +}) diff --git a/ui/package.json b/ui/package.json index fca5fef0..35441ffd 100644 --- a/ui/package.json +++ b/ui/package.json @@ -42,6 +42,6 @@ "vuetify": "^4.0.3" }, "devDependencies": { - "@data-fair/lib-express": "~1.26.0" + "@data-fair/lib-express": "~1.28.1" } } diff --git a/worker/package.json b/worker/package.json index 6491a08d..a9c24e5e 100644 --- a/worker/package.json +++ b/worker/package.json @@ -12,13 +12,13 @@ "#types": "./src/types.ts" }, "devDependencies": { - "@data-fair/lib-express": "~1.26.0", + "@data-fair/lib-express": "~1.28.1", "@types/nodemailer": "^8.0.1", "@types/resolve-path": "^1.4.3" }, "dependencies": { - "@data-fair/lib-node": "~2.13.3", - "@data-fair/lib-node-registry": "0.7.1", + "@data-fair/lib-node": "~2.15.0", + "@data-fair/lib-node-registry": "0.7.2", "@data-fair/lib-validation": "^1.0.2", "@data-fair/processings-shared": "*", "@mdi/js": "^7.4.47", diff --git a/worker/src/task/axios.ts b/worker/src/task/axios.ts index 8213dfb5..ec51611f 100644 --- a/worker/src/task/axios.ts +++ b/worker/src/task/axios.ts @@ -1,10 +1,11 @@ -import { httpAgent, httpsAgent } from '@data-fair/lib-node/http-agents.js' +import { httpAgent, httpsAgent, privateHttpAgent, privateHttpsAgent } from '@data-fair/lib-node/http-agents.js' import axios from 'axios' import axiosRetry from 'axios-retry' import config from '#config' import type { Processing } from '#api/types' import type { LogFunctions } from '@data-fair/lib-common-types/processings.js' import { prepareAxiosError, getHttpErrorMessage as getHttpErrorMessageBase } from './axios-errors.ts' +import { resolveDataFairUrl } from './data-fair-url.ts' export { prepareAxiosError } from './axios-errors.ts' @@ -30,6 +31,7 @@ export const getAxiosInstance = (processing: Processing, log: LogFunctions) => { const axiosInstance = axios.create({ // this is necessary to prevent excessive memory usage during large file uploads, see https://github.com/axios/axios/issues/1045 maxRedirects: 0, + // public agents: they refuse non public addresses (SSRF protection), the URLs come from processing configs httpAgent, httpsAgent }) @@ -41,13 +43,19 @@ export const getAxiosInstance = (processing: Processing, log: LogFunctions) => { if (cfg.url.startsWith('/')) cfg.url = config.dataFairUrl + cfg.url else cfg.url = config.dataFairUrl + '/' + cfg.url } - const isDataFairUrl = cfg.url.startsWith(config.dataFairUrl) - if (isDataFairUrl) Object.assign(cfg.headers, privateHeaders) - - // always route data-fair requests through the private url to stay within internal infrastructure - if (isDataFairUrl && config.privateDataFairUrl) { - cfg.url = cfg.url.replace(config.dataFairUrl, config.privateDataFairUrl) - cfg.headers.host = new URL(config.dataFairUrl).host + const dataFairUrl = resolveDataFairUrl(cfg.url, config.dataFairUrl, config.privateDataFairUrl) + if (dataFairUrl) { + Object.assign(cfg.headers, privateHeaders) + // data-fair is a service of our own infrastructure, it can be reached on a private address + cfg.httpAgent = privateHttpAgent + cfg.httpsAgent = privateHttpsAgent + // a redirection would keep the private agents (and the api key) towards any host + cfg.maxRedirects = 0 + // always route data-fair requests through the private url to stay within internal infrastructure + if (config.privateDataFairUrl) { + cfg.url = dataFairUrl + cfg.headers.host = new URL(config.dataFairUrl).host + } } return cfg }, error => Promise.reject(error)) diff --git a/worker/src/task/data-fair-url.ts b/worker/src/task/data-fair-url.ts new file mode 100644 index 00000000..bfbf7047 --- /dev/null +++ b/worker/src/task/data-fair-url.ts @@ -0,0 +1,18 @@ +/** + * Match a request URL against the configured data-fair URL. + * Data-fair requests receive the data-fair API key and may reach a private address, so this is a + * parsed comparison (same origin, path inside the base path) and not a string prefix: + * https://df.example.com.evil.com or https://df.example.com@evil.com must not match https://df.example.com. + * Returns undefined if the URL is not a data-fair URL, else the URL to request (rewritten to the + * private data-fair URL if there is one). + */ +export const resolveDataFairUrl = (url: string, dataFairUrl: string, privateDataFairUrl?: string | null): string | undefined => { + const base = new URL(dataFairUrl) + const target = new URL(url) + if (target.origin !== base.origin) return + const basePath = base.pathname.replace(/\/$/, '') + if (target.pathname !== basePath && !target.pathname.startsWith(basePath + '/')) return + if (!privateDataFairUrl) return url + const privateBase = new URL(privateDataFairUrl) + return privateBase.origin + privateBase.pathname.replace(/\/$/, '') + target.pathname.slice(basePath.length) + target.search +} diff --git a/worker/src/task/index.ts b/worker/src/task/index.ts index 862a208d..cb0b9138 100644 --- a/worker/src/task/index.ts +++ b/worker/src/task/index.ts @@ -20,7 +20,8 @@ process.on('SIGTERM', function onSigterm () { }) await mongo.init() -const mailTransport = nodemailer.createTransport(config.mails.transport) +// plugins compose the whole message: forbid attachments and contents read from local files or fetched from URLs +const mailTransport = nodemailer.createTransport({ ...config.mails.transport, disableFileAccess: true, disableUrlAccess: true }) await wsEmitter.init(mongo.db) const err = await run(mailTransport)