fix: protect plugin requests against ssrf and reach data-fair with private agents

api/package.json

@@ -20,9 +20,9 @@
     "@types/ws": "^8.5.12"
   },
   "dependencies": {
-    "@data-fair/lib-express": "~1.26.0",
-    "@data-fair/lib-node": "~2.13.3",
-    "@data-fair/lib-node-registry": "0.7.1",
+    "@data-fair/lib-express": "~1.28.1",
+    "@data-fair/lib-node": "~2.15.0",
+    "@data-fair/lib-node-registry": "0.7.2",
     "@data-fair/lib-utils": "^1.13.1",
     "@data-fair/lib-validation": "^1.0.2",
     "@data-fair/processings-shared": "*",

api/src/processings/router.ts

@@ -15,6 +15,7 @@ import { reqOrigin, session } from '@data-fair/lib-express/index.js'
 import { ensureArtefact } from '@data-fair/lib-node-registry'
 import { httpError } from '@data-fair/lib-utils/http-errors.js'
 import { axiosBuilder } from '@data-fair/lib-node/axios.js'
+import { privateHttpAgent, privateHttpsAgent } from '@data-fair/lib-node/http-agents.js'
 import { createNext } from '@data-fair/processings-shared/runs.ts'
 import { importPluginModule } from '@data-fair/processings-shared/plugin-load.ts'
 import { applyProcessing, deleteProcessing } from '../runs/service.ts'
@@ -112,6 +113,9 @@ async function ensurePluginAndReadSchema (processing: Pick<Processing, 'plugin'
 async function fetchPluginArtefact (processing: Pick<Processing, 'plugin' | 'owner'>) {
   const ax = axiosBuilder({
     baseURL: config.privateRegistryUrl,
+    // the registry is a service of our own infrastructure
+    httpAgent: privateHttpAgent,
+    httpsAgent: privateHttpsAgent,
     headers: {
       'x-secret-key': config.secretKeys.registry,
       'x-account': JSON.stringify(registryAccount(processing.owner))

dev/init-env.sh

@@ -27,4 +27,8 @@ EVENTS_PORT=$((RANDOM_NB + 31))
 OAV_PORT=$((RANDOM_NB + 32))
 DF_PORT=$((RANDOM_NB + 33))
 REGISTRY_PORT=$((RANDOM_NB + 34))
+
+# all dev services listen on the loopback, the SSRF protection of the default http agents
+# would refuse them (cf @data-fair/lib-node http-agents)
+SSRF_PUBLIC_IPS=127.0.0.1,::1
 EOF

package.json

@@ -51,7 +51,7 @@
   "devDependencies": {
     "@commitlint/config-conventional": "^21.2.3",
     "@data-fair/lib-common-types": "^1.23.0",
-    "@data-fair/lib-node": "~2.13.3",
+    "@data-fair/lib-node": "~2.15.0",
     "@data-fair/lib-utils": "^1.13.1",
     "@playwright/test": "^1.52.0",
     "@types/config": "^3.3.5",

shared/package.json

@@ -2,7 +2,7 @@
   "name": "@data-fair/processings-shared",
   "type": "module",
   "dependencies": {
-    "@data-fair/lib-node": "~2.13.3",
+    "@data-fair/lib-node": "~2.15.0",
     "@data-fair/lib-utils": "^1.13.1",
     "croner": "^9.1.0",
     "dayjs": "^1.11.13",

tests/features/worker-utils/data-fair-url.unit.spec.ts

@@ -0,0 +1,42 @@
+import { test, expect } from '@playwright/test'
+import { resolveDataFairUrl } from '../../../worker/src/task/data-fair-url.ts'
+
+test.describe('resolveDataFairUrl', () => {
+  const dataFairUrl = 'https://df.example.com/data-fair'
+
+  test('matches URLs inside the data-fair base path', () => {
+    expect(resolveDataFairUrl('https://df.example.com/data-fair', dataFairUrl)).toBe('https://df.example.com/data-fair')
+    expect(resolveDataFairUrl('https://df.example.com/data-fair/api/v1/datasets?q=a', dataFairUrl)).toBe('https://df.example.com/data-fair/api/v1/datasets?q=a')
+  })
+
+  test('rejects lookalike URLs that a string prefix would accept', () => {
+    const rootDataFairUrl = 'https://df.example.com'
+    expect(resolveDataFairUrl('https://df.example.com.evil.com/api/v1/datasets', rootDataFairUrl)).toBeUndefined()
+    expect(resolveDataFairUrl('https://df.example.com@evil.com/api/v1/datasets', rootDataFairUrl)).toBeUndefined()
+    expect(resolveDataFairUrl('https://df.example.com:8443/api/v1/datasets', rootDataFairUrl)).toBeUndefined()
+    expect(resolveDataFairUrl('https://df.example.com/data-fair-other/api', dataFairUrl)).toBeUndefined()
+    expect(resolveDataFairUrl('https://df.example.com/data-fair/../other', dataFairUrl)).toBeUndefined()
+    expect(resolveDataFairUrl('http://df.example.com/data-fair/api', dataFairUrl)).toBeUndefined()
+    expect(resolveDataFairUrl('https://df.example.com/data-fair@evil.example/x', dataFairUrl)).toBeUndefined()
+  })
+
+  test('never lets a path inject an authority in the private URL', () => {
+    // a string replace of the public base by the private one would give http://data-fair:8080@evil.example/x
+    expect(resolveDataFairUrl('https://df.example.com/@evil.example/x', 'https://df.example.com', 'http://data-fair:8080'))
+      .toBe('http://data-fair:8080/@evil.example/x')
+    expect(new URL(resolveDataFairUrl('https://df.example.com/data-fair/@169.254.169.254/latest', dataFairUrl, 'http://data-fair:8080') as string).host)
+      .toBe('data-fair:8080')
+  })
+
+  test('matches every path of a data-fair served at the root of its domain', () => {
+    expect(resolveDataFairUrl('https://df.example.com/api/v1/datasets', 'https://df.example.com/')).toBe('https://df.example.com/api/v1/datasets')
+  })
+
+  test('rewrites to the private data-fair URL', () => {
+    expect(resolveDataFairUrl('https://df.example.com/data-fair/api/v1/datasets?q=a', dataFairUrl, 'http://data-fair:8080'))
+      .toBe('http://data-fair:8080/api/v1/datasets?q=a')
+    expect(resolveDataFairUrl('https://df.example.com/data-fair/api', dataFairUrl, 'http://data-fair:8080/data-fair/'))
+      .toBe('http://data-fair:8080/data-fair/api')
+    expect(resolveDataFairUrl('https://other.com/data-fair/api', dataFairUrl, 'http://data-fair:8080')).toBeUndefined()
+  })
+})

ui/package.json

@@ -42,6 +42,6 @@
     "vuetify": "^4.0.3"
   },
   "devDependencies": {
-    "@data-fair/lib-express": "~1.26.0"
+    "@data-fair/lib-express": "~1.28.1"
   }
 }

worker/package.json

@@ -12,13 +12,13 @@
     "#types": "./src/types.ts"
   },
   "devDependencies": {
-    "@data-fair/lib-express": "~1.26.0",
+    "@data-fair/lib-express": "~1.28.1",
     "@types/nodemailer": "^8.0.1",
     "@types/resolve-path": "^1.4.3"
   },
   "dependencies": {
-    "@data-fair/lib-node": "~2.13.3",
-    "@data-fair/lib-node-registry": "0.7.1",
+    "@data-fair/lib-node": "~2.15.0",
+    "@data-fair/lib-node-registry": "0.7.2",
     "@data-fair/lib-validation": "^1.0.2",
     "@data-fair/processings-shared": "*",
     "@mdi/js": "^7.4.47",

worker/src/task/axios.ts

@@ -1,10 +1,11 @@
-import { httpAgent, httpsAgent } from '@data-fair/lib-node/http-agents.js'
+import { httpAgent, httpsAgent, privateHttpAgent, privateHttpsAgent } from '@data-fair/lib-node/http-agents.js'
 import axios from 'axios'
 import axiosRetry from 'axios-retry'
 import config from '#config'
 import type { Processing } from '#api/types'
 import type { LogFunctions } from '@data-fair/lib-common-types/processings.js'
 import { prepareAxiosError, getHttpErrorMessage as getHttpErrorMessageBase } from './axios-errors.ts'
+import { resolveDataFairUrl } from './data-fair-url.ts'
 
 export { prepareAxiosError } from './axios-errors.ts'
 
@@ -30,6 +31,7 @@ export const getAxiosInstance = (processing: Processing, log: LogFunctions) => {
   const axiosInstance = axios.create({
     // this is necessary to prevent excessive memory usage during large file uploads, see https://github.com/axios/axios/issues/1045
     maxRedirects: 0,
+    // public agents: they refuse non public addresses (SSRF protection), the URLs come from processing configs
     httpAgent,
     httpsAgent
   })
@@ -41,13 +43,19 @@ export const getAxiosInstance = (processing: Processing, log: LogFunctions) => {
       if (cfg.url.startsWith('/')) cfg.url = config.dataFairUrl + cfg.url
       else cfg.url = config.dataFairUrl + '/' + cfg.url
     }
-    const isDataFairUrl = cfg.url.startsWith(config.dataFairUrl)
-    if (isDataFairUrl) Object.assign(cfg.headers, privateHeaders)
-
-    // always route data-fair requests through the private url to stay within internal infrastructure
-    if (isDataFairUrl && config.privateDataFairUrl) {
-      cfg.url = cfg.url.replace(config.dataFairUrl, config.privateDataFairUrl)
-      cfg.headers.host = new URL(config.dataFairUrl).host
+    const dataFairUrl = resolveDataFairUrl(cfg.url, config.dataFairUrl, config.privateDataFairUrl)
+    if (dataFairUrl) {
+      Object.assign(cfg.headers, privateHeaders)
+      // data-fair is a service of our own infrastructure, it can be reached on a private address
+      cfg.httpAgent = privateHttpAgent
+      cfg.httpsAgent = privateHttpsAgent
+      // a redirection would keep the private agents (and the api key) towards any host
+      cfg.maxRedirects = 0
+      // always route data-fair requests through the private url to stay within internal infrastructure
+      if (config.privateDataFairUrl) {
+        cfg.url = dataFairUrl
+        cfg.headers.host = new URL(config.dataFairUrl).host
+      }
     }
     return cfg
   }, error => Promise.reject(error))

worker/src/task/data-fair-url.ts

@@ -0,0 +1,18 @@
+/**
+ * Match a request URL against the configured data-fair URL.
+ * Data-fair requests receive the data-fair API key and may reach a private address, so this is a
+ * parsed comparison (same origin, path inside the base path) and not a string prefix:
+ * https://df.example.com.evil.com or https://df.example.com@evil.com must not match https://df.example.com.
+ * Returns undefined if the URL is not a data-fair URL, else the URL to request (rewritten to the
+ * private data-fair URL if there is one).
+ */
+export const resolveDataFairUrl = (url: string, dataFairUrl: string, privateDataFairUrl?: string | null): string | undefined => {
+  const base = new URL(dataFairUrl)
+  const target = new URL(url)
+  if (target.origin !== base.origin) return
+  const basePath = base.pathname.replace(/\/$/, '')
+  if (target.pathname !== basePath && !target.pathname.startsWith(basePath + '/')) return
+  if (!privateDataFairUrl) return url
+  const privateBase = new URL(privateDataFairUrl)
+  return privateBase.origin + privateBase.pathname.replace(/\/$/, '') + target.pathname.slice(basePath.length) + target.search
+}

worker/src/task/index.ts

@@ -20,7 +20,8 @@ process.on('SIGTERM', function onSigterm () {
 })
 
 await mongo.init()
-const mailTransport = nodemailer.createTransport(config.mails.transport)
+// plugins compose the whole message: forbid attachments and contents read from local files or fetched from URLs
+const mailTransport = nodemailer.createTransport({ ...config.mails.transport, disableFileAccess: true, disableUrlAccess: true })
 await wsEmitter.init(mongo.db)
 
 const err = await run(mailTransport)