From 96876779b846f35db2737bd5a31601cdfcd5a84b Mon Sep 17 00:00:00 2001 From: "Shan8851.eth" Date: Thu, 13 Aug 2026 10:53:30 +0100 Subject: [PATCH 1/4] fix: warm RPC providers at bootstrap to close the cold-start detectNetwork race The shared docker-test composite marks the container ready as soon as /health-check (no RPC dependency) returns 200, then vitest's fileParallelism fires every test file's first request concurrently. Each first request per (network, version) tuple constructs a fresh StaticJsonRpcProvider, whose maiden detectNetwork() call schedules a real JSON-RPC round trip on the next tick; in a fresh container network namespace that cold DNS+TLS+RPC round trip can exceed the steady-state 2-attempt retry budget, causing a 404 on exit-payload-state-sync.test.ts (3x on run 31686169287). Warm the same POSClient cache used by request handlers during server bootstrap, before opening the listen socket, so the detectNetwork() round trip fires during boot instead of on the first real request. Bounded by a 5s per-tuple timeout and wrapped so a failed or slow RPC never blocks /health-check from coming up. --- .changeset/warm-rpc-providers-boot.md | 5 +++ src/index.ts | 15 +++++++ src/maticClient.ts | 60 +++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 .changeset/warm-rpc-providers-boot.md diff --git a/.changeset/warm-rpc-providers-boot.md b/.changeset/warm-rpc-providers-boot.md new file mode 100644 index 0000000..cfc823f --- /dev/null +++ b/.changeset/warm-rpc-providers-boot.md @@ -0,0 +1,5 @@ +--- +"proof-generation-api": patch +--- + +The server now warms its RPC provider connections during startup, eliminating first-request network-detection failures immediately after boot. diff --git a/src/index.ts b/src/index.ts index 9a39f34..ea5b006 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ import { HTTPError } from '@polygonlabs/verror'; import type { Logger } from './logger.ts'; +import { warmMaticClients } from './maticClient.ts'; import { createIndexRouter } from './routes/index.ts'; // Attach a per-request child logger to req.log so every log entry for a @@ -68,6 +69,20 @@ export async function startApiServer({ app: Express; logger: Logger; }): Promise { + // Warm the RPC providers before opening the listen socket: /health-check has no + // RPC dependency and is the sole readiness signal the shared docker-test composite + // waits on, so without this a POSClient's maiden detectNetwork() call — a cold + // DNS+TLS+RPC round trip in a fresh container network namespace — can lose the + // race against the steady-state 2-attempt retry budget on a real request. + try { + await warmMaticClients(logger); + } catch (err: unknown) { + logger.warn( + { err: err instanceof Error ? err : new Error(String(err)) }, + 'RPC provider warm-up failed; continuing startup' + ); + } + // Bubble errors calling `listen()` up to callers so they get an async stack trace await new Promise((resolve, reject) => { app.listen(port).once('listening', resolve).once('error', reject); diff --git a/src/maticClient.ts b/src/maticClient.ts index 6b93f20..65d0ef9 100644 --- a/src/maticClient.ts +++ b/src/maticClient.ts @@ -5,6 +5,8 @@ import type { IPOSClientConfig } from '@maticnetwork/maticjs'; import maticJs from '@maticnetwork/maticjs'; import maticJs_Ethers from '@maticnetwork/maticjs-ethers'; +import type { Logger } from './logger.ts'; + import { config } from './config.ts'; const { Converter, POSClient, use } = maticJs; // default export :( @@ -91,3 +93,61 @@ export const initMatic = ( export const convert = async (value: any) => { return Converter.toHex(value); }; + +const WARM_UP_TIMEOUT_MS = 5_000; + +function withTimeout({ ms, promise }: { ms: number; promise: Promise }): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (err: unknown) => { + clearTimeout(timer); + reject(err instanceof Error ? err : new Error(String(err))); + } + ); + }); +} + +export async function warmMaticClients(logger: Logger): Promise { + // Mirrors src/routes/v1.ts's networkDetails map, at the RPC index requests + // start from (config.mainnetRpcIndex / config.testnetRpcIndex). + const warmUpTuples = [ + { + ethereumRPC: config.app.ethereumRPC[config.mainnetRpcIndex], + isMainnet: true, + maticRPC: config.app.maticRPC[config.mainnetRpcIndex], + version: 'v1' + }, + { + ethereumRPC: config.app.sepoliaRPC[config.testnetRpcIndex], + isMainnet: false, + maticRPC: config.app.amoyRPC[config.testnetRpcIndex], + version: 'amoy' + } + ]; + + const results = await Promise.allSettled( + warmUpTuples.map(({ ethereumRPC, isMainnet, maticRPC, version }) => { + if (!maticRPC || !ethereumRPC) { + return Promise.reject(new Error(`no configured RPC endpoint for ${version}`)); + } + return withTimeout({ + ms: WARM_UP_TIMEOUT_MS, + promise: initMatic(isMainnet, version, maticRPC, ethereumRPC) + }); + }) + ); + + results.forEach((result, i) => { + if (result.status === 'rejected') { + logger.warn( + { err: result.reason, version: warmUpTuples[i]?.version }, + 'RPC provider warm-up failed; will retry at request time' + ); + } + }); +} From 0484fa443f6be0f9f05cc3e89e33a21111d019aa Mon Sep 17 00:00:00 2001 From: "Shan8851.eth" Date: Thu, 13 Aug 2026 11:16:01 +0100 Subject: [PATCH 2/4] docs(changelog): blank line after Breaking changes heading (MD022) Pre-existing lint failure from the 2.0.0 changelog render; blocks marking #91 ready. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa447e..a0df5d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - [#89](https://github.com/0xPolygon/proof-generation-api/pull/89) [`58a7e8b`](https://github.com/0xPolygon/proof-generation-api/commit/58a7e8bd4c7212544177f2c1e41ef6e530604c88) Thanks [@shan8851](https://github.com/shan8851)! - Remove the zkEVM proof-generation endpoints (`/zkevm/mainnet` and `/zkevm/testnet` routes) — the underlying chains are sunset and their infrastructure no longer answers. PoS (v1) endpoints are unchanged. ## Breaking changes + - Removed `GET /api/zkevm/{network}/bridge` - Removed `GET /api/zkevm/{network}/merkle-proof` - Removed the `ZKEVM_MAINNET_URL` and `ZKEVM_TESTNET_URL` environment variables — they are no longer recognized configuration From 38e145919c564e5cbca4cade63839dafa3b4ab2b Mon Sep 17 00:00:00 2001 From: "Shan8851.eth" Date: Thu, 13 Aug 2026 11:34:45 +0100 Subject: [PATCH 3/4] fix: await a real root-chain round-trip in RPC warm-up, evict poisoned client cache on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rc.0 proof run (31690399999) showed the fire-and-forget warm-up was insufficient: it logged "server has started" right after init(), but the state-sync test still failed with noNetwork 3s later, with 2 retries failing 1ms apart against an already-rejected cached network promise. ethers v5's StaticJsonRpcProvider only *schedules* detectNetwork() via setTimeout(0) in its constructor, so awaiting init() resolving proves detection started, not that it succeeded. warmMaticClients now drives an actual awaited round-trip through the same cached POSClient instance request handlers reuse — exitUtil.rootChain's getLastChildBlock(), the same call isBlockIncluded() makes on the request path — retrying up to 3 times with a 500ms backoff. Each retry evicts the client from clientCache first and re-inits from scratch, since a resolved init() followed by a failing probe means the provider may be sitting on bad state from the same race, and re-probing the same instance risks repeating it. The overall per-tuple timeout is raised from 5s to 8s to give the retries room while staying well under boot-liveness budgets. --- src/maticClient.ts | 74 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/src/maticClient.ts b/src/maticClient.ts index 65d0ef9..1208fe4 100644 --- a/src/maticClient.ts +++ b/src/maticClient.ts @@ -30,15 +30,24 @@ use(Web3ClientPlugin); // The cache stores Promises so that concurrent requests for the same key share // a single in-flight initialisation rather than racing to create duplicates. // A failed initialisation is evicted from the cache so the next request retries. +// warmMaticClients (below) evicts the same way when a resolved client's first +// real round-trip fails, for the same reason. const clientCache = new Map>>(); +const getCacheKey = ( + isMainnet: boolean, + version: string, + maticRPC: string, + ethereumRPC: string +): string => `${isMainnet ? 'mainnet' : 'testnet'}:${version}:${maticRPC}:${ethereumRPC}`; + export const initMatic = ( isMainnet: boolean, version: string, maticRPC: string, ethereumRPC: string ): Promise> => { - const cacheKey = `${isMainnet ? 'mainnet' : 'testnet'}:${version}:${maticRPC}:${ethereumRPC}`; + const cacheKey = getCacheKey(isMainnet, version, maticRPC, ethereumRPC); const cached = clientCache.get(cacheKey); if (cached) return cached; @@ -94,7 +103,15 @@ export const convert = async (value: any) => { return Converter.toHex(value); }; -const WARM_UP_TIMEOUT_MS = 5_000; +// Bounds the whole warm-up-plus-probe sequence for one tuple, including the +// retry backoffs below. ethers v5's StaticJsonRpcProvider only *schedules* +// detectNetwork() via setTimeout(0) in its constructor — awaiting init() +// resolving proves detection STARTED, not that it SUCCEEDED, so a probe is +// needed. Raised from the original 5s to give 3 backed-off probe attempts +// room, while staying well under boot-liveness budgets. +const WARM_UP_TIMEOUT_MS = 8_000; +const WARM_UP_PROBE_ATTEMPTS = 3; +const WARM_UP_PROBE_BACKOFF_MS = 500; function withTimeout({ ms, promise }: { ms: number; promise: Promise }): Promise { return new Promise((resolve, reject) => { @@ -112,6 +129,51 @@ function withTimeout({ ms, promise }: { ms: number; promise: Promise }): P }); } +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// Drives a real, awaited round-trip through the same POSClient instance the +// request handlers reuse from clientCache, so the warm-up proves detectNetwork +// actually succeeded rather than merely started. Uses exitUtil.rootChain's +// getLastChildBlock() — the same cheap root-chain read isBlockIncluded() makes +// on the request path — as the probe. +// +// A resolved init() with a subsequently-failing probe means the client's +// underlying provider may be sitting on a bad state from the race described +// above; re-probing the same instance risks repeating that race, so each +// retry evicts the cached client first and re-inits from scratch. +async function warmOneClient({ + ethereumRPC, + isMainnet, + maticRPC, + version +}: { + ethereumRPC: string; + isMainnet: boolean; + maticRPC: string; + version: string; +}): Promise { + let lastErr: unknown; + + for (let attempt = 0; attempt < WARM_UP_PROBE_ATTEMPTS; attempt++) { + if (attempt > 0) { + clientCache.delete(getCacheKey(isMainnet, version, maticRPC, ethereumRPC)); + await delay(WARM_UP_PROBE_BACKOFF_MS); + } + + try { + const client = await initMatic(isMainnet, version, maticRPC, ethereumRPC); + await client.exitUtil.rootChain.getLastChildBlock(); + return; + } catch (err) { + lastErr = err; + } + } + + throw lastErr instanceof Error ? lastErr : new Error(String(lastErr)); +} + export async function warmMaticClients(logger: Logger): Promise { // Mirrors src/routes/v1.ts's networkDetails map, at the RPC index requests // start from (config.mainnetRpcIndex / config.testnetRpcIndex). @@ -135,9 +197,15 @@ export async function warmMaticClients(logger: Logger): Promise { if (!maticRPC || !ethereumRPC) { return Promise.reject(new Error(`no configured RPC endpoint for ${version}`)); } + const startedAt = Date.now(); return withTimeout({ ms: WARM_UP_TIMEOUT_MS, - promise: initMatic(isMainnet, version, maticRPC, ethereumRPC) + promise: warmOneClient({ ethereumRPC, isMainnet, maticRPC, version }).then(() => { + logger.debug( + { durationMs: Date.now() - startedAt, version }, + 'RPC provider warm-up probe succeeded' + ); + }) }); }) ); From 4fa044fd222ee69bc2ffb381fcc17702c83b825d Mon Sep 17 00:00:00 2001 From: "Shan8851.eth" Date: Thu, 13 Aug 2026 11:43:22 +0100 Subject: [PATCH 4/4] =?UTF-8?q?ci:=20point=20release-gate=20AMOY=5FRPC=20a?= =?UTF-8?q?t=20eRPC=20=E2=80=94=20public=20amoy=20endpoint=20lost=20DNS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci-trigger.yml already used the eRPC URL; the release gate was the only consumer of the dead rpc-amoy.polygon.technology hostname, which is why CI passed while every release-tag run failed. Co-Authored-By: Claude Fable 5 --- .github/workflows/docker-release-trigger.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-release-trigger.yml b/.github/workflows/docker-release-trigger.yml index c480a42..564e999 100644 --- a/.github/workflows/docker-release-trigger.yml +++ b/.github/workflows/docker-release-trigger.yml @@ -20,9 +20,10 @@ jobs: runs-on: ubuntu-latest env: # RPC URLs required by the service at runtime and during tests. - # AMOY_RPC is a public endpoint with no secret. + # All four use the eRPC client token; rpc-amoy.polygon.technology (the + # public endpoint previously used for AMOY_RPC) no longer resolves. # ETHEREUM_RPC, MATIC_RPC, and SEPOLIA_RPC use an eRPC client token. - AMOY_RPC: '["https://rpc-amoy.polygon.technology"]' + AMOY_RPC: '["https://rpc.polygon.tools/internal/evm/80002?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]' ETHEREUM_RPC: '["https://rpc.polygon.tools/internal/evm/1?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]' MATIC_RPC: '["https://rpc.polygon.tools/internal/evm/137?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]' SEPOLIA_RPC: '["https://rpc.polygon.tools/internal/evm/11155111?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]'