diff --git a/.changeset/warm-rpc-providers-boot.md b/.changeset/warm-rpc-providers-boot.md new file mode 100644 index 0000000..cfc823f --- /dev/null +++ b/.changeset/warm-rpc-providers-boot.md @@ -0,0 +1,5 @@ +--- +"proof-generation-api": patch +--- + +The server now warms its RPC provider connections during startup, eliminating first-request network-detection failures immediately after boot. diff --git a/.github/workflows/docker-release-trigger.yml b/.github/workflows/docker-release-trigger.yml index c480a42..564e999 100644 --- a/.github/workflows/docker-release-trigger.yml +++ b/.github/workflows/docker-release-trigger.yml @@ -20,9 +20,10 @@ jobs: runs-on: ubuntu-latest env: # RPC URLs required by the service at runtime and during tests. - # AMOY_RPC is a public endpoint with no secret. + # All four use the eRPC client token; rpc-amoy.polygon.technology (the + # public endpoint previously used for AMOY_RPC) no longer resolves. # ETHEREUM_RPC, MATIC_RPC, and SEPOLIA_RPC use an eRPC client token. - AMOY_RPC: '["https://rpc-amoy.polygon.technology"]' + AMOY_RPC: '["https://rpc.polygon.tools/internal/evm/80002?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]' ETHEREUM_RPC: '["https://rpc.polygon.tools/internal/evm/1?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]' MATIC_RPC: '["https://rpc.polygon.tools/internal/evm/137?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]' SEPOLIA_RPC: '["https://rpc.polygon.tools/internal/evm/11155111?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]' diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa447e..a0df5d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - [#89](https://github.com/0xPolygon/proof-generation-api/pull/89) [`58a7e8b`](https://github.com/0xPolygon/proof-generation-api/commit/58a7e8bd4c7212544177f2c1e41ef6e530604c88) Thanks [@shan8851](https://github.com/shan8851)! - Remove the zkEVM proof-generation endpoints (`/zkevm/mainnet` and `/zkevm/testnet` routes) — the underlying chains are sunset and their infrastructure no longer answers. PoS (v1) endpoints are unchanged. ## Breaking changes + - Removed `GET /api/zkevm/{network}/bridge` - Removed `GET /api/zkevm/{network}/merkle-proof` - Removed the `ZKEVM_MAINNET_URL` and `ZKEVM_TESTNET_URL` environment variables — they are no longer recognized configuration diff --git a/src/index.ts b/src/index.ts index 9a39f34..ea5b006 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ import { HTTPError } from '@polygonlabs/verror'; import type { Logger } from './logger.ts'; +import { warmMaticClients } from './maticClient.ts'; import { createIndexRouter } from './routes/index.ts'; // Attach a per-request child logger to req.log so every log entry for a @@ -68,6 +69,20 @@ export async function startApiServer({ app: Express; logger: Logger; }): Promise { + // Warm the RPC providers before opening the listen socket: /health-check has no + // RPC dependency and is the sole readiness signal the shared docker-test composite + // waits on, so without this a POSClient's maiden detectNetwork() call — a cold + // DNS+TLS+RPC round trip in a fresh container network namespace — can lose the + // race against the steady-state 2-attempt retry budget on a real request. + try { + await warmMaticClients(logger); + } catch (err: unknown) { + logger.warn( + { err: err instanceof Error ? err : new Error(String(err)) }, + 'RPC provider warm-up failed; continuing startup' + ); + } + // Bubble errors calling `listen()` up to callers so they get an async stack trace await new Promise((resolve, reject) => { app.listen(port).once('listening', resolve).once('error', reject); diff --git a/src/maticClient.ts b/src/maticClient.ts index 6b93f20..1208fe4 100644 --- a/src/maticClient.ts +++ b/src/maticClient.ts @@ -5,6 +5,8 @@ import type { IPOSClientConfig } from '@maticnetwork/maticjs'; import maticJs from '@maticnetwork/maticjs'; import maticJs_Ethers from '@maticnetwork/maticjs-ethers'; +import type { Logger } from './logger.ts'; + import { config } from './config.ts'; const { Converter, POSClient, use } = maticJs; // default export :( @@ -28,15 +30,24 @@ use(Web3ClientPlugin); // The cache stores Promises so that concurrent requests for the same key share // a single in-flight initialisation rather than racing to create duplicates. // A failed initialisation is evicted from the cache so the next request retries. +// warmMaticClients (below) evicts the same way when a resolved client's first +// real round-trip fails, for the same reason. const clientCache = new Map>>(); +const getCacheKey = ( + isMainnet: boolean, + version: string, + maticRPC: string, + ethereumRPC: string +): string => `${isMainnet ? 'mainnet' : 'testnet'}:${version}:${maticRPC}:${ethereumRPC}`; + export const initMatic = ( isMainnet: boolean, version: string, maticRPC: string, ethereumRPC: string ): Promise> => { - const cacheKey = `${isMainnet ? 'mainnet' : 'testnet'}:${version}:${maticRPC}:${ethereumRPC}`; + const cacheKey = getCacheKey(isMainnet, version, maticRPC, ethereumRPC); const cached = clientCache.get(cacheKey); if (cached) return cached; @@ -91,3 +102,120 @@ export const initMatic = ( export const convert = async (value: any) => { return Converter.toHex(value); }; + +// Bounds the whole warm-up-plus-probe sequence for one tuple, including the +// retry backoffs below. ethers v5's StaticJsonRpcProvider only *schedules* +// detectNetwork() via setTimeout(0) in its constructor — awaiting init() +// resolving proves detection STARTED, not that it SUCCEEDED, so a probe is +// needed. Raised from the original 5s to give 3 backed-off probe attempts +// room, while staying well under boot-liveness budgets. +const WARM_UP_TIMEOUT_MS = 8_000; +const WARM_UP_PROBE_ATTEMPTS = 3; +const WARM_UP_PROBE_BACKOFF_MS = 500; + +function withTimeout({ ms, promise }: { ms: number; promise: Promise }): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (err: unknown) => { + clearTimeout(timer); + reject(err instanceof Error ? err : new Error(String(err))); + } + ); + }); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// Drives a real, awaited round-trip through the same POSClient instance the +// request handlers reuse from clientCache, so the warm-up proves detectNetwork +// actually succeeded rather than merely started. Uses exitUtil.rootChain's +// getLastChildBlock() — the same cheap root-chain read isBlockIncluded() makes +// on the request path — as the probe. +// +// A resolved init() with a subsequently-failing probe means the client's +// underlying provider may be sitting on a bad state from the race described +// above; re-probing the same instance risks repeating that race, so each +// retry evicts the cached client first and re-inits from scratch. +async function warmOneClient({ + ethereumRPC, + isMainnet, + maticRPC, + version +}: { + ethereumRPC: string; + isMainnet: boolean; + maticRPC: string; + version: string; +}): Promise { + let lastErr: unknown; + + for (let attempt = 0; attempt < WARM_UP_PROBE_ATTEMPTS; attempt++) { + if (attempt > 0) { + clientCache.delete(getCacheKey(isMainnet, version, maticRPC, ethereumRPC)); + await delay(WARM_UP_PROBE_BACKOFF_MS); + } + + try { + const client = await initMatic(isMainnet, version, maticRPC, ethereumRPC); + await client.exitUtil.rootChain.getLastChildBlock(); + return; + } catch (err) { + lastErr = err; + } + } + + throw lastErr instanceof Error ? lastErr : new Error(String(lastErr)); +} + +export async function warmMaticClients(logger: Logger): Promise { + // Mirrors src/routes/v1.ts's networkDetails map, at the RPC index requests + // start from (config.mainnetRpcIndex / config.testnetRpcIndex). + const warmUpTuples = [ + { + ethereumRPC: config.app.ethereumRPC[config.mainnetRpcIndex], + isMainnet: true, + maticRPC: config.app.maticRPC[config.mainnetRpcIndex], + version: 'v1' + }, + { + ethereumRPC: config.app.sepoliaRPC[config.testnetRpcIndex], + isMainnet: false, + maticRPC: config.app.amoyRPC[config.testnetRpcIndex], + version: 'amoy' + } + ]; + + const results = await Promise.allSettled( + warmUpTuples.map(({ ethereumRPC, isMainnet, maticRPC, version }) => { + if (!maticRPC || !ethereumRPC) { + return Promise.reject(new Error(`no configured RPC endpoint for ${version}`)); + } + const startedAt = Date.now(); + return withTimeout({ + ms: WARM_UP_TIMEOUT_MS, + promise: warmOneClient({ ethereumRPC, isMainnet, maticRPC, version }).then(() => { + logger.debug( + { durationMs: Date.now() - startedAt, version }, + 'RPC provider warm-up probe succeeded' + ); + }) + }); + }) + ); + + results.forEach((result, i) => { + if (result.status === 'rejected') { + logger.warn( + { err: result.reason, version: warmUpTuples[i]?.version }, + 'RPC provider warm-up failed; will retry at request time' + ); + } + }); +}