fix: warm RPC providers at bootstrap to close the cold-start detectNetwork race

.changeset/warm-rpc-providers-boot.md

@@ -0,0 +1,5 @@
+---
+"proof-generation-api": patch
+---
+
+The server now warms its RPC provider connections during startup, eliminating first-request network-detection failures immediately after boot.

.github/workflows/docker-release-trigger.yml

@@ -20,9 +20,10 @@ jobs:
     runs-on: ubuntu-latest
     env:
       # RPC URLs required by the service at runtime and during tests.
-      # AMOY_RPC is a public endpoint with no secret.
+      # All four use the eRPC client token; rpc-amoy.polygon.technology (the
+      # public endpoint previously used for AMOY_RPC) no longer resolves.
       # ETHEREUM_RPC, MATIC_RPC, and SEPOLIA_RPC use an eRPC client token.
-      AMOY_RPC: '["https://rpc-amoy.polygon.technology"]'
+      AMOY_RPC: '["https://rpc.polygon.tools/internal/evm/80002?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]'
       ETHEREUM_RPC: '["https://rpc.polygon.tools/internal/evm/1?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]'
       MATIC_RPC: '["https://rpc.polygon.tools/internal/evm/137?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]'
       SEPOLIA_RPC: '["https://rpc.polygon.tools/internal/evm/11155111?token=${{ secrets.ERPC_CLIENT_TOKEN }}"]'

CHANGELOG.md

@@ -7,6 +7,7 @@
 - [#89](https://github.com/0xPolygon/proof-generation-api/pull/89) [`58a7e8b`](https://github.com/0xPolygon/proof-generation-api/commit/58a7e8bd4c7212544177f2c1e41ef6e530604c88) Thanks [@shan8851](https://github.com/shan8851)! - Remove the zkEVM proof-generation endpoints (`/zkevm/mainnet` and `/zkevm/testnet` routes) — the underlying chains are sunset and their infrastructure no longer answers. PoS (v1) endpoints are unchanged.
 
   ## Breaking changes
+
   - Removed `GET /api/zkevm/{network}/bridge`
   - Removed `GET /api/zkevm/{network}/merkle-proof`
   - Removed the `ZKEVM_MAINNET_URL` and `ZKEVM_TESTNET_URL` environment variables — they are no longer recognized configuration

src/index.ts

@@ -9,6 +9,7 @@ import { HTTPError } from '@polygonlabs/verror';
 
 import type { Logger } from './logger.ts';
 
+import { warmMaticClients } from './maticClient.ts';
 import { createIndexRouter } from './routes/index.ts';
 
 // Attach a per-request child logger to req.log so every log entry for a
@@ -68,6 +69,20 @@ export async function startApiServer({
   app: Express;
   logger: Logger;
 }): Promise<void> {
+  // Warm the RPC providers before opening the listen socket: /health-check has no
+  // RPC dependency and is the sole readiness signal the shared docker-test composite
+  // waits on, so without this a POSClient's maiden detectNetwork() call — a cold
+  // DNS+TLS+RPC round trip in a fresh container network namespace — can lose the
+  // race against the steady-state 2-attempt retry budget on a real request.
+  try {
+    await warmMaticClients(logger);
+  } catch (err: unknown) {
+    logger.warn(
+      { err: err instanceof Error ? err : new Error(String(err)) },
+      'RPC provider warm-up failed; continuing startup'
+    );
+  }
+
   // Bubble errors calling `listen()` up to callers so they get an async stack trace
   await new Promise((resolve, reject) => {
     app.listen(port).once('listening', resolve).once('error', reject);

src/maticClient.ts

@@ -5,6 +5,8 @@ import type { IPOSClientConfig } from '@maticnetwork/maticjs';
 import maticJs from '@maticnetwork/maticjs';
 import maticJs_Ethers from '@maticnetwork/maticjs-ethers';
 
+import type { Logger } from './logger.ts';
+
 import { config } from './config.ts';
 
 const { Converter, POSClient, use } = maticJs; // default export :(
@@ -28,15 +30,24 @@ use(Web3ClientPlugin);
 // The cache stores Promises so that concurrent requests for the same key share
 // a single in-flight initialisation rather than racing to create duplicates.
 // A failed initialisation is evicted from the cache so the next request retries.
+// warmMaticClients (below) evicts the same way when a resolved client's first
+// real round-trip fails, for the same reason.
 const clientCache = new Map<string, Promise<InstanceType<typeof POSClient>>>();
 
+const getCacheKey = (
+  isMainnet: boolean,
+  version: string,
+  maticRPC: string,
+  ethereumRPC: string
+): string => `${isMainnet ? 'mainnet' : 'testnet'}:${version}:${maticRPC}:${ethereumRPC}`;
+
 export const initMatic = (
   isMainnet: boolean,
   version: string,
   maticRPC: string,
   ethereumRPC: string
 ): Promise<InstanceType<typeof POSClient>> => {
-  const cacheKey = `${isMainnet ? 'mainnet' : 'testnet'}:${version}:${maticRPC}:${ethereumRPC}`;
+  const cacheKey = getCacheKey(isMainnet, version, maticRPC, ethereumRPC);
 
   const cached = clientCache.get(cacheKey);
   if (cached) return cached;
@@ -91,3 +102,120 @@ export const initMatic = (
 export const convert = async (value: any) => {
   return Converter.toHex(value);
 };
+
+// Bounds the whole warm-up-plus-probe sequence for one tuple, including the
+// retry backoffs below. ethers v5's StaticJsonRpcProvider only *schedules*
+// detectNetwork() via setTimeout(0) in its constructor — awaiting init()
+// resolving proves detection STARTED, not that it SUCCEEDED, so a probe is
+// needed. Raised from the original 5s to give 3 backed-off probe attempts
+// room, while staying well under boot-liveness budgets.
+const WARM_UP_TIMEOUT_MS = 8_000;
+const WARM_UP_PROBE_ATTEMPTS = 3;
+const WARM_UP_PROBE_BACKOFF_MS = 500;
+
+function withTimeout<T>({ ms, promise }: { ms: number; promise: Promise<T> }): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (err: unknown) => {
+        clearTimeout(timer);
+        reject(err instanceof Error ? err : new Error(String(err)));
+      }
+    );
+  });
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// Drives a real, awaited round-trip through the same POSClient instance the
+// request handlers reuse from clientCache, so the warm-up proves detectNetwork
+// actually succeeded rather than merely started. Uses exitUtil.rootChain's
+// getLastChildBlock() — the same cheap root-chain read isBlockIncluded() makes
+// on the request path — as the probe.
+//
+// A resolved init() with a subsequently-failing probe means the client's
+// underlying provider may be sitting on a bad state from the race described
+// above; re-probing the same instance risks repeating that race, so each
+// retry evicts the cached client first and re-inits from scratch.
+async function warmOneClient({
+  ethereumRPC,
+  isMainnet,
+  maticRPC,
+  version
+}: {
+  ethereumRPC: string;
+  isMainnet: boolean;
+  maticRPC: string;
+  version: string;
+}): Promise<void> {
+  let lastErr: unknown;
+
+  for (let attempt = 0; attempt < WARM_UP_PROBE_ATTEMPTS; attempt++) {
+    if (attempt > 0) {
+      clientCache.delete(getCacheKey(isMainnet, version, maticRPC, ethereumRPC));
+      await delay(WARM_UP_PROBE_BACKOFF_MS);
+    }
+
+    try {
+      const client = await initMatic(isMainnet, version, maticRPC, ethereumRPC);
+      await client.exitUtil.rootChain.getLastChildBlock();
+      return;
+    } catch (err) {
+      lastErr = err;
+    }
+  }
+
+  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+}
+
+export async function warmMaticClients(logger: Logger): Promise<void> {
+  // Mirrors src/routes/v1.ts's networkDetails map, at the RPC index requests
+  // start from (config.mainnetRpcIndex / config.testnetRpcIndex).
+  const warmUpTuples = [
+    {
+      ethereumRPC: config.app.ethereumRPC[config.mainnetRpcIndex],
+      isMainnet: true,
+      maticRPC: config.app.maticRPC[config.mainnetRpcIndex],
+      version: 'v1'
+    },
+    {
+      ethereumRPC: config.app.sepoliaRPC[config.testnetRpcIndex],
+      isMainnet: false,
+      maticRPC: config.app.amoyRPC[config.testnetRpcIndex],
+      version: 'amoy'
+    }
+  ];
+
+  const results = await Promise.allSettled(
+    warmUpTuples.map(({ ethereumRPC, isMainnet, maticRPC, version }) => {
+      if (!maticRPC || !ethereumRPC) {
+        return Promise.reject(new Error(`no configured RPC endpoint for ${version}`));
+      }
+      const startedAt = Date.now();
+      return withTimeout({
+        ms: WARM_UP_TIMEOUT_MS,
+        promise: warmOneClient({ ethereumRPC, isMainnet, maticRPC, version }).then(() => {
+          logger.debug(
+            { durationMs: Date.now() - startedAt, version },
+            'RPC provider warm-up probe succeeded'
+          );
+        })
+      });
+    })
+  );
+
+  results.forEach((result, i) => {
+    if (result.status === 'rejected') {
+      logger.warn(
+        { err: result.reason, version: warmUpTuples[i]?.version },
+        'RPC provider warm-up failed; will retry at request time'
+      );
+    }
+  });
+}