From 2bbae16a11e23f0c72978a31401c638d7d5f0868 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 21 Aug 2026 11:06:35 -0700 Subject: [PATCH 01/12] README: time-stamp the supported-chain roster Bitcoin, Litecoin, and Dogecoin are where the platform runs today, not the definition of it; the README says so where it names them. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 718f805..fd51b73 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,15 @@ Coverage

-CLI management and orchestration tool for the XChain Platform. Installs, configures, and manages all XChain services and coin nodes (bitcoind, litecoind, dogecoind) as Docker containers. Generates per-service environment variables from a two-layer configuration system, manages LevelDB state, provisions MariaDB databases, and provides multi-pane log monitoring. +CLI management and orchestration tool for the XChain Platform. Installs, configures, and manages all XChain services and coin nodes (bitcoind, litecoind, and dogecoind today; any Bitcoin-RPC-compatible UTXO chain can be added by configuration) as Docker containers. Generates per-service environment variables from a two-layer configuration system, manages LevelDB state, provisions MariaDB databases, and provides multi-pane log monitoring. ## Features -- **Multi-chain orchestration**: manages Bitcoin, Litecoin, and Dogecoin across mainnet, testnet, and regtest; each chain/network gets its own Docker network and container set +- **Multi-chain orchestration**: manages Bitcoin, Litecoin, and Dogecoin today across mainnet, testnet, and regtest; each chain/network gets its own Docker network and container set - **Order-independent argument parsing**: CLI arguments auto-classified as service, coin, network, or branch name regardless of position - **Docker container lifecycle**: install, start, stop, restart, update, uninstall, and reset services with single commands - **Configuration generation**: two-layer system (hardcoded defaults + config file overrides) producing 40+ environment variables per service -- **Crypto node management**: downloads Bitcoin Core, Litecoin, and Dogecoin binaries from official sources with SHA-256 verification; includes per-chain regtest tuning applied automatically +- **Crypto node management**: downloads the Bitcoin Core, Litecoin, and Dogecoin binaries (the chains supported today) from official sources with SHA-256 verification; includes per-chain regtest tuning applied automatically - **Database orchestration**: provisions shared MariaDB, creates per-service databases and users with subnet-based permissions - **Bootstrap snapshots**: create and restore gzipped snapshots of UTXO tracker, decoder, and indexer data; integrity is double-verified with SHA-256 checksums and a detached Ed25519 signature pinned to a bundled public key - **Validator mode**: `validator init` generates an Ed25519 signing key and capabilities config; the hub boots in PBFT validator mode when a key is present From 149861b03f956c03633132081648dc02621c62c8 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 22 Aug 2026 10:27:26 -0700 Subject: [PATCH 02/12] Fix four first-run install failures Module clones use the public HTTPS GitHub URLs, so a machine without a GitHub SSH key can install; SSH and local-source workflows keep the XCHAIN_NODE_MODULES_URLS_OVERRIDE escape hatch. Downloading a coin node creates the crypto-nodes directory before opening the write stream. The default directory ships in the repo, so this only ever failed for installs pointing CRYPTO_NODES_DIR at a fresh volume. Bootstrap auto-restore routes its download directory through ensureDirWritable, recovering a destination a service container already created root-owned, the same failure the create path already handles. The explorer install health wait grows from ten seconds to about two minutes, covering a cold container's warm-up instead of reporting a hard failure a rerun immediately contradicts. --- CHANGELOG.md | 8 ++++++++ src/config/constants.js | 27 +++++++++++++++------------ src/services/BootstrapService.js | 5 ++++- src/services/ExplorerService.js | 11 ++++++++--- src/services/NodeService.js | 4 ++++ 5 files changed, 39 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c1dd85..316f5dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- Module clones now use public HTTPS URLs, so installs work without a GitHub SSH key. +- Downloading a coin node now creates the crypto-nodes directory first, fixing installs pointed at a fresh custom volume. +- Bootstrap auto-restore downloads now recover a root-owned destination directory instead of failing with a permission error. +- The explorer install health wait now allows about two minutes of container warm-up instead of ten seconds. + ## [0.10.0] - 2026-08-22 ### Added diff --git a/src/config/constants.js b/src/config/constants.js index 42078b7..dbb9f2f 100644 --- a/src/config/constants.js +++ b/src/config/constants.js @@ -96,23 +96,26 @@ const projectFolders = { "xchain-vm": "XChainVM" } +// HTTPS, not SSH: the repos are public, and a fresh machine has no GitHub SSH +// key, so git@ URLs fail the very first module clone of a documented install. +// SSH/fork/local-source workflows go through XCHAIN_NODE_MODULES_URLS_OVERRIDE. const modulesUrls = { - "xchain-encoder": "git@github.com:XChain-Platform/xchain-encoder.git", - "xchain-decoder": "git@github.com:XChain-Platform/xchain-decoder.git", - "xchain-utxo-tracker": "git@github.com:XChain-Platform/xchain-utxo-tracker.git", - "xchain-indexer": "git@github.com:XChain-Platform/xchain-indexer.git", - "xchain-regtest-miner": "git@github.com:XChain-Platform/xchain-regtest-miner.git", - "xchain-hub": "git@github.com:XChain-Platform/xchain-hub.git", - "xchain-explorer": "git@github.com:XChain-Platform/xchain-explorer.git", - "xchain-e2e-test": "git@github.com:XChain-Platform/xchain-e2e-test.git", - "xchain-sync": "git@github.com:XChain-Platform/xchain-sync.git", - "xchain-vm": "git@github.com:XChain-Platform/xchain-vm.git", + "xchain-encoder": "https://github.com/XChain-Platform/xchain-encoder.git", + "xchain-decoder": "https://github.com/XChain-Platform/xchain-decoder.git", + "xchain-utxo-tracker": "https://github.com/XChain-Platform/xchain-utxo-tracker.git", + "xchain-indexer": "https://github.com/XChain-Platform/xchain-indexer.git", + "xchain-regtest-miner": "https://github.com/XChain-Platform/xchain-regtest-miner.git", + "xchain-hub": "https://github.com/XChain-Platform/xchain-hub.git", + "xchain-explorer": "https://github.com/XChain-Platform/xchain-explorer.git", + "xchain-e2e-test": "https://github.com/XChain-Platform/xchain-e2e-test.git", + "xchain-sync": "https://github.com/XChain-Platform/xchain-sync.git", + "xchain-vm": "https://github.com/XChain-Platform/xchain-vm.git", // Not an installable service; listed so LIBRARY_BUNDLES can stage it // into the xchain-e2e-test build context (test:sdk suites). - "xchain-sdk": "git@github.com:XChain-Platform/xchain-sdk.git", + "xchain-sdk": "https://github.com/XChain-Platform/xchain-sdk.git", // Also not installable; staged into the e2e-test build context so the // template suites (amm/escrow/crowdsale/vesting) can load their source. - "xchain-contracts": "git@github.com:XChain-Platform/xchain-contracts.git" + "xchain-contracts": "https://github.com/XChain-Platform/xchain-contracts.git" } // Optional env-var override for local-source workflows. Lets you point any diff --git a/src/services/BootstrapService.js b/src/services/BootstrapService.js index 394818c..7e76464 100644 --- a/src/services/BootstrapService.js +++ b/src/services/BootstrapService.js @@ -845,7 +845,10 @@ async function utxoTrackerVolumeHasData(coin, network) { async function downloadBootstrap(coin, network, module, destDir) { const url = `${BOOTSTRAP_BASE_URL}/${module}/${coin}/${network}/latest.tgz` const destPath = path.join(destDir, 'latest.tgz') - ensureDir(destDir) + // destDir is the bind-mounted bootstrap volume, which a service container + // may already have created root-owned; the writable variant chowns it back + // (same failure ensureDirWritable was written for on the create path). + await ensureDirWritable(destDir) const response = await axios({ method: 'get', diff --git a/src/services/ExplorerService.js b/src/services/ExplorerService.js index e48a64e..a7e4076 100644 --- a/src/services/ExplorerService.js +++ b/src/services/ExplorerService.js @@ -145,7 +145,12 @@ async function installExplorerModule(force = false, branch = null) { // explorer holding no pools is a real fault and must still fail the install. const coinsPresent = Object.keys(await getInstalledCoinsAndNetworks()).length > 0 - let tries = 10 + // 60 tries x 2s = a ~2 minute budget. A freshly built container can spend + // a minute-plus warming (DB pools, first hub poll) while answering 503, and + // the old 10x1s loop gave up mid-warmup on slow hosts, reporting a hard + // failure for a container that went healthy moments later (a rerun then + // succeeded immediately via the ping short-circuit above). + let tries = 60 while (tries > 0) { const { answering, healthy } = await explorerConnector.probe() if (healthy || (answering && !coinsPresent)) { @@ -153,7 +158,7 @@ async function installExplorerModule(force = false, branch = null) { await updateExplorer() } catch { tries-- - await sleep(1000) + await sleep(2000) continue } if (!healthy) { @@ -162,7 +167,7 @@ async function installExplorerModule(force = false, branch = null) { } return true } else { - await sleep(1000) + await sleep(2000) } tries-- } diff --git a/src/services/NodeService.js b/src/services/NodeService.js index edb52f8..8c2b5c9 100644 --- a/src/services/NodeService.js +++ b/src/services/NodeService.js @@ -42,6 +42,10 @@ async function getCryptoNode(coin, network, version) { const destination = cryptoNodesDir + "/bitcoin" const filePath = destination + "/bitcoin" + version + ".tar.gz" + // The default crypto_nodes/bitcoin ships in the repo, but a custom + // XCHAIN_NODE_CRYPTO_NODES_DIR (the documented big-volume setup) + // starts empty, and createWriteStream does not create directories. + fs.mkdirSync(destination, { recursive: true }) const bitcoinNodeFile = fs.createWriteStream(filePath) // Pick the right prebuilt tarball for the host architecture. // bitcoincore.org publishes x86_64-linux-gnu and aarch64-linux-gnu builds. From 805a057afb6da151509612c1f5e0a46a0e9281e9 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 22 Aug 2026 14:29:46 -0700 Subject: [PATCH 03/12] Survive a broken mirror, and report a bootstrap that did not restore bitcoincore.org is several mirrors behind one name and they are not equivalent: at least one serves a leaf-only certificate chain, which Node rejects where curl and browsers recover by fetching the missing intermediate. Whoever DNS sent there could not install at all, and retrying did not help because the resolver kept returning the same address. A failed download now enumerates the site's addresses and tries each one, pinning only which mirror is dialled: the URL, SNI and certificate check still run against the hostname, and the pinned SHA-256 is still verified before the tarball is used. Only transport failures widen this way, since every mirror answers a 404 alike, and the ordinary path is unchanged. Failures name the URL, the mirror and the cause, plus a way forward when all of them fail. Separately, a bootstrap restore that fails leaves the service syncing from block 0, which is hours to days of work, and its only trace was one warning in the middle of a long install log. Install and update now end with a restore summary per service. Because a service that starts syncing no longer reads as fresh, the failed attempt was also the only attempt; XCHAIN_NODE_FORCE_BOOTSTRAP=1 takes it again, kept opt-in because the restore wipes the data directory. --- CHANGELOG.md | 6 + src/operations/moduleOperations.js | 7 ++ src/services/BootstrapService.js | 61 ++++++++- src/services/ModuleService.js | 8 +- src/services/NodeService.js | 196 ++++++++++++++++++++--------- test/unit/BootstrapService.test.js | 127 +++++++++++++++++++ test/unit/NodeService.test.js | 121 +++++++++++++++++- 7 files changed, 456 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 316f5dc..9ef063d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Install and update end with a bootstrap restore summary, so a restore that did not happen is stated rather than left as one warning mid-log. +- `XCHAIN_NODE_FORCE_BOOTSTRAP=1` restores a published bootstrap over an already-populated service, for when the install that would have taken it failed. + ### Fixed +- A failed bitcoind download now retries against the site's other mirror addresses, so one mirror serving a broken certificate chain no longer blocks the install. +- Download failures name the URL, the mirror and the cause instead of a generic message. - Module clones now use public HTTPS URLs, so installs work without a GitHub SSH key. - Downloading a coin node now creates the crypto-nodes directory first, fixing installs pointed at a fresh custom volume. - Bootstrap auto-restore downloads now recover a root-owned destination directory instead of failing with a permission error. diff --git a/src/operations/moduleOperations.js b/src/operations/moduleOperations.js index 6bab3fd..98295e1 100644 --- a/src/operations/moduleOperations.js +++ b/src/operations/moduleOperations.js @@ -78,6 +78,9 @@ async function installModules(servicesList, ref = null) { // passes the branch, exactly as before. const branch = target.kind === 'release' ? null : target.ref const outcome = { installed: [], skipped: [] } + // Per-run, so a second install in the same process reports its own + // restores rather than replaying the first one's. + require('../services/BootstrapService').resetBootstrapOutcomes() for (const nextCoin in servicesList) { for (const nextNetwork in servicesList[nextCoin]) { @@ -102,6 +105,10 @@ async function installModules(servicesList, ref = null) { + ' - already installed. Use `update` to rebuild.') } + // A bootstrap that did not restore costs hours of resync, and its only + // trace was one warning far up a long install log. + require('../services/BootstrapService').reportBootstrapOutcomes() + // The explorer is installed in the shared bucket, which runs BEFORE the // coin stacks, and it learns its coins by polling the hub. So a run that // installed a coin leaves it serving 503 for up to a poll interval after diff --git a/src/services/BootstrapService.js b/src/services/BootstrapService.js index 7e76464..63fe89a 100644 --- a/src/services/BootstrapService.js +++ b/src/services/BootstrapService.js @@ -911,12 +911,57 @@ async function downloadBootstrap(coin, network, module, destDir) { return 'latest.tgz' } +// What each service's bootstrap attempt did, for the end-of-install summary: +// a skipped restore is the difference between a published height and hours of +// rescanning, too costly to leave as one warning mid-log. Reset per run. +const bootstrapOutcomes = [] + +function recordBootstrapOutcome(module, status, detail) { + bootstrapOutcomes.push({ module, status, detail }) +} + +function resetBootstrapOutcomes() { + bootstrapOutcomes.length = 0 +} + +// Printed at the end of install/update. Says nothing when no bootstrap was +// attempted, so ordinary runs stay quiet. +function reportBootstrapOutcomes() { + if (bootstrapOutcomes.length === 0) return + const failed = bootstrapOutcomes.filter((o) => o.status === 'failed') + console.log('\nBootstrap restore summary:') + for (const o of bootstrapOutcomes) { + const line = o.status === 'restored' ? 'restored' + : o.status === 'none-published' ? 'none published, syncing from scratch' + : o.status === 'disabled' ? 'disabled by XCHAIN_NODE_NO_BOOTSTRAP' + : `NOT restored: ${o.detail}` + console.log(` ${o.module}: ${line}`) + } + if (failed.length > 0) { + console.log( + '\nThose services are now syncing from block 0, which takes hours to days\n' + + 'rather than minutes. Fix the cause above, then re-run install with\n' + + 'XCHAIN_NODE_FORCE_BOOTSTRAP=1 to take the restore again: without it a\n' + + 'service that has already started syncing is left alone.\n' + ) + } +} + +// Opt-in restore over an already-populated service. Off by default because the +// restore wipes the data directory; needed because a failed restore leaves a +// service scratch-syncing, which reads as populated to every later run. +function forceBootstrapRequested() { + const v = process.env.XCHAIN_NODE_FORCE_BOOTSTRAP + return v !== undefined && v !== '' && v !== '0' +} + // On a FRESH utxo-tracker install, download the published bootstrap and restore // it. Best-effort: any failure (no bootstrap published, download/restore error) // logs a warning and returns so the install proceeds with a normal sync. async function ensureBootstrapUtxoTracker(coin, network) { if (process.env.XCHAIN_NODE_NO_BOOTSTRAP) { console.log('Bootstrap auto-restore disabled (XCHAIN_NODE_NO_BOOTSTRAP): syncing from scratch') + recordBootstrapOutcome(XChainService.XCHAIN_UTXO_TRACKER, 'disabled') return false } try { @@ -927,13 +972,17 @@ async function ensureBootstrapUtxoTracker(coin, network) { const fileName = await downloadBootstrap(coin, network, XChainService.XCHAIN_UTXO_TRACKER, bootstrapDir) if (!fileName) { console.log('No bootstrap available; the tracker will sync from scratch') + recordBootstrapOutcome(XChainService.XCHAIN_UTXO_TRACKER, 'none-published') return false } await restoreBootstrap(coin, network, XChainService.XCHAIN_UTXO_TRACKER, fileName) console.log('Bootstrap installed; tracker will continue from the bootstrap height') + recordBootstrapOutcome(XChainService.XCHAIN_UTXO_TRACKER, 'restored') return true } catch (err) { - console.log(`WARNING: bootstrap auto-restore failed (${redactSecrets(err.message)}): the tracker will sync from scratch`) + const reason = redactSecrets(err.message) + console.log(`WARNING: bootstrap auto-restore failed (${reason}): the tracker will sync from scratch`) + recordBootstrapOutcome(XChainService.XCHAIN_UTXO_TRACKER, 'failed', reason) return false } } @@ -1007,6 +1056,7 @@ async function mariaDbModuleHasData(coin, network, module) { async function ensureBootstrapMariaDb(coin, network, module) { if (process.env.XCHAIN_NODE_NO_BOOTSTRAP) { console.log('Bootstrap auto-restore disabled (XCHAIN_NODE_NO_BOOTSTRAP): syncing from scratch') + recordBootstrapOutcome(module, 'disabled') return false } try { @@ -1019,13 +1069,17 @@ async function ensureBootstrapMariaDb(coin, network, module) { const fileName = await downloadBootstrap(coin, network, module, bootstrapDir) if (!fileName) { console.log('No bootstrap available; the service will sync from scratch') + recordBootstrapOutcome(module, 'none-published') return false } await restoreBootstrap(coin, network, module, fileName) console.log('Bootstrap installed; the service will continue from the bootstrap height') + recordBootstrapOutcome(module, 'restored') return true } catch (err) { - console.log(`WARNING: bootstrap auto-restore failed (${redactSecrets(err.message)}): the service will sync from scratch`) + const reason = redactSecrets(err.message) + console.log(`WARNING: bootstrap auto-restore failed (${reason}): the service will sync from scratch`) + recordBootstrapOutcome(module, 'failed', reason) return false } } @@ -1071,6 +1125,9 @@ module.exports = { ensureBootstrapUtxoTracker, mariaDbModuleHasData, ensureBootstrapMariaDb, + forceBootstrapRequested, + reportBootstrapOutcomes, + resetBootstrapOutcomes, // Bootstrap signing (supply-chain integrity) signBootstrapArchive, verifyBootstrapSignature, diff --git a/src/services/ModuleService.js b/src/services/ModuleService.js index 8662783..122bcab 100644 --- a/src/services/ModuleService.js +++ b/src/services/ModuleService.js @@ -1146,16 +1146,16 @@ async function installModule(module, coin, network, remoteUpdate = false, overwr // tracker (a fresh tracker creates an empty LevelDB immediately). let utxoWasFresh = false if (module === XChainService.XCHAIN_UTXO_TRACKER && !onlyExecution) { - const { utxoTrackerVolumeHasData } = require('./BootstrapService') - utxoWasFresh = !(await utxoTrackerVolumeHasData(coin, network)) + const { utxoTrackerVolumeHasData, forceBootstrapRequested } = require('./BootstrapService') + utxoWasFresh = !(await utxoTrackerVolumeHasData(coin, network)) || forceBootstrapRequested() } // Decoder/indexer freshness must also be sampled BEFORE buildAndUp; // once the service starts it fills its `blocks` table, which would // make a fresh install look populated. let mariaWasFresh = false if ((module === XChainService.XCHAIN_DECODER || module === XChainService.XCHAIN_INDEXER) && !onlyExecution) { - const { mariaDbModuleHasData } = require('./BootstrapService') - mariaWasFresh = !(await mariaDbModuleHasData(coin, network, module)) + const { mariaDbModuleHasData, forceBootstrapRequested } = require('./BootstrapService') + mariaWasFresh = !(await mariaDbModuleHasData(coin, network, module)) || forceBootstrapRequested() } const containerId = await buildAndUp(module, coin, network, overwriteContainerId, onlyExecution, dockerCmdArgs) if (module === XChainService.XCHAIN_DECODER || module === XChainService.XCHAIN_INDEXER) { diff --git a/src/services/NodeService.js b/src/services/NodeService.js index 8c2b5c9..e8b8a81 100644 --- a/src/services/NodeService.js +++ b/src/services/NodeService.js @@ -17,6 +17,7 @@ const { execFile } = require('child_process') const { https } = require('follow-redirects') +const dns = require('dns') const fs = require('fs') const path = require('path') const semver = require('semver') @@ -34,6 +35,76 @@ const { getDockerContainerImageName, getDockerNetwork, getDefaultConfig, validat const { statusChanged } = require('./StatusService') const { checkRemoteNodeVersion } = require('./VersionService') +// Enumerate a host's mirror addresses so a failover can dial one of them. +// Pinning the address changes nothing else: URL, SNI and certificate checks +// still run against the hostname. Returns [] when resolution fails. +async function resolveMirrorAddresses(hostname) { + try { + const records = await dns.promises.lookup(hostname, { all: true }) + return records.map((r) => ({ address: r.address, family: r.family })) + } catch { + return [] + } +} + +// Only a transport failure is worth another mirror: every mirror answers an +// HTTP status alike. Flagged on the error rather than matched from its +// message, which is wrapped by the time it is read. +function isTransportFailure(err) { + return err instanceof Error && err.transportFailure === true +} + +// One download attempt against one mirror address (or the default resolver +// when `pinned` is null). Resolves once the tarball is fully written. +function downloadTarball(downloadUrl, filePath, pinned) { + return new Promise((resolve, reject) => { + // No options argument unpinned: the common path keeps the plain + // https.get shape and does no name resolution of its own. + const options = pinned + ? { lookup: (hostname, opts, cb) => cb(null, pinned.address, pinned.family) } + : null + const target = pinned ? `${downloadUrl} via ${pinned.address}` : downloadUrl + const file = fs.createWriteStream(filePath) + let settled = false + const fail = (err, transportFailure = true) => { + if (settled) return + settled = true + file.destroy() + // Name the URL, the mirror dialled and the cause: a broken mirror is + // otherwise indistinguishable from a broken installer. + const wrapped = new Error(`Bitcoin Core download failed from ${target}: ${err.message}`, { cause: err }) + wrapped.transportFailure = transportFailure + reject(wrapped) + } + + const onResponse = (response) => { + // Fail closed on a non-success response (404 / redirect to an + // error page / etc.) instead of piping an HTML error body into + // the tarball and only discovering it later. + if (response.statusCode !== 200) { + response.resume() // drain + fail(new Error(`HTTP ${response.statusCode}`), false) + return + } + response.pipe(file) + response.on("error", fail) + file.on("error", fail) + file.on("finish", () => { + if (settled) return + settled = true + file.close() + resolve(true) + }) + } + + const request = options ? https.get(downloadUrl, options, onResponse) : https.get(downloadUrl, onResponse) + + // Surface transport-level failures (DNS, connection reset, TLS) as a + // rejection instead of leaving the promise to hang forever. + request.on("error", fail) + }) +} + async function getCryptoNode(coin, network, version) { if (coin === Coin.BITCOIN) { if (version.startsWith("v")) version = version.substring(1) @@ -46,7 +117,6 @@ async function getCryptoNode(coin, network, version) { // XCHAIN_NODE_CRYPTO_NODES_DIR (the documented big-volume setup) // starts empty, and createWriteStream does not create directories. fs.mkdirSync(destination, { recursive: true }) - const bitcoinNodeFile = fs.createWriteStream(filePath) // Pick the right prebuilt tarball for the host architecture. // bitcoincore.org publishes x86_64-linux-gnu and aarch64-linux-gnu builds. const archMap = { x64: 'x86_64', arm64: 'aarch64' } @@ -54,67 +124,71 @@ async function getCryptoNode(coin, network, version) { if (!arch) throw new Error("Unsupported architecture for Bitcoin Core download: " + process.arch) const downloadUrl = "https://bitcoincore.org/bin/bitcoin-core-" + version + "/bitcoin-" + version + "-" + arch + "-linux-gnu.tar.gz" - await new Promise((resolve, reject) => { - const request = https.get(downloadUrl, (response) => { - // Fail closed on a non-success response (404 / redirect to an - // error page / etc.) instead of piping an HTML error body into - // the tarball and only discovering it later. - if (response.statusCode !== 200) { - response.resume() // drain - reject(new Error(`Bitcoin Core download failed: HTTP ${response.statusCode} from ${downloadUrl}`)) - return + // One attempt at the resolver's choice; only a transport failure widens + // into a mirror-by-mirror search, so a healthy install pays nothing. + const failures = [] + let downloaded = false + try { + await downloadTarball(downloadUrl, filePath, null) + downloaded = true + } catch (err) { + failures.push(err.message) + // A partial file would otherwise be hashed as if it were the download. + try { fs.rmSync(filePath, { force: true }) } catch { /* best-effort */ } + if (!isTransportFailure(err)) throw err + + const mirrors = await resolveMirrorAddresses(new URL(downloadUrl).hostname) + for (const pinned of mirrors) { + console.log(`Retrying the bitcoin node download via mirror ${pinned.address}...`) + try { + await downloadTarball(downloadUrl, filePath, pinned) + downloaded = true + break + } catch (retryErr) { + failures.push(retryErr.message) + try { fs.rmSync(filePath, { force: true }) } catch { /* best-effort */ } } + } + } + if (!downloaded) { + throw new Error( + "Couldn't download the bitcoin node. Every mirror for " + downloadUrl + " failed:\n " + + failures.join("\n ") + + "\nIf these are certificate errors the mirror is serving an incomplete chain, not your CA store." + + " Fetch the tarball with curl --resolve against a working mirror, put it at " + filePath + + ", and re-run install: the pinned SHA-256 is still verified before it is used." + ) + } - response.pipe(bitcoinNodeFile) - - bitcoinNodeFile.on("error", (err) => { - console.log("An error happened while trying to download the bitcoin node") - reject(err) - }) - - bitcoinNodeFile.on("finish", async () => { - bitcoinNodeFile.close() - try { - // Supply-chain guard: bitcoind is a prebuilt binary fetched - // straight from bitcoincore.org over the wire. Verify the - // downloaded tarball against the project's published - // SHA-256 (github_hashes.json, sourced from the GPG-signed - // SHA256SUMS) BEFORE decompressing, so a tampered or - // truncated download can never reach the build/run path. - // Fails closed: unknown version/arch or any mismatch throws. - await gitHubDownloader.verifyFileHash(filePath, 'bitcoin/bitcoin', 'v' + version, arch) - - console.log("Decompressing bitcoin node files...") - await decompressTarGz(filePath) - - if (fs.existsSync(destination + "/bitcoin")) { - if (semver.gte(nodeVersion, "14.14.0")) { - fs.rmSync(destination + "/bitcoin", { recursive: true, force: true }) - } else { - fs.rmdirSync(destination + "/bitcoin", { recursive: true }) - } - } - - fs.renameSync(destination + "/bitcoin-" + version, destination + "/bitcoin") - fs.writeFileSync(destination + "/bitcoin/" + NODE_VERSION_FILE_NAME, version) - } catch (err) { - // Remove the unverified/failed tarball so a later retry - // re-downloads cleanly instead of trusting a cached bad file. - try { fs.rmSync(filePath, { force: true }) } catch { /* best-effort */ } - reject(err) - return - } - resolve(true) - }) - }) + try { + // Supply-chain guard: bitcoind is a prebuilt binary fetched + // straight from bitcoincore.org over the wire. Verify the + // downloaded tarball against the project's published + // SHA-256 (github_hashes.json, sourced from the GPG-signed + // SHA256SUMS) BEFORE decompressing, so a tampered or + // truncated download can never reach the build/run path. + // Fails closed: unknown version/arch or any mismatch throws. + await gitHubDownloader.verifyFileHash(filePath, 'bitcoin/bitcoin', 'v' + version, arch) + + console.log("Decompressing bitcoin node files...") + await decompressTarGz(filePath) + + if (fs.existsSync(destination + "/bitcoin")) { + if (semver.gte(nodeVersion, "14.14.0")) { + fs.rmSync(destination + "/bitcoin", { recursive: true, force: true }) + } else { + fs.rmdirSync(destination + "/bitcoin", { recursive: true }) + } + } - // Surface transport-level failures (DNS, connection reset, TLS) as a - // rejection instead of leaving the promise to hang forever. - request.on("error", (err) => { - console.log("An error happened while trying to download the bitcoin node") - reject(err) - }) - }) + fs.renameSync(destination + "/bitcoin-" + version, destination + "/bitcoin") + fs.writeFileSync(destination + "/bitcoin/" + NODE_VERSION_FILE_NAME, version) + } catch (err) { + // Remove the unverified/failed tarball so a later retry + // re-downloads cleanly instead of trusting a cached bad file. + try { fs.rmSync(filePath, { force: true }) } catch { /* best-effort */ } + throw err + } } else if (coin === Coin.DOGECOIN) { await gitHubDownloader.downloadRepoVersion("dogecoin", "dogecoin", version, { outputPath: cryptoNodesDir + "/dogecoin" }) } else if (coin === Coin.LITECOIN) { @@ -467,8 +541,8 @@ async function installNode(coin, network) { console.log("Downloading xchain-utxo-tracker...") await cloneGit(XChainService.XCHAIN_UTXO_TRACKER, true) console.log("Building xchain-utxo-tracker...") - const { utxoTrackerVolumeHasData, ensureBootstrapUtxoTracker } = require('./BootstrapService') - const utxoWasFresh = !(await utxoTrackerVolumeHasData(coin, network)) + const { utxoTrackerVolumeHasData, ensureBootstrapUtxoTracker, forceBootstrapRequested } = require('./BootstrapService') + const utxoWasFresh = !(await utxoTrackerVolumeHasData(coin, network)) || forceBootstrapRequested() await buildAndUp(XChainService.XCHAIN_UTXO_TRACKER, coin, network) if (utxoWasFresh) await ensureBootstrapUtxoTracker(coin, network) diff --git a/test/unit/BootstrapService.test.js b/test/unit/BootstrapService.test.js index 7397e71..f9492d0 100644 --- a/test/unit/BootstrapService.test.js +++ b/test/unit/BootstrapService.test.js @@ -606,6 +606,133 @@ describe('BootstrapService', function () { }) }) + // A restore that does not happen costs hours of rescanning from block 0, so + // the run must say so and must offer a way to take it again: a service that + // starts scratch-syncing reads as populated to every later run. + describe('bootstrap restore is reported, not just logged', function () { + + afterEach(function () { + delete process.env.XCHAIN_NODE_NO_BOOTSTRAP + delete process.env.XCHAIN_NODE_FORCE_BOOTSTRAP + }) + + function captureReport(bs) { + const lines = [] + const realLog = console.log + console.log = (...args) => lines.push(args.join(' ')) + try { bs.reportBootstrapOutcomes() } finally { console.log = realLog } + return lines.join('\n') + } + + it('says nothing at all when no bootstrap was attempted', function () { + const bs = loadBootstrapService(makeStubs()) + bs.resetBootstrapOutcomes() + expect(captureReport(bs)).to.equal('') + }) + + it('names the service and the reason when a restore fails', async function () { + const stubs = makeStubs() + stubs.axios.rejects(new Error('EACCES: permission denied')) + stubs.fs.existsSync.returns(true) + const bs = loadBootstrapService(stubs) + bs.resetBootstrapOutcomes() + + await bs.ensureBootstrapUtxoTracker(COIN, NETWORK) + const report = captureReport(bs) + + expect(report).to.contain(XChainService.XCHAIN_UTXO_TRACKER) + expect(report).to.contain('NOT restored') + expect(report).to.contain('EACCES') + // The operator has to be told the run is now a from-scratch sync and + // how to take the restore again; that is the whole point of the summary. + expect(report).to.contain('block 0') + expect(report).to.contain('XCHAIN_NODE_FORCE_BOOTSTRAP') + }) + + it('distinguishes "none published" from a failure', async function () { + const stubs = makeStubs() + stubs.axios.resolves({ status: 404, headers: {}, data: new PassThrough() }) + stubs.fs.existsSync.returns(true) + const bs = loadBootstrapService(stubs) + bs.resetBootstrapOutcomes() + + await bs.ensureBootstrapUtxoTracker(COIN, NETWORK) + const report = captureReport(bs) + + expect(report).to.contain('none published') + expect(report).to.not.contain('NOT restored') + expect(report).to.not.contain('XCHAIN_NODE_FORCE_BOOTSTRAP') + }) + + it('reports a disabled run as disabled rather than failed', async function () { + process.env.XCHAIN_NODE_NO_BOOTSTRAP = '1' + const bs = loadBootstrapService(makeStubs()) + bs.resetBootstrapOutcomes() + + await bs.ensureBootstrapUtxoTracker(COIN, NETWORK) + const report = captureReport(bs) + + expect(report).to.contain('disabled by XCHAIN_NODE_NO_BOOTSTRAP') + expect(report).to.not.contain('NOT restored') + }) + + it('reports each service separately when several fail in one run', async function () { + const stubs = makeStubs() + stubs.axios.rejects(new Error('EACCES: permission denied')) + stubs.fs.existsSync.returns(true) + const bs = loadBootstrapService(stubs) + bs.resetBootstrapOutcomes() + + await bs.ensureBootstrapUtxoTracker(COIN, NETWORK) + await bs.ensureBootstrapMariaDb(COIN, NETWORK, XChainService.XCHAIN_DECODER) + const report = captureReport(bs) + + expect(report).to.contain(XChainService.XCHAIN_UTXO_TRACKER) + expect(report).to.contain(XChainService.XCHAIN_DECODER) + }) + + it('starts a fresh report per run instead of replaying the last one', async function () { + const stubs = makeStubs() + stubs.axios.rejects(new Error('EACCES: permission denied')) + stubs.fs.existsSync.returns(true) + const bs = loadBootstrapService(stubs) + bs.resetBootstrapOutcomes() + await bs.ensureBootstrapUtxoTracker(COIN, NETWORK) + expect(captureReport(bs)).to.contain('NOT restored') + + bs.resetBootstrapOutcomes() + expect(captureReport(bs)).to.equal('') + }) + }) + + describe('forceBootstrapRequested()', function () { + + afterEach(function () { + delete process.env.XCHAIN_NODE_FORCE_BOOTSTRAP + }) + + it('is off when the variable is unset, so a healthy service is never wiped', function () { + const bs = loadBootstrapService(makeStubs()) + expect(bs.forceBootstrapRequested()).to.be.false + }) + + it('is off for an explicit 0 or empty value', function () { + const bs = loadBootstrapService(makeStubs()) + for (const v of ['0', '']) { + process.env.XCHAIN_NODE_FORCE_BOOTSTRAP = v + expect(bs.forceBootstrapRequested(), `value ${JSON.stringify(v)}`).to.be.false + } + }) + + it('is on for a set value, which is what re-offers a spent restore', function () { + const bs = loadBootstrapService(makeStubs()) + for (const v of ['1', 'true', 'yes']) { + process.env.XCHAIN_NODE_FORCE_BOOTSTRAP = v + expect(bs.forceBootstrapRequested(), `value ${v}`).to.be.true + } + }) + }) + describe('ensureBootstrapUtxoTracker()', function () { afterEach(function () { diff --git a/test/unit/NodeService.test.js b/test/unit/NodeService.test.js index c0090cc..9e5df4c 100644 --- a/test/unit/NodeService.test.js +++ b/test/unit/NodeService.test.js @@ -68,6 +68,10 @@ function loadNodeService(stubs) { return proxyquire('../../src/services/NodeService', { 'child_process': { execFile: stubs.execFile }, 'follow-redirects': { https: stubs.https || { get: sinon.stub() } }, + // Stubbed so no test resolves a real hostname: the mirror failover only + // enumerates addresses after a transport failure, and a unit suite must + // not depend on what DNS answers that day. + 'dns': stubs.dns || { promises: { lookup: async () => [] } }, 'fs': stubs.fs, 'semver': require('semver'), '../state': { @@ -136,7 +140,8 @@ function loadNodeService(stubs) { }, './BootstrapService': { utxoTrackerVolumeHasData: sinon.stub().resolves(true), - ensureBootstrapUtxoTracker: sinon.stub().resolves() + ensureBootstrapUtxoTracker: sinon.stub().resolves(), + forceBootstrapRequested: () => false } }) } @@ -144,6 +149,9 @@ function loadNodeService(stubs) { function makeFakeHttps(stubs, { decompressErr = null, statusCode = 200 } = {}) { const writableEmitter = new EventEmitter() writableEmitter.close = sinon.stub() + // A real fs WriteStream has destroy(); the download path calls it to release + // the handle on a failed attempt before the partial file is removed. + writableEmitter.destroy = sinon.stub() stubs.fs.createWriteStream.returns(writableEmitter) // On decompressTarGz resolution/rejection, control the finish event @@ -304,6 +312,112 @@ describe('NodeService: getCryptoNode()', function () { } }) + // One name, several mirrors, not equivalent: a leaf-only certificate chain + // is fatal to Node where curl recovers via AIA, and a plain retry keeps + // landing on the same address. + describe('a broken mirror is not the whole install', function () { + + // Fails the unpinned attempt the way a bad TLS chain does, then serves + // 200 to any attempt pinned to a specific address. + function makeFailoverHttps(stubs, { failures = 1 } = {}) { + const writableEmitter = new EventEmitter() + writableEmitter.close = sinon.stub() + writableEmitter.destroy = sinon.stub() + stubs.fs.createWriteStream.returns(writableEmitter) + + let seen = 0 + const get = sinon.stub().callsFake((url, optionsOrCb, maybeCb) => { + const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb + const request = new EventEmitter() + seen++ + if (seen <= failures) { + setImmediate(() => request.emit('error', new Error('unable to verify the first certificate'))) + return request + } + const response = new EventEmitter() + response.pipe = sinon.stub() + response.resume = sinon.stub() + response.statusCode = 200 + cb(response) + setImmediate(() => writableEmitter.emit('finish')) + return request + }) + return { get } + } + + let origArch + beforeEach(function () { + origArch = process.arch + Object.defineProperty(process, 'arch', { value: 'x64', configurable: true }) + }) + afterEach(function () { + Object.defineProperty(process, 'arch', { value: origArch, configurable: true }) + }) + + it('retries on another mirror address after a TLS failure and succeeds', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFailoverHttps(stubs, { failures: 1 }) + stubs.dns = { promises: { lookup: async () => [{ address: '198.251.83.116', family: 4 }] } } + stubs.fs.existsSync.returns(false) + + const ns = loadNodeService(stubs) + await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') + + expect(stubs.https.get.callCount).to.equal(2) + // The retry must pin the address while still requesting the same URL, + // so the certificate is still checked against the hostname. + const retryArgs = stubs.https.get.secondCall.args + expect(retryArgs[0]).to.contain('https://bitcoincore.org/') + expect(retryArgs[1]).to.have.property('lookup').that.is.a('function') + // The download still has to clear the pinned hash before it is used. + expect(stubs.gitHubDownloader.verifyFileHash.called).to.be.true + }) + + it('does not try other mirrors for an HTTP status, where every mirror agrees', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFakeHttps(stubs, { statusCode: 404 }) + stubs.dns = { promises: { lookup: async () => [{ address: '198.251.83.116', family: 4 }] } } + + const ns = loadNodeService(stubs) + let threw = null + try { await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') } catch (err) { threw = err } + + expect(threw).to.be.an('error') + expect(stubs.https.get.callCount).to.equal(1) + }) + + it('names the URL, the cause and a way forward when every mirror fails', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFailoverHttps(stubs, { failures: 99 }) + stubs.dns = { promises: { lookup: async () => [{ address: '194.204.0.12', family: 4 }] } } + + const ns = loadNodeService(stubs) + let threw = null + try { await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') } catch (err) { threw = err } + + expect(threw).to.be.an('error') + // The old message named neither the URL nor the cause, so a broken + // mirror was indistinguishable from a broken installer. + expect(threw.message).to.contain('bitcoincore.org') + expect(threw.message).to.contain('unable to verify the first certificate') + expect(threw.message).to.contain('curl --resolve') + expect(stubs.gitHubDownloader.verifyFileHash.called).to.be.false + }) + + it('still makes one attempt when the name cannot be resolved at all', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFailoverHttps(stubs, { failures: 99 }) + stubs.dns = { promises: { lookup: async () => { throw new Error('EAI_AGAIN') } } } + + const ns = loadNodeService(stubs) + let threw = null + try { await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') } catch (err) { threw = err } + + expect(threw).to.be.an('error') + expect(stubs.https.get.callCount).to.equal(1) + }) + }) + it('downloads dogecoin node via gitHubDownloader', async function () { const stubs = makeNodeServiceStubs() const ns = loadNodeService(stubs) @@ -902,7 +1016,8 @@ describe('NodeService: installNode()', function () { }, './BootstrapService': { utxoTrackerVolumeHasData: sinon.stub().resolves(true), - ensureBootstrapUtxoTracker: sinon.stub().resolves() + ensureBootstrapUtxoTracker: sinon.stub().resolves(), + forceBootstrapRequested: () => false } }) @@ -943,7 +1058,7 @@ describe('NodeService: installNode()', function () { './DockerService': { createDockerNetwork: sinon.stub().resolves(), forceRemoveContainerByName: sinon.stub().resolves(true) }, './DatabaseService': { buildDatabaseModule: sinon.stub().resolves(), setDatabaseParameters: sinon.stub().resolves() }, './ModuleService': { cloneGit: cloneGitStub, buildAndUp: buildAndUpStub, assertNoHostPortConflicts: sinon.stub().resolves() }, - './BootstrapService': { utxoTrackerVolumeHasData: sinon.stub().resolves(true), ensureBootstrapUtxoTracker: sinon.stub().resolves() } + './BootstrapService': { utxoTrackerVolumeHasData: sinon.stub().resolves(true), ensureBootstrapUtxoTracker: sinon.stub().resolves(), forceBootstrapRequested: () => false } }) const result = await ns.installNode('bitcoin', 'mainnet') From e65cb875b46d76f57d6377e284ffcc846379b8ec Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sun, 23 Aug 2026 08:07:29 -0700 Subject: [PATCH 04/12] Run the CLI from any working directory The launcher resolved src/index.js relative to the caller's working directory, so invoking the installed symlink from anywhere other than the checkout failed to find the entry point. Scheduled jobs, which do not change directory first, were the ones this broke. Resolve the script's own location through the symlink and change into it before starting node. --- xchain-node.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xchain-node.sh b/xchain-node.sh index 1948ca8..86ac706 100755 --- a/xchain-node.sh +++ b/xchain-node.sh @@ -20,5 +20,9 @@ # sudo ln -s ~/xchain-node/xchain-node.sh /usr/local/bin/xchain-node # ####################################################################### -#cd ~/xchain-node/ +# Resolve to the checkout this script lives in (following the +# /usr/local/bin symlink), so the CLI works from any working directory: +# cron jobs and scripts invoke `xchain-node` without cd-ing first, and a +# bare `node src/index.js` resolves against their cwd and dies. +cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" /usr/bin/env node src/index.js "$@" From 73cc57adb38065f291c86c2f7dc4ca58ae4d9aa7 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sun, 23 Aug 2026 08:12:30 -0700 Subject: [PATCH 05/12] Fix four install-path failures found on a Raspberry Pi Four defects an external operator hit reinstalling clean on ARM64, each verified against the code before it was changed. The mirror failover could never succeed. Since Node 20 autoSelectFamily is on by default, so net.connect calls a custom lookup with all:true and expects an array of records; the pinned lookup answered with the single address form, and every retry died reading address off undefined before a socket was opened. Three tests asserted the retry existed and none of them ever called it the way Node does. The remediation those failures print asked for a file and then destroyed it. It tells an operator whose mirrors all serve a broken certificate chain to fetch the tarball by hand, place it, and re-run, but the re-run went straight into the download, truncated the placed file, and deleted it in the catch. A tarball already at the path is now accepted against the pinned hash and the download skipped. The hash is checked at the point of acceptance rather than only at the gate before decompression, so a corrupt leftover from an interrupted run is still discarded and re-downloaded instead of failing the install outright. The bootstrap restore summary was printed after the install loop with nothing guarding it, so a run that threw partway printed nothing. That is the run whose partial restores an operator most needs accounted for. It is now in a finally, and the failure still propagates. A published bootstrap is not a free starting point: a service walks forward from the restored tip, and one that has aged past the chain it lands on can be left unable to continue, which for the utxo-tracker means a halt and a full rebuild. The age of the archive that was actually resolved is now reported during the download, with a warning past ten days. Ten rather than seven because testnet trackers publish weekly, so a healthy archive reaches six days old routinely and a warning that fires on healthy state is one nobody reads. --- CHANGELOG.md | 4 + src/operations/moduleOperations.js | 45 +++++---- src/services/BootstrapService.js | 38 ++++++++ src/services/NodeService.js | 45 +++++++-- test/unit/BootstrapService.test.js | 152 +++++++++++++++++++++++++++++ test/unit/NodeService.test.js | 115 +++++++++++++++++++++- test/unit/moduleOperations.test.js | 33 +++++++ 7 files changed, 400 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ef063d..66c5e1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `XCHAIN_NODE_FORCE_BOOTSTRAP=1` restores a published bootstrap over an already-populated service, for when the install that would have taken it failed. ### Fixed +- The bitcoind mirror retry now answers the address-list form Node asks for, so the retry actually dials instead of failing immediately on every supported Node version. +- A bitcoind tarball already present at the download path is used and verified against the pinned hash rather than overwritten, so the manual workaround the failure message describes now works. +- The bootstrap restore summary is printed even when the install fails partway, which is when it matters most. +- A published bootstrap older than a week is called out during the download, since a snapshot that has aged past the chain can leave a service unable to continue from it. - A failed bitcoind download now retries against the site's other mirror addresses, so one mirror serving a broken certificate chain no longer blocks the install. - Download failures name the URL, the mirror and the cause instead of a generic message. - Module clones now use public HTTPS URLs, so installs work without a GitHub SSH key. diff --git a/src/operations/moduleOperations.js b/src/operations/moduleOperations.js index 98295e1..84646ae 100644 --- a/src/operations/moduleOperations.js +++ b/src/operations/moduleOperations.js @@ -82,33 +82,36 @@ async function installModules(servicesList, ref = null) { // restores rather than replaying the first one's. require('../services/BootstrapService').resetBootstrapOutcomes() - for (const nextCoin in servicesList) { - for (const nextNetwork in servicesList[nextCoin]) { - if (nextCoin && nextNetwork) { - await createDockerNetwork(getDockerNetwork(nextCoin, nextNetwork)) - await buildDatabaseModule(nextCoin, nextNetwork) - } - for (const nextModule of servicesList[nextCoin][nextNetwork]) { - const result = await installModule(nextModule, nextCoin, nextNetwork, false, null, false, branch) - if (result === false) { - outcome.skipped.push({ module: nextModule, coin: nextCoin, network: nextNetwork, reason: 'already-installed' }) - } else { - outcome.installed.push({ module: nextModule, coin: nextCoin, network: nextNetwork }) + try { + for (const nextCoin in servicesList) { + for (const nextNetwork in servicesList[nextCoin]) { + if (nextCoin && nextNetwork) { + await createDockerNetwork(getDockerNetwork(nextCoin, nextNetwork)) + await buildDatabaseModule(nextCoin, nextNetwork) + } + for (const nextModule of servicesList[nextCoin][nextNetwork]) { + const result = await installModule(nextModule, nextCoin, nextNetwork, false, null, false, branch) + if (result === false) { + outcome.skipped.push({ module: nextModule, coin: nextCoin, network: nextNetwork, reason: 'already-installed' }) + } else { + outcome.installed.push({ module: nextModule, coin: nextCoin, network: nextNetwork }) + } } } } - } - if (outcome.skipped.length > 0) { - console.log('install: nothing to do for ' + outcome.skipped - .map(s => `${s.module} (${s.coin} ${s.network})`).join(', ') - + ' - already installed. Use `update` to rebuild.') + if (outcome.skipped.length > 0) { + console.log('install: nothing to do for ' + outcome.skipped + .map(s => `${s.module} (${s.coin} ${s.network})`).join(', ') + + ' - already installed. Use `update` to rebuild.') + } + } finally { + // In a finally because a run that throws is the one whose summary + // matters most: it leaves some services restored and some facing + // hours of resync, and the error alone does not say which. + require('../services/BootstrapService').reportBootstrapOutcomes() } - // A bootstrap that did not restore costs hours of resync, and its only - // trace was one warning far up a long install log. - require('../services/BootstrapService').reportBootstrapOutcomes() - // The explorer is installed in the shared bucket, which runs BEFORE the // coin stacks, and it learns its coins by polling the hub. So a run that // installed a coin leaves it serving 503 for up to a poll interval after diff --git a/src/services/BootstrapService.js b/src/services/BootstrapService.js index 63fe89a..bf939bc 100644 --- a/src/services/BootstrapService.js +++ b/src/services/BootstrapService.js @@ -839,6 +839,24 @@ async function utxoTrackerVolumeHasData(coin, network) { } } +// Age in whole days of the resolved archive, from the UTC +// stamp in its name (the field latest.php orders by). Null when the name +// carries none, as a hand-placed latest.tgz does. +function bootstrapArchiveAgeDays(archiveUrl, now = Date.now()) { + const stamp = /(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})/.exec(path.basename(archiveUrl || '')) + if (!stamp) return null + const [, y, mo, d, h, mi, s] = stamp + const published = Date.UTC(Number(y), Number(mo) - 1, Number(d), Number(h), Number(mi), Number(s)) + if (!Number.isFinite(published)) return null + const days = Math.floor((now - published) / 86400000) + return days >= 0 ? days : null +} + +// Days before a published archive is called out at restore time. Warn, never +// refuse: a stale archive still beats the days of scratch sync refusing costs. +// Above the weekly publish cadence, so only a publisher that missed runs trips it. +const BOOTSTRAP_STALE_AFTER_DAYS = 10 + // Stream ////latest.tgz into destDir // as latest.tgz. Returns the filename on success, null when none is published // (404). Follows the http→https redirect. Throws on other network errors. @@ -893,6 +911,24 @@ async function downloadBootstrap(coin, network, module, destDir) { // latest.tgz is served directly and its .sig sits beside it). const finalUrl = response.request && response.request.res && response.request.res.responseUrl ? response.request.res.responseUrl : url + + // Report the age during the install, not after a halt traced back to it. + // Only the tracker can be left unable to walk forward, so only it is warned + // about that; the others just resync from the archive height. + const ageDays = bootstrapArchiveAgeDays(finalUrl) + if (ageDays !== null && ageDays >= BOOTSTRAP_STALE_AFTER_DAYS) { + const consequence = module === XChainService.XCHAIN_UTXO_TRACKER + ? ' A snapshot whose tip has drifted past the chain it is restored onto can leave the tracker\n' + + ' unable to walk forward, which halts it until it is reset and rebuilt. If that happens, the\n' + + ' archive is the cause, not your host.' + : ' It still restores; the service resyncs forward from the archive height, which just takes longer\n' + + ' the older the archive is.' + console.log( + `WARNING: the published ${module} bootstrap for ${coin}/${network} is ${ageDays} days old ` + + `(${path.basename(finalUrl)}).\n` + consequence + ) + } + const sigPath = destPath + BOOTSTRAP_SIG_SUFFIX const sigResponse = await axios({ method: 'get', @@ -1128,6 +1164,8 @@ module.exports = { forceBootstrapRequested, reportBootstrapOutcomes, resetBootstrapOutcomes, + bootstrapArchiveAgeDays, + BOOTSTRAP_STALE_AFTER_DAYS, // Bootstrap signing (supply-chain integrity) signBootstrapArchive, verifyBootstrapSignature, diff --git a/src/services/NodeService.js b/src/services/NodeService.js index e8b8a81..6642113 100644 --- a/src/services/NodeService.js +++ b/src/services/NodeService.js @@ -54,15 +54,23 @@ function isTransportFailure(err) { return err instanceof Error && err.transportFailure === true } +// Answer dns.lookup with the one pinned address, in both callback shapes. +// autoSelectFamily (default since Node 20) calls it with `all: true` and +// requires an ARRAY; the single-address form fails the connect before it dials. +function pinnedLookup(pinned) { + return (hostname, opts, cb) => { + if (opts && opts.all) return cb(null, [{ address: pinned.address, family: pinned.family }]) + return cb(null, pinned.address, pinned.family) + } +} + // One download attempt against one mirror address (or the default resolver // when `pinned` is null). Resolves once the tarball is fully written. function downloadTarball(downloadUrl, filePath, pinned) { return new Promise((resolve, reject) => { // No options argument unpinned: the common path keeps the plain // https.get shape and does no name resolution of its own. - const options = pinned - ? { lookup: (hostname, opts, cb) => cb(null, pinned.address, pinned.family) } - : null + const options = pinned ? { lookup: pinnedLookup(pinned) } : null const target = pinned ? `${downloadUrl} via ${pinned.address}` : downloadUrl const file = fs.createWriteStream(filePath) let settled = false @@ -124,13 +132,29 @@ async function getCryptoNode(coin, network, version) { if (!arch) throw new Error("Unsupported architecture for Bitcoin Core download: " + process.arch) const downloadUrl = "https://bitcoincore.org/bin/bitcoin-core-" + version + "/bitcoin-" + version + "-" + arch + "-linux-gnu.tar.gz" - // One attempt at the resolver's choice; only a transport failure widens - // into a mirror-by-mirror search, so a healthy install pays nothing. + // Honour a tarball the operator placed here, which is the only route left + // when every resolved mirror serves a broken certificate chain. + // Hash-checked here so a corrupt leftover is discarded and re-downloaded + // rather than failing the install; the verify below still gates every path. const failures = [] let downloaded = false + if (fs.existsSync(filePath)) { + try { + await gitHubDownloader.verifyFileHash(filePath, 'bitcoin/bitcoin', 'v' + version, arch) + console.log(`Using the bitcoin node tarball already at ${filePath}: it matches the pinned SHA-256.`) + downloaded = true + } catch { + console.log(`Discarding the file at ${filePath}: it does not match the pinned SHA-256.`) + try { fs.rmSync(filePath, { force: true }) } catch { /* best-effort */ } + } + } + // One attempt at the resolver's choice; only a transport failure widens + // into a mirror-by-mirror search, so a healthy install pays nothing. try { - await downloadTarball(downloadUrl, filePath, null) - downloaded = true + if (!downloaded) { + await downloadTarball(downloadUrl, filePath, null) + downloaded = true + } } catch (err) { failures.push(err.message) // A partial file would otherwise be hashed as if it were the download. @@ -154,9 +178,10 @@ async function getCryptoNode(coin, network, version) { throw new Error( "Couldn't download the bitcoin node. Every mirror for " + downloadUrl + " failed:\n " + failures.join("\n ") + - "\nIf these are certificate errors the mirror is serving an incomplete chain, not your CA store." + - " Fetch the tarball with curl --resolve against a working mirror, put it at " + filePath + - ", and re-run install: the pinned SHA-256 is still verified before it is used." + "\nIf these are certificate errors the mirror is serving an incomplete chain, not your CA store," + + " and every address your resolver returns can be serving the same one." + + " Fetch the tarball with curl (which recovers the missing intermediate where Node cannot), put it at " + filePath + + ", and re-run install: a tarball already at that path is used as-is, and its pinned SHA-256 is still verified before it is used." ) } diff --git a/test/unit/BootstrapService.test.js b/test/unit/BootstrapService.test.js index f9492d0..386c962 100644 --- a/test/unit/BootstrapService.test.js +++ b/test/unit/BootstrapService.test.js @@ -477,6 +477,158 @@ describe('BootstrapService', function () { expect(stubs.axios.secondCall.args[0].url).to.match(/\/latest\.tgz\.sig$/) }) + // A published archive is not simply a head start: the service walks + // forward from the restored tip, so one that aged past the chain it + // lands on can halt the tracker with the node parked behind it. + describe('the age of the archive an operator is about to take', function () { + + const AUG_23 = Date.UTC(2026, 7, 23, 12, 0, 0) + + it('reads the publisher timestamp out of the archive name', function () { + const bs = loadBootstrapService(makeStubs()) + const url = 'https://sync.example/b/testnet-xchain-utxo-tracker-20260801_082909.tar.gz' + expect(bs.bootstrapArchiveAgeDays(url, AUG_23)).to.equal(22) + }) + + it('reads a same-day archive as zero days old, not stale', function () { + const bs = loadBootstrapService(makeStubs()) + const url = 'https://sync.example/b/testnet-xchain-utxo-tracker-20260823_042943.tar.gz' + expect(bs.bootstrapArchiveAgeDays(url, AUG_23)).to.equal(0) + }) + + it('says nothing about a hand-placed name that carries no timestamp', function () { + const bs = loadBootstrapService(makeStubs()) + expect(bs.bootstrapArchiveAgeDays('https://sync.example/b/latest.tgz', AUG_23)).to.be.null + expect(bs.bootstrapArchiveAgeDays('', AUG_23)).to.be.null + expect(bs.bootstrapArchiveAgeDays(undefined, AUG_23)).to.be.null + }) + + it('does not report a negative age for a clock-skewed future stamp', function () { + const bs = loadBootstrapService(makeStubs()) + const url = 'https://sync.example/b/testnet-xchain-utxo-tracker-20260901_000000.tar.gz' + expect(bs.bootstrapArchiveAgeDays(url, AUG_23)).to.be.null + }) + + it('warns during the download when the resolved archive has aged out', async function () { + const stubs = makeStubs() + stubs.fs.existsSync.returns(true) + + const dataStream = new PassThrough() + const writeStream = new PassThrough() + drainPassThrough(writeStream) + stubs.fs.createWriteStream.returns(writeStream) + + const staleUrl = 'https://sync.example/b/testnet-xchain-utxo-tracker-20200101_000000.tar.gz' + stubs.axios.onFirstCall().resolves({ + status: 200, + headers: { 'content-length': '100' }, + data: dataStream, + request: { res: { responseUrl: staleUrl } }, + }) + stubs.axios.onSecondCall().resolves({ status: 200, data: 'sig-bytes' }) + + const logged = [] + const origLog = console.log + console.log = (...args) => logged.push(args.join(' ')) + try { + const bs = loadBootstrapService(stubs) + const promise = bs.downloadBootstrap(COIN, NETWORK, XChainService.XCHAIN_UTXO_TRACKER, '/tmp/dest') + setImmediate(() => { dataStream.end(); writeStream.emit('finish') }) + await promise + } finally { + console.log = origLog + } + + const warning = logged.find(l => l.includes('days old')) + expect(warning, 'no staleness warning was printed').to.be.a('string') + expect(warning).to.contain('testnet-xchain-utxo-tracker-20200101_000000.tar.gz') + }) + + // A weekly publisher puts a healthy archive at six days old + // routinely, and a warning that fires on healthy state is one + // nobody reads by the second month. + it('stays quiet through a normal weekly publish cadence', async function () { + const bs = loadBootstrapService(makeStubs()) + const sixDays = Date.UTC(2026, 7, 7, 0, 0, 0) + const url = 'https://sync.example/b/testnet-xchain-utxo-tracker-20260801_000000.tar.gz' + expect(bs.bootstrapArchiveAgeDays(url, sixDays)).to.equal(6) + expect(bs.BOOTSTRAP_STALE_AFTER_DAYS).to.be.above(7) + }) + + it('names the halt for the tracker and only the slower resync for the rest', async function () { + const staleUrl = mod => `https://sync.example/b/testnet-${mod}-20200101_000000.tar.gz` + + async function warningFor(module) { + const stubs = makeStubs() + stubs.fs.existsSync.returns(true) + const dataStream = new PassThrough() + const writeStream = new PassThrough() + drainPassThrough(writeStream) + stubs.fs.createWriteStream.returns(writeStream) + stubs.axios.onFirstCall().resolves({ + status: 200, + headers: { 'content-length': '100' }, + data: dataStream, + request: { res: { responseUrl: staleUrl(module) } }, + }) + stubs.axios.onSecondCall().resolves({ status: 200, data: 'sig-bytes' }) + + const logged = [] + const origLog = console.log + console.log = (...args) => logged.push(args.join(' ')) + try { + const bs = loadBootstrapService(stubs) + const promise = bs.downloadBootstrap(COIN, NETWORK, module, '/tmp/dest') + setImmediate(() => { dataStream.end(); writeStream.emit('finish') }) + await promise + } finally { + console.log = origLog + } + return logged.find(l => l.includes('days old')) || '' + } + + expect(await warningFor(XChainService.XCHAIN_UTXO_TRACKER)).to.contain('halts it') + const decoderWarning = await warningFor(XChainService.XCHAIN_DECODER) + expect(decoderWarning).to.contain('resyncs forward') + expect(decoderWarning).to.not.contain('halts it') + }) + + it('stays quiet about an archive published today', async function () { + const stubs = makeStubs() + stubs.fs.existsSync.returns(true) + + const dataStream = new PassThrough() + const writeStream = new PassThrough() + drainPassThrough(writeStream) + stubs.fs.createWriteStream.returns(writeStream) + + const pad = n => String(n).padStart(2, '0') + const now = new Date() + const today = `${now.getUTCFullYear()}${pad(now.getUTCMonth() + 1)}${pad(now.getUTCDate())}_000000` + stubs.axios.onFirstCall().resolves({ + status: 200, + headers: { 'content-length': '100' }, + data: dataStream, + request: { res: { responseUrl: `https://sync.example/b/testnet-xchain-utxo-tracker-${today}.tar.gz` } }, + }) + stubs.axios.onSecondCall().resolves({ status: 200, data: 'sig-bytes' }) + + const logged = [] + const origLog = console.log + console.log = (...args) => logged.push(args.join(' ')) + try { + const bs = loadBootstrapService(stubs) + const promise = bs.downloadBootstrap(COIN, NETWORK, XChainService.XCHAIN_UTXO_TRACKER, '/tmp/dest') + setImmediate(() => { dataStream.end(); writeStream.emit('finish') }) + await promise + } finally { + console.log = origLog + } + + expect(logged.find(l => l.includes('days old'))).to.be.undefined + }) + }) + it('creates destDir when it does not exist', async function () { const stubs = makeStubs() stubs.fs.existsSync.returns(false) diff --git a/test/unit/NodeService.test.js b/test/unit/NodeService.test.js index 9e5df4c..389b568 100644 --- a/test/unit/NodeService.test.js +++ b/test/unit/NodeService.test.js @@ -400,7 +400,10 @@ describe('NodeService: getCryptoNode()', function () { // mirror was indistinguishable from a broken installer. expect(threw.message).to.contain('bitcoincore.org') expect(threw.message).to.contain('unable to verify the first certificate') - expect(threw.message).to.contain('curl --resolve') + expect(threw.message).to.contain('curl') + // The remediation names a path, so it has to name the path the + // installer will actually read a placed tarball back from. + expect(threw.message).to.contain('/crypto_nodes/bitcoin/bitcoin28.1.tar.gz') expect(stubs.gitHubDownloader.verifyFileHash.called).to.be.false }) @@ -416,6 +419,116 @@ describe('NodeService: getCryptoNode()', function () { expect(threw).to.be.an('error') expect(stubs.https.get.callCount).to.equal(1) }) + + // Asserting the lookup exists proves nothing: under autoSelectFamily + // net.connect calls it with { all: true } and an answer in the + // single-address form fails the connect before a socket is opened. + it('answers the all:true form Node actually calls it with', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFailoverHttps(stubs, { failures: 1 }) + stubs.dns = { promises: { lookup: async () => [{ address: '198.251.83.116', family: 4 }] } } + + const ns = loadNodeService(stubs) + await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') + + const { lookup } = stubs.https.get.secondCall.args[1] + + // What Node passes under autoSelectFamily: an array of records, or + // the connect attempt throws before it dials. + const all = await new Promise((resolve, reject) => + lookup('bitcoincore.org', { family: 0, hints: 32, all: true }, + (err, res) => err ? reject(err) : resolve(res))) + expect(all).to.be.an('array').with.lengthOf(1) + expect(all[0]).to.include({ address: '198.251.83.116', family: 4 }) + + // The legacy shape still has to work: options without `all` take the + // (err, address, family) callback instead. + const single = await new Promise((resolve, reject) => + lookup('bitcoincore.org', { family: 0 }, + (err, address, family) => err ? reject(err) : resolve({ address, family }))) + expect(single).to.deep.equal({ address: '198.251.83.116', family: 4 }) + }) + }) + + // The failure message tells an operator to place the tarball and re-run, + // the only escape when every resolved address serves the same broken + // certificate chain. These hold that instruction to being true. + describe('a tarball the operator placed by hand is used, not destroyed', function () { + + let origArch + beforeEach(function () { + origArch = process.arch + Object.defineProperty(process, 'arch', { value: 'x64', configurable: true }) + }) + afterEach(function () { + Object.defineProperty(process, 'arch', { value: origArch, configurable: true }) + }) + + function existingTarballFs(stubs) { + // Only the tarball is present; the extracted directory is not, so + // the post-verify rename path runs as it does after a download. + stubs.fs.existsSync.callsFake(p => String(p).endsWith('.tar.gz')) + } + + it('skips the download entirely when the tarball is already at the path', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFakeHttps(stubs) + existingTarballFs(stubs) + + const ns = loadNodeService(stubs) + await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') + + expect(stubs.https.get.called).to.be.false + expect(stubs.fs.createWriteStream.called).to.be.false + }) + + it('still verifies the placed tarball against the pinned hash before using it', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFakeHttps(stubs) + existingTarballFs(stubs) + + const ns = loadNodeService(stubs) + await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') + + // Once to decide the placed file is trustworthy at all, and once at + // the gate every path goes through before decompression. The gate is + // deliberately not conditional on how the bytes arrived. + expect(stubs.gitHubDownloader.verifyFileHash.callCount).to.equal(2) + expect(stubs.decompressTarGz.calledOnce).to.be.true + }) + + // Trusting a placed file must not cost the self-heal a half-written + // leftover depends on: bytes that are not the pinned ones get discarded + // and re-downloaded, never handed to the install. + it('discards a file that fails the pinned hash and downloads instead', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFakeHttps(stubs) + existingTarballFs(stubs) + stubs.gitHubDownloader.verifyFileHash + .onFirstCall().rejects(new Error('SHA-256 mismatch')) + .onSecondCall().resolves() + + const ns = loadNodeService(stubs) + await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') + + expect(stubs.fs.rmSync.called).to.be.true + expect(stubs.https.get.calledOnce).to.be.true + expect(stubs.decompressTarGz.calledOnce).to.be.true + }) + + it('never decompresses bytes that fail the pinned hash', async function () { + const stubs = makeNodeServiceStubs() + stubs.https = makeFakeHttps(stubs) + existingTarballFs(stubs) + stubs.gitHubDownloader.verifyFileHash.rejects(new Error('SHA-256 mismatch')) + + const ns = loadNodeService(stubs) + let threw = null + try { await ns.getCryptoNode('bitcoin', 'mainnet', 'v28.1') } catch (err) { threw = err } + + expect(threw).to.be.an('error') + expect(stubs.decompressTarGz.called).to.be.false + }) }) it('downloads dogecoin node via gitHubDownloader', async function () { diff --git a/test/unit/moduleOperations.test.js b/test/unit/moduleOperations.test.js index 3670eff..f21372d 100644 --- a/test/unit/moduleOperations.test.js +++ b/test/unit/moduleOperations.test.js @@ -64,6 +64,10 @@ function makeStubs() { execFile: sinon.stub(), fs: { existsSync: sinon.stub().returns(false) + }, + bootstrapService: { + resetBootstrapOutcomes: sinon.stub(), + reportBootstrapOutcomes: sinon.stub() } } } @@ -124,6 +128,7 @@ function loadOperations(stubs) { '../services/StatusService': { statusChanged: stubs.statusChanged }, + '../services/BootstrapService': stubs.bootstrapService, 'child_process': { execFile: stubs.execFile }, 'fs': stubs.fs, 'util': { @@ -215,6 +220,34 @@ describe('moduleOperations', function () { { module: 'xchain-encoder', coin: 'bitcoin', network: 'mainnet', reason: 'already-installed' } ]) }) + + // A failed install is exactly where the summary earns its place: it + // leaves some services restored and some facing days of resync, and + // the error alone does not say which. + it('reports the bootstrap outcomes even when the install throws', async function () { + const stubs = makeStubs() + stubs.installModule.rejects(new Error("Couldn't download the bitcoin node")) + const ops = loadOperations(stubs) + + let threw = null + try { + await ops.installModules({ bitcoin: { mainnet: ['node'] } }) + } catch (err) { threw = err } + + expect(threw).to.be.an('error') + expect(stubs.bootstrapService.reportBootstrapOutcomes.calledOnce).to.be.true + // The failure still surfaces: the summary is added to it, not + // substituted for it. + expect(threw.message).to.contain("Couldn't download the bitcoin node") + }) + + it('still reports the bootstrap outcomes on a clean install', async function () { + const stubs = makeStubs() + const ops = loadOperations(stubs) + await ops.installModules({ bitcoin: { mainnet: ['xchain-encoder'] } }) + expect(stubs.bootstrapService.resetBootstrapOutcomes.calledOnce).to.be.true + expect(stubs.bootstrapService.reportBootstrapOutcomes.calledOnce).to.be.true + }) }) // ------------------------------------------------------------------- From 06746018314b5c8b650abeb57f8872f134e5c2c0 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sun, 23 Aug 2026 16:23:02 -0700 Subject: [PATCH 06/12] Refuse a one-sided reset of the decoder and indexer The indexer tracks reorgs by a decoder event id, and the decoder never deletes those rows, so wiping the decoder alone restarts the ids underneath a cursor that now points past them. The indexer then aborts with a reorg-cursor error and stops committing blocks. An operator recovering a stuck decoder hit exactly this, and reset let them do it. `reset xchain-decoder` now checks for an installed indexer and refuses before anything is stopped or wiped, naming the joint form rather than just the problem. `--with-indexer` resets the pair together, which also clears the hub price fence the wiped indexer would otherwise trip. The coupling is one-directional, so an indexer-only reset stays allowed: it re-derives from an intact decoder, which is an ordinary reindex. A decoder-only reset also stays allowed where no indexer is installed to strand. Six tests, falsified against the unguarded path. --- CHANGELOG.md | 1 + src/cli.js | 4 +- src/operations/moduleOperations.js | 27 +++++++- test/unit/moduleOperations.test.js | 104 ++++++++++++++++++++++++++++- 4 files changed, 132 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66c5e1c..5677220 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `XCHAIN_NODE_FORCE_BOOTSTRAP=1` restores a published bootstrap over an already-populated service, for when the install that would have taken it failed. ### Fixed +- `reset xchain-decoder` now refuses while an indexer is installed and names the `--with-indexer` joint form, because resetting one half of the pair leaves the other unable to commit blocks. - The bitcoind mirror retry now answers the address-list form Node asks for, so the retry actually dials instead of failing immediately on every supported Node version. - A bitcoind tarball already present at the download path is used and verified against the pinned hash rather than overwritten, so the manual workaround the failure message describes now works. - The bootstrap restore summary is printed even when the install fails partway, which is when it matters most. diff --git a/src/cli.js b/src/cli.js index 2ec61ab..54b4812 100644 --- a/src/cli.js +++ b/src/cli.js @@ -495,8 +495,10 @@ gate could report them, so the cron exited 0 while a consumer archive went stale .argument('', '(bitcoin, litecoin, dogecoin)') .argument('', '(mainnet, testnet, regtest)') .option('--yes', 'Skip the destructive-reset confirmation prompt (for CI/scripted resets)') + .option('--with-indexer', 'Reset xchain-indexer alongside xchain-decoder; the pair is only coherent when both move together') .action(async (service, chain, network, options) => { - const confirmed = await resetModules(service, chain, network, !!(options && options.yes)) + const confirmed = await resetModules(service, chain, network, + !!(options && options.yes), !!(options && options.withIndexer)) return process.exit(confirmed ? 0 : 1) }) diff --git a/src/operations/moduleOperations.js b/src/operations/moduleOperations.js index 84646ae..5cca121 100644 --- a/src/operations/moduleOperations.js +++ b/src/operations/moduleOperations.js @@ -726,7 +726,7 @@ const RESETTABLE_SERVICES = [ XChainService.XCHAIN_INDEXER ] -async function resetModules(service, coin, network, force = false) { +async function resetModules(service, coin, network, force = false, withIndexer = false) { if (!RESETTABLE_SERVICES.includes(service)) { throw new Error("reset: unknown service '" + service + "'; expected one of " + RESETTABLE_SERVICES.join(', ')) @@ -743,7 +743,30 @@ async function resetModules(service, coin, network, force = false) { const resetNode = resetAll || service === NODE_MODULE_NAME const resetUtxoTracker = resetAll || service === XChainService.XCHAIN_UTXO_TRACKER const resetDecoder = resetAll || service === XChainService.XCHAIN_DECODER - const resetIndexer = resetAll || service === XChainService.XCHAIN_INDEXER + const resetIndexer = resetAll || service === XChainService.XCHAIN_INDEXER || (withIndexer && resetDecoder) + + // The indexer's rollback cursor IS a decoder `events` id, and the decoder + // never deletes those rows, so wiping the decoder alone restarts the ids + // under a cursor that now points past them: the indexer fails RE-1 and stops + // committing. The pair only has a coherent state when both move together. + // Asymmetric on purpose - resetting the indexer alone re-derives it from an + // intact decoder, which is an ordinary reindex and stays allowed. + if (resetDecoder && !resetIndexer) { + let indexerInstalled = null + try { + indexerInstalled = await db.getModuleContainer(XChainService.XCHAIN_INDEXER, coin, network) + } catch { /* registry unreadable: fall through, nothing to strand that we can prove */ } + if (indexerInstalled) { + console.log(`Aborted: resetting ${XChainService.XCHAIN_DECODER} alone would leave ` + + `${XChainService.XCHAIN_INDEXER} incoherent. No data was touched.`) + console.log(" The indexer tracks reorgs by a decoder event id. Wiping the decoder restarts") + console.log(" those ids, so the indexer would abort with a reorg-cursor error (RE-1) and stop") + console.log(" committing blocks until both are rebuilt together.") + console.log(` Reset the pair: xchain-node reset ${XChainService.XCHAIN_DECODER} ${coin} ${network} --with-indexer`) + console.log(` Or the whole stack (also re-syncs the chain): xchain-node reset all ${coin} ${network}`) + return false + } + } // Relocated blocks/txindex host paths (XCHAIN_NODE_BLOCKS_DIR mode): these // live OUTSIDE the in-datadir path the node wipe clears, so they must be diff --git a/test/unit/moduleOperations.test.js b/test/unit/moduleOperations.test.js index f21372d..a315320 100644 --- a/test/unit/moduleOperations.test.js +++ b/test/unit/moduleOperations.test.js @@ -1051,7 +1051,7 @@ describe('moduleOperations', function () { stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) const ops = loadOperations(stubs) const clock = sinon.useFakeTimers() - const promise = ops.resetModules('xchain-decoder', 'bitcoin', 'mainnet', true) + const promise = ops.resetModules('xchain-decoder', 'bitcoin', 'mainnet', true, true) await clock.tickAsync(6000) clock.restore() const result = await promise @@ -1136,9 +1136,14 @@ describe('moduleOperations', function () { expect(stubs.clearHubPriceIngestWatermark.calledBefore(stubs.startContainer)).to.be.true }) + // A decoder-only reset is reachable only where no indexer is installed to + // strand; with one present the pair guard refuses. The fence belongs to the + // indexer's push generations, so an untouched indexer keeps its fence. it('leaves the fence alone on a decoder-only reset (that chain keeps pushing prices)', async function () { const stubs = makeStubs() stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + stubs.db.getModuleContainer.callsFake(async (module) => + module === 'xchain-indexer' ? null : 'container-id-123') const ops = loadOperations(stubs) const clock = sinon.useFakeTimers() const promise = ops.resetModules('xchain-decoder', 'bitcoin', 'mainnet', true) @@ -1148,6 +1153,103 @@ describe('moduleOperations', function () { expect(stubs.clearHubPriceIngestWatermark.called).to.be.false }) + // The indexer tracks reorgs by a decoder event id, and the decoder never + // deletes those rows, so wiping the decoder alone restarts the ids under a + // cursor pointing past them and the indexer aborts RE-1. The pair is only + // coherent when both move together, so a one-sided reset is refused. + describe('the coupled decoder/indexer pair', function () { + + it('refuses a decoder-only reset while an indexer is installed', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + const ops = loadOperations(stubs) + + const result = await ops.resetModules('xchain-decoder', 'bitcoin', 'mainnet', true) + + expect(result).to.be.false + // Refused BEFORE anything destructive, not partway through. + expect(stubs.resetDatabases.called).to.be.false + expect(stubs.stopContainer.called).to.be.false + }) + + it('names the joint form in the refusal, so the remedy is runnable', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + const logged = [] + const log = sinon.stub(console, 'log').callsFake((...a) => logged.push(a.join(' '))) + try { + const ops = loadOperations(stubs) + await ops.resetModules('xchain-decoder', 'bitcoin', 'mainnet', true) + } finally { + log.restore() + } + const text = logged.join('\n') + expect(text).to.contain('--with-indexer') + expect(text).to.contain('bitcoin mainnet') + expect(text).to.contain('No data was touched') + }) + + it('resets both halves together when the joint form is used', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + const ops = loadOperations(stubs) + const clock = sinon.useFakeTimers() + const promise = ops.resetModules('xchain-decoder', 'bitcoin', 'mainnet', true, true) + await clock.tickAsync(6000) + clock.restore() + + expect(await promise).to.be.true + const modules = stubs.resetDatabases.firstCall.args[2] + expect(modules).to.have.members(['xchain-decoder', 'xchain-indexer']) + // A wiped indexer restarts its push generations, so the hub fence has + // to be cleared on this path or the chain's price rail dies silently. + expect(stubs.clearHubPriceIngestWatermark.called).to.be.true + }) + + it('allows a decoder-only reset when no indexer is installed to strand', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + stubs.db.getModuleContainer.callsFake(async (module) => + module === 'xchain-indexer' ? null : 'container-id-123') + const ops = loadOperations(stubs) + const clock = sinon.useFakeTimers() + const promise = ops.resetModules('xchain-decoder', 'bitcoin', 'mainnet', true) + await clock.tickAsync(6000) + clock.restore() + + expect(await promise).to.be.true + expect(stubs.resetDatabases.firstCall.args[2]).to.deep.equal(['xchain-decoder']) + }) + + // Asymmetric by design: the indexer re-derives from an intact decoder, + // which is an ordinary reindex and must stay available. + it('still allows an indexer-only reset', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + const ops = loadOperations(stubs) + const clock = sinon.useFakeTimers() + const promise = ops.resetModules('xchain-indexer', 'bitcoin', 'mainnet', true) + await clock.tickAsync(6000) + clock.restore() + + expect(await promise).to.be.true + expect(stubs.resetDatabases.firstCall.args[2]).to.deep.equal(['xchain-indexer']) + }) + + it('leaves `reset all` alone, which already moves both', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + const ops = loadOperations(stubs) + const clock = sinon.useFakeTimers() + const promise = ops.resetModules('all', 'bitcoin', 'mainnet', true) + await clock.tickAsync(6000) + clock.restore() + + expect(await promise).to.be.true + expect(stubs.resetDatabases.firstCall.args[2]).to.have.members(['xchain-decoder', 'xchain-indexer']) + }) + }) + it('reports a fence-clear failure without aborting the restart pass', async function () { const stubs = makeStubs() stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) From 5fb01366102a3ef53709a01557916db3f604f54f Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sun, 23 Aug 2026 16:30:11 -0700 Subject: [PATCH 07/12] Stop the migration refusal printing a command it cannot vouch for The refusal named a scoped migrate command to run inside the container being replaced. That container runs the build being upgraded away from, and a build without per-file targeting does not reject the flag: it ignores it and applies every pending manual migration, which on a live database can mean a data backfill and a dedup-then-unique nobody asked for. The refusal now reads the target container's own migrate entry point and prints the scoped command only when that build is confirmed to honour it. When the capability is absent, or the container cannot be read at all, it prints no runnable command and instead names every pending manual migration an unscoped run would apply, marking the one that is actually needed. The probe runs only on the refusal path, so a healthy deploy is unaffected and a probe that fails still refuses. --- CHANGELOG.md | 1 + src/services/MigrationPreconditionService.js | 129 +++++++++++++++++- .../unit/MigrationPreconditionService.test.js | 103 +++++++++++++- 3 files changed, 224 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5677220..f8cff09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `XCHAIN_NODE_FORCE_BOOTSTRAP=1` restores a published bootstrap over an already-populated service, for when the install that would have taken it failed. ### Fixed +- The migration precondition refusal only prints a scoped migrate command when the running build is confirmed to support one, and otherwise names every migration an unscoped run would apply. - `reset xchain-decoder` now refuses while an indexer is installed and names the `--with-indexer` joint form, because resetting one half of the pair leaves the other unable to commit blocks. - The bitcoind mirror retry now answers the address-list form Node asks for, so the retry actually dials instead of failing immediately on every supported Node version. - A bitcoind tarball already present at the download path is used and verified against the pinned hash rather than overwritten, so the manual workaround the failure message describes now works. diff --git a/src/services/MigrationPreconditionService.js b/src/services/MigrationPreconditionService.js index f0e41d3..dbf587d 100644 --- a/src/services/MigrationPreconditionService.js +++ b/src/services/MigrationPreconditionService.js @@ -101,6 +101,76 @@ function migrationDeclaresDeployPrecondition(raw) { return /^\s*--\s*xchain:migration\b[^\n]*\bdeploy-precondition\s*=\s*required\b/im.test(prologue.join('\n')) } +/** + * The `mode=` a migration header declares, or null when it declares none. + * Prologue-anchored exactly like migrationDeclaresDeployPrecondition, so a token + * in body prose or a data literal cannot answer for the file. + * + * Twin of the modules' own Database._migrationMode, duplicated for the reason + * given above: this tool reads a cloned tree whose dependencies are not + * installed. Keep them in step. + */ +function migrationMode(raw) { + const prologue = [] + for (const line of String(raw).split('\n')) { + const trimmed = line.trim() + if (trimmed === '' || trimmed.startsWith('--')) { prologue.push(line); continue } + break + } + const m = prologue.join('\n').match(/^\s*--\s*xchain:migration\b[^\n]*\bmode\s*=\s*([A-Za-z]+)/im) + return m ? m[1].toLowerCase() : null +} + +/** + * Every gated (mode=manual) migration in `dir` that the ledger has not recorded, + * sorted. This is the blast radius of an UNSCOPED migrate run against that + * database: the runner applies every pending manual file, not just the one an + * operator names. The refusal names that whole set, so the consequence is on + * screen rather than left for the operator to discover. + */ +function pendingManualMigrations(dir, applied) { + let files + try { + files = fs.readdirSync(dir).filter(f => f.endsWith('.sql')).sort() + } catch { + return [] + } + return files.filter(f => { + if (applied && applied.has(f)) return false + try { + return migrationMode(fs.readFileSync(path.join(dir, f), 'utf8')) === 'manual' + } catch { + return false + } + }) +} + +/** + * Does the build CURRENTLY RUNNING in the target container understand per-file + * migration targeting (`--file`)? + * + * This matters because the remedy an operator is about to run executes inside + * that container, on its build, not on the one being deployed. A build without + * the flag does not reject it: it ignores it and applies every pending manual + * migration, which on a live database can mean a data backfill and a + * dedup-then-unique nobody authorised. + * + * Returns true, false, or null when the container could not be read at all + * (stopped, absent, docker unreachable). Callers must treat null like false: + * an unverified capability is not a capability, and the cost of being wrong is + * asymmetric. + */ +async function runningBuildSupportsPerFileMigrations(container, deps = {}) { + try { + const cat = deps.getDockerContainerFileCat || require('./DockerService').getDockerContainerFileCat + const source = await cat(container, 'src/migrate.js') + if (!source) return null + return /['"]--file['"]/.test(String(source)) + } catch { + return null + } +} + /** * Every migration filename in `dir` whose header declares a deploy precondition, * sorted. A missing directory yields [] - a module (or a ref) with no migrations @@ -195,16 +265,41 @@ async function defaultReadAppliedMigrations({ database, coin, network }, deps = } } -function refusalMessage(module, coin, network, dbName, missing) { +function refusalMessage(module, coin, network, dbName, missing, remedy = {}) { const container = getDockerContainerImageName(module, coin, network) const files = missing.join(', ') + const plural = missing.length > 1 + + // The remedy runs on the build inside the container, which is the one being + // REPLACED. Only name the scoped command when that build was confirmed to + // honour --file; otherwise the command would quietly widen to every pending + // manual migration, so state that instead of printing it. + let instructions + if (remedy.supportsPerFile === true) { + instructions = 'apply ' + (plural ? 'them' : 'it') + + ' deliberately, with the writer quiesced, then re-run the update:\n' + + missing.map(f => ' docker exec -i ' + container + ' node src/migrate.js --file ' + f).join('\n') + } else { + const wouldApply = (remedy.pendingManual && remedy.pendingManual.length) + ? remedy.pendingManual + : missing + instructions = 'DO NOT run `node src/migrate.js` inside ' + container + '. ' + + (remedy.supportsPerFile === false + ? 'That container runs a build with no per-file targeting: it ignores --file' + : 'Whether that container\'s build honours --file could not be read, and an unverified capability is not one: it may ignore --file') + + ' and apply EVERY pending manual migration on ' + dbName + ', which is ' + + wouldApply.length + ' file(s):\n' + + wouldApply.map(f => ' ' + f + (missing.includes(f) ? ' (the one you need)' : '')).join('\n') + '\n' + + ' Apply ' + (plural ? 'the needed files' : 'the needed file') + ' with a build that supports ' + + '--file, or apply the statement by hand with the writer quiesced, then re-run the update.' + } + return 'update refused: the ' + module + ' source about to be deployed asserts migration' + - (missing.length > 1 ? 's' : '') + ' ' + files + ' at startup, but ' + dbName + - ' has not applied ' + (missing.length > 1 ? 'them' : 'it') + '. Deploying now replaces a working ' + + (plural ? 's' : '') + ' ' + files + ' at startup, but ' + dbName + + ' has not applied ' + (plural ? 'them' : 'it') + '. Deploying now replaces a working ' + 'container with one that crash-loops on boot (the 2026-08-09 mainnet halt: all three indexers went to ' + - 'Restarting(1) on exactly this). These migrations are operator-gated on purpose - apply ' + - (missing.length > 1 ? 'them' : 'it') + ' deliberately, with the writer quiesced, then re-run the update:\n' + - missing.map(f => ' docker exec -i ' + container + ' node src/migrate.js --file ' + f).join('\n') + '\n' + + 'Restarting(1) on exactly this). These migrations are operator-gated on purpose - ' + + instructions + '\n' + ' Take a fresh backup first: DEPLOY-ORDER.md says so for every migration-bearing deploy, ' + 'and the coin boxes back up only WEEKLY. ' + 'Set ' + SKIP_ENV + '=1 to override.' @@ -276,7 +371,23 @@ async function assertRequiredMigrationsApplied(module, coin, network, branch = n } const missing = required.filter(f => !result.applied.has(f)) - if (missing.length) throw new Error(refusalMessage(module, coin, network, dbName, missing)) + if (missing.length) { + // Only reached on the refusal path, so the probe costs a healthy deploy + // nothing and cannot introduce a new way for one to fail: both the probe + // and the pending-scan degrade to the cautious branch of the message. + const container = getDockerContainerImageName(module, coin, network) + const probe = deps.runningBuildSupportsPerFileMigrations || runningBuildSupportsPerFileMigrations + const listPending = deps.pendingManualMigrations || pendingManualMigrations + let supportsPerFile = null + let pendingManual = [] + try { + supportsPerFile = await probe(container, deps) + pendingManual = listPending(path.join(getModuleTmpDir(module), 'src', 'sql', 'migrations'), result.applied) + } catch { + supportsPerFile = null + } + throw new Error(refusalMessage(module, coin, network, dbName, missing, { supportsPerFile, pendingManual })) + } return { checked: true, required, missing: [], ok: true } } @@ -286,7 +397,11 @@ module.exports = { SKIP_ENV, LEDGER_TABLE, migrationDeclaresDeployPrecondition, + migrationMode, listDeployPreconditionMigrations, + pendingManualMigrations, + runningBuildSupportsPerFileMigrations, + refusalMessage, // Exported for the unit suite: the refusal path hinges on an unreachable // database returning `unreadable` rather than throwing past the guard, and // that is a property of the real driver call, not of a stub. diff --git a/test/unit/MigrationPreconditionService.test.js b/test/unit/MigrationPreconditionService.test.js index 74bd8ff..63ef60a 100644 --- a/test/unit/MigrationPreconditionService.test.js +++ b/test/unit/MigrationPreconditionService.test.js @@ -21,7 +21,9 @@ const { MIGRATION_BEARING_MODULES, SKIP_ENV, migrationDeclaresDeployPrecondition, + migrationMode, listDeployPreconditionMigrations, + pendingManualMigrations, readAppliedMigrations, assertRequiredMigrationsApplied } = require('../../src/services/MigrationPreconditionService') @@ -31,12 +33,18 @@ const GATED = '2026-07-24-pubkeys-widen-uncompressed.sql' const TAGGED = '-- xchain:migration mode=manual deploy-precondition=required\nALTER TABLE pubkeys MODIFY pubkey VARCHAR(130) NOT NULL;\n' const UNTAGGED = '-- xchain:migration mode=manual\nALTER TABLE pubkeys MODIFY pubkey VARCHAR(130) NOT NULL;\n' -function makeDeps({ required = [GATED], applied = [GATED], state = 'ledger', reason = 'connection refused', cloneErr = null } = {}) { +function makeDeps({ required = [GATED], applied = [GATED], state = 'ledger', reason = 'connection refused', cloneErr = null, + supportsPerFile = true, pendingManual = null } = {}) { return { cloneGit: cloneErr ? sinon.stub().rejects(cloneErr) : sinon.stub().resolves(), listDeployPreconditionMigrations: sinon.stub().returns(required), readAppliedMigrations: sinon.stub().resolves( - state === 'ledger' ? { state: 'ledger', applied: new Set(applied) } : { state, reason }) + state === 'ledger' ? { state: 'ledger', applied: new Set(applied) } : { state, reason }), + // The remedy the refusal prints runs on the build inside the container, + // not the one being deployed, so what that build can do is an input to + // the message and therefore stubbed here. + runningBuildSupportsPerFileMigrations: sinon.stub().resolves(supportsPerFile), + pendingManualMigrations: sinon.stub().returns(pendingManual === null ? [GATED] : pendingManual) } } @@ -74,6 +82,48 @@ describe('MigrationPreconditionService', () => { }) }) + describe('migrationMode', () => { + + it('reads the mode a header declares', () => { + expect(migrationMode(TAGGED)).to.equal('manual') + expect(migrationMode('-- xchain:migration mode=auto\nSELECT 1;\n')).to.equal('auto') + }) + + it('returns null when no mode is declared', () => { + expect(migrationMode('-- just a comment\nSELECT 1;\n')).to.equal(null) + }) + + it('ignores a mode token that appears after the prologue', () => { + // Body prose and data literals must not be able to answer for the file. + const body = '-- header\nSELECT 1;\n-- xchain:migration mode=auto\n' + expect(migrationMode(body)).to.equal(null) + }) + }) + + describe('pendingManualMigrations', () => { + + let dir + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'xc-pending-')) + fs.writeFileSync(path.join(dir, 'a-manual.sql'), TAGGED) + fs.writeFileSync(path.join(dir, 'b-auto.sql'), '-- xchain:migration mode=auto\nSELECT 1;\n') + fs.writeFileSync(path.join(dir, 'c-manual.sql'), '-- xchain:migration mode=manual\nSELECT 1;\n') + }) + afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }) }) + + it('lists only manual migrations the ledger has not recorded', () => { + expect(pendingManualMigrations(dir, new Set())).to.deep.equal(['a-manual.sql', 'c-manual.sql']) + }) + + it('excludes what the ledger already carries', () => { + expect(pendingManualMigrations(dir, new Set(['a-manual.sql']))).to.deep.equal(['c-manual.sql']) + }) + + it('yields nothing for a missing directory rather than throwing', () => { + expect(pendingManualMigrations(path.join(dir, 'nope'), new Set())).to.deep.equal([]) + }) + }) + describe('listDeployPreconditionMigrations', () => { let dir beforeEach(() => { @@ -194,9 +244,58 @@ describe('MigrationPreconditionService', () => { expect(err, 'the deploy must be refused').to.not.equal(null) expect(err.message).to.contain(GATED) expect(err.message).to.contain('XChain_BTC_Mainnet_Indexer') + // Only safe to print because this build was confirmed to honour --file. expect(err.message).to.contain('--file ' + GATED) }) + it('does NOT print a scoped command the running build would ignore', async () => { + // A build without per-file targeting does not reject --file, it ignores + // it and applies every pending manual migration, so printing the command + // hands the operator a wider action than the one it describes. + const deps = makeDeps({ + applied: [], + supportsPerFile: false, + pendingManual: [GATED, '2026-08-10-action-data-utf8mb4.sql', '2026-06-13-dispensers-expiration-bigint.sql'] + }) + let err = null + try { + await assertRequiredMigrationsApplied(XChainService.XCHAIN_INDEXER, 'bitcoin', 'mainnet', 'master', deps) + } catch (e) { err = e } + expect(err, 'the deploy must be refused').to.not.equal(null) + expect(err.message).to.not.contain('docker exec') + expect(err.message).to.contain('DO NOT run') + // The whole blast radius is named, not just the file that is needed. + expect(err.message).to.contain('3 file(s)') + expect(err.message).to.contain('2026-08-10-action-data-utf8mb4.sql') + expect(err.message).to.contain('2026-06-13-dispensers-expiration-bigint.sql') + expect(err.message).to.contain('(the one you need)') + }) + + it('treats an unreadable container as lacking the capability', async () => { + // An unverified capability is not a capability: the cost of guessing + // wrong is an unauthorised migration on a live database. + const deps = makeDeps({ applied: [], supportsPerFile: null }) + let err = null + try { + await assertRequiredMigrationsApplied(XChainService.XCHAIN_INDEXER, 'bitcoin', 'mainnet', 'master', deps) + } catch (e) { err = e } + expect(err, 'the deploy must be refused').to.not.equal(null) + expect(err.message).to.not.contain('docker exec') + expect(err.message).to.contain('could not be read') + }) + + it('still refuses when the capability probe itself throws', async () => { + const deps = makeDeps({ applied: [] }) + deps.runningBuildSupportsPerFileMigrations = sinon.stub().rejects(new Error('docker unreachable')) + let err = null + try { + await assertRequiredMigrationsApplied(XChainService.XCHAIN_INDEXER, 'bitcoin', 'mainnet', 'master', deps) + } catch (e) { err = e } + expect(err, 'the deploy must still be refused').to.not.equal(null) + expect(err.message).to.contain('update refused') + expect(err.message).to.not.contain('docker exec') + }) + it('reads the source tree about to be deployed, at the pinned ref', async () => { const deps = makeDeps() await assertRequiredMigrationsApplied(XChainService.XCHAIN_INDEXER, 'bitcoin', 'mainnet', 'release-1.2.3', deps) From 018fdcae217f92f5f77f27032f4cde199a24e4e5 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Mon, 24 Aug 2026 08:51:31 -0700 Subject: [PATCH 08/12] chore(coins): sync the fresh testnet genesis registry --- src/coins/BTC.js | 13 +++++++------ src/coins/DOGE.js | 14 +++++++------- src/coins/LTC.js | 13 +++++++------ src/coins/consensus_pin.js | 13 ++++++++++--- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/coins/BTC.js b/src/coins/BTC.js index d2b5182..c69c5bd 100644 --- a/src/coins/BTC.js +++ b/src/coins/BTC.js @@ -166,12 +166,13 @@ module.exports = { minStandardTxNonWitnessSize: 82, singleOpReturnPolicy: true, }, - // Fresh testnet genesis 2026-08-10 (operator): was 138000. Raised to - // just under the live tip (147799 at the decision) so the chain starts - // effectively empty and replays in seconds. Consensus input (folded - // into consensusSubset), so it moves the BTC testnet pin and ships in - // one wave with every other vendoring service. - firstBlock: 147500, + // Fresh testnet genesis 2026-08-24 (operator): was 147500 (the + // 2026-08-10 genesis). Raised to just under the live tip (149703 at + // the decision) so the public testnet announces with zero + // pre-announcement test actions and replays in seconds. Consensus + // input (folded into consensusSubset), so it moves the BTC testnet + // pin and ships in one wave with every other vendoring service. + firstBlock: 149700, // Block-0 hash of the chain (see mainnet above). Unpinned until the operator // reads it off the fleet's own node: `bitcoin-cli -testnet getblockhash 0`. // This is the one value that separates testnet3 from testnet4, which the diff --git a/src/coins/DOGE.js b/src/coins/DOGE.js index d7c969b..424394c 100644 --- a/src/coins/DOGE.js +++ b/src/coins/DOGE.js @@ -111,13 +111,13 @@ module.exports = { supportsSegwit: false, singleOpReturnPolicy: true, }, - // Fresh testnet genesis 2026-08-10 (operator): was 64800000, which is - // ~3.02M blocks behind the tip and ~21 hours of replay - the single - // reason this genesis is worth doing. Raised to just under the live - // tip (67819590 at the decision) so replay is minutes. Consensus input - // (folded into consensusSubset), so it moves the DOGE testnet pin and - // ships in one wave with every other vendoring service. - firstBlock: 67815000, + // Fresh testnet genesis 2026-08-24 (operator): was 67815000 (the + // 2026-08-10 genesis). Raised to just under the live tip (67847591 at + // the decision) so the public testnet announces with zero + // pre-announcement test actions and replays in minutes. Consensus + // input (folded into consensusSubset), so it moves the DOGE testnet + // pin and ships in one wave with every other vendoring service. + firstBlock: 67847500, // Block-0 hash of the chain (see mainnet above). Unpinned until the operator // reads it off the fleet's own node: `dogecoin-cli -testnet getblockhash 0`. chainGenesisHash: null, diff --git a/src/coins/LTC.js b/src/coins/LTC.js index 1bc25bf..02a23fd 100644 --- a/src/coins/LTC.js +++ b/src/coins/LTC.js @@ -109,12 +109,13 @@ module.exports = { minStandardTxNonWitnessSize: 85, singleOpReturnPolicy: true, }, - // Fresh testnet genesis 2026-08-10 (operator): was 4765000. Raised to - // just under the live tip (4855452 at the decision) so the chain - // starts effectively empty and replays in seconds. Consensus input - // (folded into consensusSubset), so it moves the LTC testnet pin and - // ships in one wave with every other vendoring service. - firstBlock: 4855000, + // Fresh testnet genesis 2026-08-24 (operator): was 4855000 (the + // 2026-08-10 genesis). Raised to just under the live tip (4862567 at + // the decision) so the public testnet announces with zero + // pre-announcement test actions and replays in seconds. Consensus + // input (folded into consensusSubset), so it moves the LTC testnet + // pin and ships in one wave with every other vendoring service. + firstBlock: 4862500, // Block-0 hash of the chain (see mainnet above). Unpinned until the operator // reads it off the fleet's own node: `litecoin-cli -testnet getblockhash 0`. chainGenesisHash: null, diff --git a/src/coins/consensus_pin.js b/src/coins/consensus_pin.js index f7ad12e..50ce2d0 100644 --- a/src/coins/consensus_pin.js +++ b/src/coins/consensus_pin.js @@ -59,10 +59,17 @@ module.exports = { // all three testnet hashes move and the one-wave rule above applies in full. // Regtest and mainnet are untouched, and their hashes were re-verified // against the canonical files as unchanged by this edit. + // + // REGENERATED 2026-08-24 (fresh testnet genesis, operator): testnet + // `firstBlock` moved to just under the live tip on all three chains + // (BTC 147500 -> 149700, LTC 4855000 -> 4862500, DOGE 67815000 -> 67847500) + // so the public testnet announces with zero pre-announcement test actions. + // Same one-wave rule as every regeneration above. Regtest and mainnet are + // untouched and were re-verified as unchanged by this edit. testnet: { - BTC: '1e45a958ff9eb6a88be8684e3801b57e7afcfc9031f7761e4f4b1dcf1c8d42a9', - LTC: '888818a874d6d8acb3363355089f0de601c355b63fc8431a44ef666f91615202', - DOGE: 'ea3ee0d1407959f3cb59e4baf66b50dfc2ada9962351e578d7c6d8586e6ff905', + BTC: 'f6589c6b88dc930db05998070ef0b73743f58623a0d23fbc30fdb158c49d1427', + LTC: '9faf066a1470be2486d8a2cd121548ca02de1397d0678a2ab8dc0e712ebfa8fd', + DOGE: '2991d7e7caf2b212de959dd5831ac1477e0b13da95ac1ed8c2b43e2704732439', }, regtest: { From f9f2b8ce831fb890c22b287654e737be87665b9e Mon Sep 17 00:00:00 2001 From: J-Dog Date: Mon, 24 Aug 2026 23:20:35 -0700 Subject: [PATCH 09/12] coins: update BTC mainnet reward pool address --- src/coins/BTC.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coins/BTC.js b/src/coins/BTC.js index c69c5bd..dc3bcb7 100644 --- a/src/coins/BTC.js +++ b/src/coins/BTC.js @@ -122,7 +122,7 @@ module.exports = { DONATE1: '1Donate1GERVKPW6GFQcnGeTa8dgL6Abyp', // Protocol Development DONATE2: '1Donate2LkbBrsanwCVRPWZCXAqQcvcqGz', // Community Development FEE_DESTINATION: '1FeesxM9LTEjBYVTkynK6jfDBgvksuh2WL', // native-fee destination (env-overridable) - REWARD: '1rewardsZAyeuLeFJKoAepYiNN5N6uSzn', // validator reward pool (COLLECT) + REWARD: '1RewardsRQTXMAytLt4bBQvPEscKsSEXt', // validator reward pool (COLLECT) EXPLORER: '1Donate3GBGSZzzrS9U9gUgURYKscAE6Yn', // display-only donation; not read by indexer }, // Genesis ledger bootstrap pin (Counterparty name carry-forward). From ef2916cd097cfd0999f07e65c469510fb115265d Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 25 Aug 2026 07:36:22 -0700 Subject: [PATCH 10/12] fix(node): the bootstrap health gate fails closed when its probe cannot read The external-DB halt-marker probe passed a clustered short-option string the native command parser did not recognise as batch mode, so every SELECT returned empty and an unreadable probe certified a halted decoder as clean. The parser now understands clustered flags and the gate treats an unreadable answer as a refusal rather than as absence of a marker. --- src/services/BootstrapHealthGate.js | 157 ++++++++++++++--- src/services/BootstrapService.js | 48 +++++- src/services/DatabaseService.js | 71 ++++++-- src/services/DockerService.js | 25 +++ test/integration/database-setup.test.js | 12 +- test/unit/BootstrapHealthGate.test.js | 217 ++++++++++++++++++++++-- test/unit/BootstrapService.test.js | 8 +- test/unit/DatabaseService.test.js | 83 ++++++++- test/unit/DockerService.test.js | 48 ++++++ 9 files changed, 615 insertions(+), 54 deletions(-) diff --git a/src/services/BootstrapHealthGate.js b/src/services/BootstrapHealthGate.js index 67ff5f7..48e196a 100644 --- a/src/services/BootstrapHealthGate.js +++ b/src/services/BootstrapHealthGate.js @@ -32,7 +32,9 @@ * - reporting an unhealthy/halted status on its own health surface * - carrying a durable halt marker in its database: a decoder REORG_HALT row * (events.code = 'REORG_HALT') or an uncleared xchain-sync divergence halt - * (sync_halt with cleared_at IS NULL) + * (sync_halt with cleared_at IS NULL). For an indexer source that means the + * PAIRED DECODER's database, which is the only place either marker is written; + * an indexer's own events table only ever carries code='REORG'. * - materially behind its node's tip * * FAIL CLOSED throughout. A probe that cannot be run, cannot be parsed, or @@ -152,8 +154,8 @@ async function inspectContainer(containerId, runner) { // Interpret a decoder/indexer/utxo-tracker health payload. Pure, so the policy // is unit-testable without docker. Field names differ per service, which is why // every known spelling is checked rather than one canonical key: -// decoder health: status, lag_blocks/blockLag, reorg_halted -// indexer health: status, lag, decoderReorgHalted, degraded +// decoder health: status, lag_blocks/blockLag, reorg_halted, reorg_halt_checked_at +// indexer health: status, lag, decoderReorgHalted, stallClass // tracker health: lag, synced, halted // any /status: status ('ok'|'healthy'|'halted'|'degraded'|'unhealthy') function evaluateStatusPayload(payload, { maxLag = DEFAULT_MAX_LAG_BLOCKS } = {}) { @@ -170,8 +172,33 @@ function evaluateStatusPayload(payload, { maxLag = DEFAULT_MAX_LAG_BLOCKS } = {} if (payload.reorg_halted === true) reasons.push('the decoder carries a durable REORG_HALT marker' + (payload.reorg_halt_reason ? `: ${payload.reorg_halt_reason}` : '')) + // "not halted" is only an answer if something actually looked. The decoder's + // marker probe is fail-soft on purpose (a DB blip keeps the last known state), + // and that state starts at false with checked_at null, so a decoder that has + // NEVER completed a probe publishes exactly what a clean one publishes. Keyed on + // OWNING reorg_halted: the boolean and its timestamp shipped in the same decoder + // commit, so a payload carrying one always carries the other, and an image + // publishing neither is unaffected. The indexer's decoderReorgHalted has no + // companion timestamp yet; extend this to it when the indexer publishes one. + if (Object.prototype.hasOwnProperty.call(payload, 'reorg_halted') + && (payload.reorg_halt_checked_at === null || payload.reorg_halt_checked_at === undefined)) + reasons.push('the decoder has never completed a REORG_HALT marker probe (reorg_halt_checked_at is ' + + 'null), so its "not halted" report is an untested default rather than a reading') if (payload.decoderReorgHalted === true) reasons.push('the upstream decoder carries a durable REORG_HALT marker, so this database is frozen behind it') + // The indexer's own single-field verdict on its block counter: + // 'none' | 'future_block_wait' | 'barrier_defer' | 'wedged'. Only 'wedged' is a + // refusal, and it needs its own leg: the indexer reports status "healthy" + // whenever its process is up and its DB circuit is closed, so a freshly-wedged + // indexer whose lag is still inside the ceiling passes every other check here. + // NOT keyed on `degraded`, which stays true throughout the healthy + // future-stamped-block wait (the permanent testnet4 steady state) and would + // refuse forever; stallClassOf resolves that wait to 'future_block_wait' before + // it can ever reach 'wedged'. Strict equality, so an older image publishing no + // stallClass keeps today's behavior exactly. + if (payload.stallClass === 'wedged') + reasons.push('the service reports its block counter WEDGED (stallClass "wedged": no commit for longer ' + + 'than its stall grace window)' + (payload.stallReason ? `: ${payload.stallReason}` : '')) if (payload.block_fetch_desync) reasons.push(`the service reports a block-fetch desync (${formatBlockFetchDesync(payload.block_fetch_desync)})`) if (payload.node_height_stale === true) @@ -259,7 +286,8 @@ async function probeServiceStatus(containerId, port, runner) { // The authoritative check for a decoder that is up and looks healthy but is // quietly carrying a stale halt marker, and the one that does not depend on // the running image being new enough to report the marker on its health -// surface: read the marker rows straight out of the database being dumped. +// surface: read the marker rows straight out of the database being dumped - and, +// for an indexer source, out of the paired decoder database that owns them. async function readHaltMarkers(coin, network, module, deps) { const { runner, @@ -269,18 +297,14 @@ async function readHaltMarkers(coin, network, module, deps) { executeNativeMariaDbCommand } = deps - const dbName = getModuleDatabaseName(module, coin, network) - // The name is derived from coin/network internally, never operator input, but - // it is interpolated into SQL below; assert the shape rather than trust it. - if (!/^[A-Za-z0-9_]+$/.test(String(dbName))) - throw new Error(`refusing to probe an unexpected database name: ${dbName}`) - - // One round trip: which marker tables exist, and how many live rows each has. - // COALESCE keeps a missing table from turning into a NULL that reads as 0. - const query = - `SELECT ` + - `(SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='${dbName}' AND TABLE_NAME='events'), ` + - `(SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='${dbName}' AND TABLE_NAME='sync_halt');` + // The names are derived from coin/network internally, never operator input, but + // they are interpolated into SQL below; assert the shape rather than trust it. + const assertDbName = (name) => { + if (!/^[A-Za-z0-9_]+$/.test(String(name))) + throw new Error(`refusing to probe an unexpected database name: ${name}`) + return String(name) + } + const dbName = assertDbName(getModuleDatabaseName(module, coin, network)) const run = async (sql) => { if (EXTERNAL_DB) { @@ -298,16 +322,82 @@ async function readHaltMarkers(coin, network, module, deps) { return String(stdout || '') } - const [hasEvents, hasSyncHalt] = String(await run(query)).trim().split(/\s+/).map(n => parseInt(n, 10)) + // Read one count, refusing on anything that is not a number. Output the probe + // could not produce (an empty string from a mis-parsed client option, a driver + // that returned nothing, a permission error rendered on stdout) parsed to NaN + // here, and NaN loses every `> 0` comparison below, so "we could not tell" + // arrived at the caller as "no halt markers" - the one collapse the file + // header forbids. + const readCount = async (sql, what) => { + const raw = String(await run(sql)).trim() + const value = parseInt(raw, 10) + if (!Number.isFinite(value)) + throw new Error(`the ${what} probe returned unreadable output: ${JSON.stringify(raw)}`) + return value + } - const markers = { reorgHalt: 0, syncHalt: 0 } - if (hasEvents > 0) { - const out = await run(`SELECT COUNT(*) FROM \`${dbName}\`.events WHERE code='REORG_HALT';`) - markers.reorgHalt = parseInt(String(out).trim(), 10) || 0 + // Probe ONE database for both durable markers. Every failure shape throws, and + // the caller turns a throw into a refusal reason: that is the whole contract. + const probeDatabase = async (name) => { + // One round trip: which marker tables exist, and how many live rows each has. + // Counting information_schema rows (rather than querying the table directly) + // keeps an absent table answerable as a 0 instead of an error; deciding what + // that 0 means is this function's job below, and it is not always "clean". + const query = + `SELECT ` + + `(SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='${name}' AND TABLE_NAME='events'), ` + + `(SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='${name}' AND TABLE_NAME='sync_halt');` + + const rawTables = String(await run(query)).trim() + const tableCounts = rawTables.split(/\s+/).map(n => parseInt(n, 10)) + if (tableCounts.length !== 2 || !tableCounts.every(Number.isFinite)) + throw new Error(`the marker-table probe for ${name} returned unreadable output: ${JSON.stringify(rawTables)}`) + const [hasEvents, hasSyncHalt] = tableCounts + + // `events` is not optional on a decoder/indexer database: both provision it + // unconditionally at startup (each repo's verifyTables creates every + // src/sql/*.sql table), and it is the only durable home of the REORG_HALT + // marker. Its absence therefore means the probe did not read the database + // it was aimed at - a name drift, a wrong host - which is a refusal, not a + // clean bill of health. + if (hasEvents === 0) + throw new Error(`${name} reports no events table, so the REORG_HALT marker could not be read`) + + const found = { reorgHalt: 0, syncHalt: 0 } + found.reorgHalt = await readCount( + `SELECT COUNT(*) FROM \`${name}\`.events WHERE code='REORG_HALT';`, 'REORG_HALT marker') + // sync_halt IS optional: xchain-sync provisions it only where it runs, so an + // absent table here is a genuine "no such marker", not an unread database. + if (hasSyncHalt > 0) + found.syncHalt = await readCount( + `SELECT COUNT(*) FROM \`${name}\`.sync_halt WHERE cleared_at IS NULL;`, 'sync_halt marker') + return found } - if (hasSyncHalt > 0) { - const out = await run(`SELECT COUNT(*) FROM \`${dbName}\`.sync_halt WHERE cleared_at IS NULL;`) - markers.syncHalt = parseInt(String(out).trim(), 10) || 0 + + const markers = await probeDatabase(dbName) + + // An xchain-indexer database structurally CANNOT carry the REORG_HALT marker, so + // the probe above is a guaranteed zero for an indexer source and this backstop had + // no reach there at all: the indexer only ever writes code='REORG' into its own + // events table and reads the halt marker out of the DECODER's connection, while + // the marker row is written solely into the decoder database. Nothing else in the + // gate covers the gap either - the indexer's decoderReorgHalted mirror defaults + // false and keeps its last value on any probe fault, and its published lag is + // measured against the halted decoder's own frozen height, so it reads 0. Probe + // the paired decoder database as well, fail-closed: probeDatabase throws on an + // absent database, an absent events table and an unreadable count alike, and every + // one of those means we could not tell. + if (module === XChainService.XCHAIN_INDEXER) { + const decoderDbName = assertDbName(getModuleDatabaseName(XChainService.XCHAIN_DECODER, coin, network)) + let upstream + try { + upstream = await probeDatabase(decoderDbName) + } catch (err) { + // Name the database that actually failed: the caller's wrapper names the + // GATED module, which would otherwise blame the indexer for the decoder. + throw new Error(`the paired decoder database ${decoderDbName} could not be probed: ${err && err.message}`) + } + markers.upstream = { dbName: decoderDbName, ...upstream } } return markers } @@ -387,9 +477,10 @@ async function assertBootstrapSourceHealthy(coin, network, module, deps = {}) { } } - // 3. Durable halt markers in the database that is about to be dumped. This is - // the check that catches a decoder which is up, healthy-looking, and quietly - // carrying a REORG_HALT row, including on an older image whose health + // 3. Durable halt markers in the database that is about to be dumped, plus (for + // an indexer) the paired decoder database that actually owns the REORG_HALT row. + // This is the check that catches a decoder which is up, healthy-looking, and + // quietly carrying a REORG_HALT row, including on an older image whose health // surface does not report it. if (MARIADB_MODULES.has(module)) { try { @@ -401,6 +492,18 @@ async function assertBootstrapSourceHealthy(coin, network, module, deps = {}) { if (markers.syncHalt > 0) reasons.push('the database carries an uncleared xchain-sync divergence halt ' + '(sync_halt with cleared_at IS NULL): its contents are known to diverge from the source of truth.') + // An indexer's own database cannot hold these rows; the paired decoder's can, + // and an indexer frozen behind a halted decoder is exactly as unfit to publish. + if (markers.upstream && markers.upstream.reorgHalt > 0) + reasons.push(`the paired decoder database ${markers.upstream.dbName} carries a durable REORG_HALT ` + + "marker (events.code='REORG_HALT'), so this indexer is frozen behind a decoder that aborted " + + 'mid-rollback. Its own health surface reports lag 0 only because that lag is measured against ' + + 'the frozen decoder height. Recovery is a full resync of the decoder and this indexer from a ' + + 'known-good snapshot.') + if (markers.upstream && markers.upstream.syncHalt > 0) + reasons.push(`the paired decoder database ${markers.upstream.dbName} carries an uncleared ` + + 'xchain-sync divergence halt (sync_halt with cleared_at IS NULL), so the rows this indexer ' + + 'derived from it are known to diverge from the source of truth.') } catch (err) { reasons.push(`could not read the halt markers from the ${module} database: ${err && err.message}`) } diff --git a/src/services/BootstrapService.js b/src/services/BootstrapService.js index bf939bc..57f9e77 100644 --- a/src/services/BootstrapService.js +++ b/src/services/BootstrapService.js @@ -696,11 +696,29 @@ async function restoreBootstrapUtxoTracker(coin, network, fileName) { fs.rmSync(workDir, { recursive: true }) console.log('Bootstrap restore complete') - } finally { - console.log(`Starting ${XChainService.XCHAIN_UTXO_TRACKER} container...`) - await startContainer(containerId) + } catch (err) { + // Post-wipe regime (uuid:7edc76f3). Every statement in the try above runs + // at or after `find /data -mindepth 1 -delete`, so a failure here leaves + // the LevelDB store partially wiped. Restarting the container over it (the + // old unconditional `finally`) boots a fresh XChainUtxoTracker with + // halted=false, so get_sync_status / GET /status report a normal non-503 + // status and BootstrapHealthGate has nothing to refuse on: an emptied + // store reads as caught up. Mirrors the tracker's own contract in + // xchain-utxo-tracker/src/bootstrap-recovery.js `handleRestoreFailure`, + // where a post-wipe abort fails loud instead of resuming. + err.postWipe = true + console.log( + `[fatal] ${XChainService.XCHAIN_UTXO_TRACKER} bootstrap restore failed AFTER the LevelDB\n` + + `volume was wiped; the store is incomplete and the container has been left STOPPED so it\n` + + `cannot report a wiped store as caught up. Re-run the restore with\n` + + `XCHAIN_NODE_FORCE_BOOTSTRAP=1, or clear the volume and resync from scratch.` + ) + throw err } + console.log(`Starting ${XChainService.XCHAIN_UTXO_TRACKER} container...`) + await startContainer(containerId) + return true } @@ -965,14 +983,27 @@ function resetBootstrapOutcomes() { function reportBootstrapOutcomes() { if (bootstrapOutcomes.length === 0) return const failed = bootstrapOutcomes.filter((o) => o.status === 'failed') + // Wiped-then-failed is NOT part of `failed`: those services are down, not + // syncing from block 0, so the paragraph below would misdescribe them. + const wipedDown = bootstrapOutcomes.filter((o) => o.status === 'wiped-left-down') console.log('\nBootstrap restore summary:') for (const o of bootstrapOutcomes) { const line = o.status === 'restored' ? 'restored' : o.status === 'none-published' ? 'none published, syncing from scratch' : o.status === 'disabled' ? 'disabled by XCHAIN_NODE_NO_BOOTSTRAP' + : o.status === 'wiped-left-down' ? `NOT restored, DATA WIPED, container left stopped: ${o.detail}` : `NOT restored: ${o.detail}` console.log(` ${o.module}: ${line}`) } + if (wipedDown.length > 0) { + console.log( + '\nThose services had their data directory wiped by a restore that then failed, so\n' + + 'their containers were deliberately left STOPPED rather than restarted over an\n' + + 'incomplete store that would report itself caught up. Re-run install with\n' + + 'XCHAIN_NODE_FORCE_BOOTSTRAP=1 to take the restore again, or clear the volume and\n' + + 'start the service to resync from block 0.\n' + ) + } if (failed.length > 0) { console.log( '\nThose services are now syncing from block 0, which takes hours to days\n' + @@ -1017,6 +1048,17 @@ async function ensureBootstrapUtxoTracker(coin, network) { return true } catch (err) { const reason = redactSecrets(err.message) + // A post-wipe abort did NOT leave a scratch-syncing tracker: the store was + // emptied and the container was left stopped (uuid:7edc76f3), so the old + // "will sync from scratch" wording described a state that is not on disk. + if (err.postWipe) { + console.log( + `WARNING: bootstrap auto-restore failed (${reason}) AFTER the LevelDB volume was wiped: ` + + `the tracker store is incomplete and its container was left stopped, not syncing.` + ) + recordBootstrapOutcome(XChainService.XCHAIN_UTXO_TRACKER, 'wiped-left-down', reason) + return false + } console.log(`WARNING: bootstrap auto-restore failed (${reason}): the tracker will sync from scratch`) recordBootstrapOutcome(XChainService.XCHAIN_UTXO_TRACKER, 'failed', reason) return false diff --git a/src/services/DatabaseService.js b/src/services/DatabaseService.js index be60241..a6ba1de 100644 --- a/src/services/DatabaseService.js +++ b/src/services/DatabaseService.js @@ -30,7 +30,7 @@ const { sleep, redactSecrets } = require('../utils/helpers') const { assertSafeDbIdentifier, escapeSqlStringLiteral } = require('../utils/sqlSafety') const { dockerMariadbArgs, mariadbEnv } = require('../utils/dockerMariadb') const { getDefaultConfig, getDockerContainerImageName, getDockerNetwork, getModuleDatabaseName, validatePort } = require('./ConfigService') -const { getStatusFromContainer, getDockerNetworkInspect, addContainerToNetwork, forceRemoveContainerByName } = require('./DockerService') +const { getStatusFromContainer, getDockerNetworkInspect, addContainerToNetwork, forceRemoveContainerByName, probeContainerPresenceByName } = require('./DockerService') const { assertNoDbCredentialDrift, isDbCredentialDriftError } = require('./DbCredentialDrift') const { statusChanged } = require('./StatusService') const { @@ -248,14 +248,40 @@ async function _pingMariaDb({ host, port, root_user, root_password }) { } } +// Read a mariadb client option string the way the client itself reads argv: +// short flags cluster, so "-BN" means "-B -N". The docker path hands this same +// string to a real client that clusters (executeDockerMariaDbCommand splits it +// straight into argv), and the native helper below claims to mirror it, so a +// clustered spelling must not quietly mean "not batch mode": that returns '' for +// a SELECT, and every caller parseInts '' into NaN, which compares false against +// every threshold and reads as "nothing found" (an absent halt marker, an empty +// database). Only the flags this helper implements are recognized; an unknown +// one is ignored exactly as it is today. +function parseMariaDbClientOptions(commandOptions) { + let batchMode = false + let noHeaders = false + for (const token of String(commandOptions || '').trim().split(/\s+/)) { + if (!token || token[0] !== '-') continue + if (token.startsWith('--')) { + if (token === '--batch') batchMode = true + if (token === '--skip-column-names') noHeaders = true + continue + } + // A short-flag token is a cluster of single letters ("-BN" == "-B -N"). + if (token.includes('B')) batchMode = true + if (token.includes('N')) noHeaders = true + } + return { batchMode, noHeaders } +} + // Execute a single SQL statement against the external (host-native) MariaDB // as the root user. Mirrors the interface of executeDockerMariaDbCommand // so callers can switch on EXTERNAL_DB without changing their structure. -// commandOptions is honored for "-B -N" (batch, no-headers) which existing -// callers use to parse single-value queries. +// commandOptions is honored for batch/no-headers ("-B -N", the clustered +// "-BN", and the long forms) which existing callers use to parse single-value +// queries. async function executeNativeMariaDbCommand(externalCfg, command, commandOptions = "") { - const batchMode = /(^|\s)-B(\s|$)/.test(commandOptions) || /(^|\s)--batch(\s|$)/.test(commandOptions) - const noHeaders = /(^|\s)-N(\s|$)/.test(commandOptions) || /(^|\s)--skip-column-names(\s|$)/.test(commandOptions) + const { batchMode, noHeaders } = parseMariaDbClientOptions(commandOptions) const conn = await mariadb.createConnection({ host: externalCfg.host, @@ -1019,11 +1045,36 @@ async function buildDatabaseModule(coin, network) { // allocated". Lazy require avoids a load-time cycle (ModuleService // requires this module at top). // Name-keyed cleanup immediately before `docker run --name`, making - // (re)creation idempotent against a leftover carcass this registry-gated - // branch (`if (!existingId)`) cannot see: a container that exists but - // whose registry insert failed on an earlier run. Runs before the - // port-conflict check so this container's own carcass never self-flags - // as a conflict (uuid:9533ee7a). + // (re)creation idempotent against a leftover `Created`-state carcass. + // Runs before the port-conflict check so this container's own carcass + // never self-flags as a conflict (uuid:9533ee7a). + // + // Gated on a POSITIVE absence, unlike the module and crypto-node install + // paths that share this shape (ModuleService.buildAndUp, + // NodeService.buildCryptoNode), because their target container is + // disposable and this one is the stack's only persistent data store: it + // holds xchain_node.modules plus every per-coin decoder/indexer schema, + // on an anonymous volume a recreate orphans. + // + // The branch condition is NOT evidence of absence. It reads + // checkIfDatabaseModuleExists, which swallows every error and also + // answers null on any inspect output that is not clean 64-hex, so a + // daemon hiccup, a slow inspect, or a container whose State probe failed + // all arrive here looking exactly like a fresh install, and the + // force-remove below would then be `docker rm -f` against a LIVE MariaDB. + // So re-probe and demand docker's own "no such container" before + // deleting anything, the same fail-safe StatusService.isContainerGoneError + // applies before dropping a registry row (uuid:8a3e5182). + const dbPresence = await probeContainerPresenceByName(containerPrefix) + if (dbPresence !== 'gone') { + throw new Error( + "Refusing to (re)create the MariaDB container: docker reports '" + dbPresence + "' for container '" + + containerPrefix + "', which is not a confirmed absence. This install path force-removes that container " + + "and would orphan the database volume holding every module's data. Run `docker inspect " + containerPrefix + + "` and `docker ps -a` to see what is actually there, then re-run once docker answers cleanly. " + + "If the container is genuinely dead and you want it rebuilt from empty, remove it yourself first." + ) + } try { await forceRemoveContainerByName(containerPrefix) } catch { /* tolerant by design; see DockerService.forceRemoveContainerByName */ } diff --git a/src/services/DockerService.js b/src/services/DockerService.js index 43fd4b6..c41a0b3 100644 --- a/src/services/DockerService.js +++ b/src/services/DockerService.js @@ -510,6 +510,30 @@ async function forceRemoveContainerByName(name) { }) } +// Tri-state presence probe by NAME: 'exists', 'gone', or 'unknown'. +// +// Every other lookup in this codebase collapses "genuinely absent", "daemon +// hiccup", "inspect timed out" and "payload I could not parse" into one falsy +// answer. That is right for a READ and wrong for a DELETE: a caller about to +// force-remove a STATEFUL container needs positive evidence there is nothing +// there to destroy, and a falsy lookup is not that evidence. Classification +// follows the house rule already stated at StatusService.isContainerGoneError: +// docker SAYING "no such object/container" is the only thing that means gone, +// and everything else is 'unknown' and must be treated as possibly-live. +function probeContainerPresenceByName(name) { + return new Promise((resolve) => { + execFile('docker', ['inspect', '--type', 'container', '--format', '{{.Id}}', name], (error, stdout, stderr) => { + if (error) { + const text = String((stderr || '') + ' ' + (error.stderr || '') + ' ' + (error.message || '')).toLowerCase() + resolve(/no such (object|container)/.test(text) ? 'gone' : 'unknown') + return + } + // A clean exit that did not yield an id is not an absence either. + resolve(/^[a-f0-9]{64}$/.test(String(stdout).trim()) ? 'exists' : 'unknown') + }) + }) +} + // Bind mounts of a container, by name, as [{ source, destination }]. Returns [] // when the container does not exist (or inspect output is unparseable): the // caller (the mount-drift guard in NodeService.buildCryptoNode) treats "no @@ -583,6 +607,7 @@ module.exports = { killContainer, getContainerBindMounts, forceRemoveContainerByName, + probeContainerPresenceByName, execContainer, shellContainer, logContainer, diff --git a/test/integration/database-setup.test.js b/test/integration/database-setup.test.js index c87831f..2193f7b 100644 --- a/test/integration/database-setup.test.js +++ b/test/integration/database-setup.test.js @@ -151,7 +151,12 @@ describe('Integration: Database Service Chain', function () { getDockerNetworkInspect: async () => ({ IPAM: { Config: [{ Gateway: '172.18.0.1' }] } }), - addContainerToNetwork: async () => true + addContainerToNetwork: async () => true, + // Fresh install: docker positively reports the DB container gone, + // which is what the install branch now requires before it may + // force-remove anything (uuid:8a3e5182). + probeContainerPresenceByName: async () => 'gone', + forceRemoveContainerByName: async () => false }, '../utils/helpers': { sleep: async () => {}, @@ -257,7 +262,10 @@ describe('Integration: Database Service Chain', function () { addContainerToNetwork: async (id, network) => { networkConnections.push({ id, network }) return true - } + }, + // Fresh install: see the identical note in makeDatabaseService. + probeContainerPresenceByName: async () => 'gone', + forceRemoveContainerByName: async () => false }, // See makeDatabaseService's identical comment above: DatabaseService // logs through redactSecrets(), so a bare `{ sleep }` mock throws diff --git a/test/unit/BootstrapHealthGate.test.js b/test/unit/BootstrapHealthGate.test.js index 025e28b..31c6f2e 100644 --- a/test/unit/BootstrapHealthGate.test.js +++ b/test/unit/BootstrapHealthGate.test.js @@ -23,13 +23,26 @@ const COIN = 'litecoin' const NETWORK = 'mainnet' const SVC_CONTAINER = 'c'.repeat(64) const DB_CONTAINER = 'd'.repeat(64) +const DECODER_DB = 'xchain_ltc_mainnet_decoder' +const INDEXER_DB = 'xchain_ltc_mainnet_indexer' // A container that is up, stable, and passing its healthcheck. function healthyInspect({ started = '2026-01-01T00:00:00.000Z' } = {}) { return `running|false|0|${started}|healthy\n` } -function loadGate({ external = false } = {}) { +// The external-DB helper answers per query by default (both marker tables +// present, no marker rows), the same clean-database shape the docker runner +// fakes. A test that wants the "helper answered with nothing" shape - a +// mis-parsed client option, a driver that returned no rows - passes +// nativeResolves: '' and gets that for every call. +function nativeHelperStub(nativeResolves) { + if (nativeResolves !== null) return sinon.stub().resolves(nativeResolves) + return sinon.stub().callsFake(async (cfg, sql) => + (/information_schema\.TABLES/.test(sql) ? '1\t1' : '0')) +} + +function loadGate({ external = false, nativeResolves = null } = {}) { return proxyquire('../../src/services/BootstrapHealthGate', { '../config/constants': { XChainService, @@ -42,13 +55,16 @@ function loadGate({ external = false } = {}) { INDEXER_API_PORT: 3004, UTXO_TRACKER_API_PORT: 3001 }), - getModuleDatabaseName: sinon.stub().returns('xchain_ltc_mainnet_decoder') + // Module-aware, because gating an indexer must probe TWO databases and a + // test cannot express "decoder dirty, indexer clean" while both share a name. + getModuleDatabaseName: sinon.stub().callsFake(m => + (m === XChainService.XCHAIN_INDEXER ? INDEXER_DB : DECODER_DB)) }, './DatabaseService': { getDatabaseContainerId: sinon.stub().resolves(DB_CONTAINER), askMariadbRootPassword: sinon.stub().resolves('rootpass'), getExternalDbConfig: sinon.stub().resolves({ host: 'h', port: 3306, root_user: 'root', root_password: 'p' }), - executeNativeMariaDbCommand: sinon.stub().resolves('0\t0') + executeNativeMariaDbCommand: nativeHelperStub(nativeResolves) }, '../utils/dockerMariadb': { dockerMariadbArgs: (id, args) => ['exec', '-e', 'MYSQL_PWD', id, ...args], @@ -61,13 +77,22 @@ function loadGate({ external = false } = {}) { // so each test states only what it changes. function makeRunner({ inspect = healthyInspect(), - status = { status: 'healthy', lag_blocks: 0, reorg_halted: false }, + // A real decoder publishes reorg_halt_checked_at beside reorg_halted (both shipped + // in the same commit), and only a decoder that never completed a marker probe + // leaves it null. The fixture said "not halted" without ever having looked. + status = { status: 'healthy', lag_blocks: 0, reorg_halted: false, reorg_halt_checked_at: 1756000000000 }, tables = '1\t1', reorgHaltRows = '0', syncHaltRows = '0', inspectThrows = null, statusThrows = null, - sqlThrows = null + sqlThrows = null, + // Answers for the PAIRED DECODER database an indexer gate probes second. Left + // null, that database answers exactly as the gated one does; set it to express + // the scenario the gate exists for and a single-database fake cannot reach: + // decoder dirty, indexer clean. Accepts { tables, reorgHaltRows, syncHaltRows, + // throws }. + decoder = null } = {}) { return sinon.stub().callsFake(async (cmd, args) => { if (args[0] === 'inspect') { @@ -83,9 +108,12 @@ function makeRunner({ // invocation carries its own `-e MYSQL_PWD` ahead of the client's `-e `. const sql = args[args.lastIndexOf('-e') + 1] || '' if (sqlThrows) throw sqlThrows - if (/information_schema\.TABLES/.test(sql)) return { stdout: tables } - if (/REORG_HALT/.test(sql)) return { stdout: reorgHaltRows } - if (/sync_halt/.test(sql)) return { stdout: syncHaltRows } + const up = (decoder && sql.includes(DECODER_DB)) ? decoder : null + if (up && up.throws) throw up.throws + const pick = (key, fallback) => (up && up[key] !== undefined) ? up[key] : fallback + if (/information_schema\.TABLES/.test(sql)) return { stdout: pick('tables', tables) } + if (/REORG_HALT/.test(sql)) return { stdout: pick('reorgHaltRows', reorgHaltRows) } + if (/sync_halt/.test(sql)) return { stdout: pick('syncHaltRows', syncHaltRows) } return { stdout: '' } }) } @@ -147,7 +175,7 @@ describe('BootstrapHealthGate', function () { expect(res.skipped).to.equal(false) }) - it('skips the marker queries for tables the schema does not have', async function () { + it('skips the sync_halt query on a schema that has no sync_halt table', async function () { const gate = loadGate() const runner = makeRunner({ tables: '1\t0' }) await callGate(gate, { runner }) @@ -155,12 +183,123 @@ describe('BootstrapHealthGate', function () { expect(sqls.some(s => /FROM `[^`]+`\.sync_halt/.test(s))).to.equal(false) }) + // The header contract says a probe that cannot be PARSED is a refusal too, + // not only one that throws. Each of these parses to NaN, loses every + // `> 0` comparison, and reads as "healthy, no halt markers" without it. + it('REFUSES when the marker-table probe returns nothing (unreadable, not clean)', async function () { + const gate = loadGate() + const err = await refusal(callGate(gate, { runner: makeRunner({ tables: '' }) })) + expect(err.message).to.match(/marker-table probe[\s\S]*returned unreadable output/) + }) + + it('REFUSES when the marker-table probe returns non-numeric output', async function () { + const gate = loadGate() + const err = await refusal(callGate(gate, { runner: makeRunner({ tables: 'x\ty' }) })) + expect(err.message).to.match(/marker-table probe[\s\S]*returned unreadable output/) + }) + + // A decoder/indexer always provisions `events`; a probe that cannot see it + // is not looking at the database that is about to be dumped. + it('REFUSES a MariaDB source whose schema reports no events table', async function () { + const gate = loadGate() + const err = await refusal(callGate(gate, { runner: makeRunner({ tables: '0\t1' }) })) + expect(err.message).to.match(/reports no events table/) + }) + + it('REFUSES when the REORG_HALT count itself is unreadable', async function () { + const gate = loadGate() + const err = await refusal(callGate(gate, { runner: makeRunner({ reorgHaltRows: '' }) })) + expect(err.message).to.match(/REORG_HALT marker probe returned unreadable output/) + }) + + it('REFUSES when the sync_halt count itself is unreadable', async function () { + const gate = loadGate() + const err = await refusal(callGate(gate, { runner: makeRunner({ syncHaltRows: 'nope' }) })) + expect(err.message).to.match(/sync_halt marker probe returned unreadable output/) + }) + + // External-DB mode reads the markers over the native driver rather than + // docker exec. An empty answer there (the shape a mis-parsed client option + // string produced) must refuse, not certify the archive. + it('REFUSES in external-DB mode when the native probe answers with nothing', async function () { + const gate = loadGate({ external: true, nativeResolves: '' }) + const err = await refusal(callGate(gate, { runner: makeRunner() })) + expect(err.message).to.match(/could not read the halt markers/) + expect(err.message).to.match(/returned unreadable output/) + }) + it('REFUSES when the marker query itself fails (fail closed, never assume clean)', async function () { const gate = loadGate() const err = await refusal(callGate(gate, { runner: makeRunner({ sqlThrows: new Error('access denied') }) })) expect(err.message).to.match(/could not read the halt markers/) }) + // An indexer's own events table only ever carries code='REORG'; the REORG_HALT + // row lives solely in the paired decoder database. Probing only the gated + // module's own database therefore asked a question that could not come back + // yes, and the gate's one image-independent backstop had no reach at all here. + it('REFUSES an indexer whose PAIRED DECODER database carries a REORG_HALT row', async function () { + const gate = loadGate() + const runner = makeRunner({ + status: { status: 'healthy', lag: 0, decoderReorgHalted: false }, + decoder: { reorgHaltRows: '1' } + }) + const err = await refusal(callGate(gate, { module: XChainService.XCHAIN_INDEXER, runner })) + expect(err.message).to.match(new RegExp(`paired decoder database ${DECODER_DB} carries a durable REORG_HALT`)) + expect(err.message).to.match(/frozen behind a decoder that aborted mid-rollback/) + }) + + it('REFUSES an indexer whose paired decoder database carries an uncleared sync halt', async function () { + const gate = loadGate() + const runner = makeRunner({ + status: { status: 'healthy', lag: 0, decoderReorgHalted: false }, + decoder: { syncHaltRows: '2' } + }) + const err = await refusal(callGate(gate, { module: XChainService.XCHAIN_INDEXER, runner })) + expect(err.message).to.match(/paired decoder database[\s\S]*uncleared[\s\S]*sync_halt/) + }) + + // Fail closed on the UPSTREAM probe too: "the decoder database is not there" + // must not arrive as "the decoder has no halt marker". + it('REFUSES an indexer when the paired decoder database is absent', async function () { + const gate = loadGate() + const runner = makeRunner({ + status: { status: 'healthy', lag: 0, decoderReorgHalted: false }, + decoder: { tables: '0\t0' } + }) + const err = await refusal(callGate(gate, { module: XChainService.XCHAIN_INDEXER, runner })) + expect(err.message).to.match(/paired decoder database[\s\S]*could not be probed/) + expect(err.message).to.match(/reports no events table/) + }) + + it('REFUSES an indexer when the paired decoder probe throws', async function () { + const gate = loadGate() + const runner = makeRunner({ + status: { status: 'healthy', lag: 0, decoderReorgHalted: false }, + decoder: { throws: new Error('access denied') } + }) + const err = await refusal(callGate(gate, { module: XChainService.XCHAIN_INDEXER, runner })) + expect(err.message).to.match(/paired decoder database[\s\S]*could not be probed[\s\S]*access denied/) + }) + + it('passes an indexer when BOTH its own and the decoder database are clean', async function () { + const gate = loadGate() + const runner = makeRunner({ status: { status: 'healthy', lag: 0, decoderReorgHalted: false } }) + const res = await callGate(gate, { module: XChainService.XCHAIN_INDEXER, runner }) + expect(res.skipped).to.equal(false) + const sqls = runner.getCalls().map(c => (c.args[1] || []).join(' ')) + expect(sqls.some(s => s.includes(INDEXER_DB))).to.equal(true) + expect(sqls.some(s => s.includes(DECODER_DB))).to.equal(true) + }) + + it('gating a decoder queries no second database', async function () { + const gate = loadGate() + const runner = makeRunner() + await callGate(gate, { runner }) + const sqls = runner.getCalls().map(c => (c.args[1] || []).join(' ')) + expect(sqls.some(s => s.includes(INDEXER_DB))).to.equal(false) + }) + it('does not run marker queries for the utxo-tracker (LevelDB, no such table)', async function () { const gate = loadGate() const runner = makeRunner({ status: { status: 'ok', lag: 0 } }) @@ -290,7 +429,11 @@ describe('BootstrapHealthGate', function () { if (isRpc) return { stdout: JSON.stringify({ jsonrpc: '2.0', id: 1, error: { message: 'Method not found' } }) } return { stdout: JSON.stringify({ status: 'healthy', db: true, running: true, lag_blocks: 2 }) } } - const sql = args[args.indexOf('-e') + 1] || '' + // lastIndexOf, as in makeRunner: the docker invocation carries its + // own `-e MYSQL_PWD` ahead of the client's `-e `. indexOf reads + // MYSQL_PWD as the statement, which makes this fake answer the + // marker probe with '0' and leaves the marker path unexercised. + const sql = args[args.lastIndexOf('-e') + 1] || '' if (/information_schema\.TABLES/.test(sql)) return { stdout: '1\t1' } return { stdout: '0' } }) @@ -337,6 +480,60 @@ describe('BootstrapHealthGate', function () { expect(reasons.join(' ')).to.match(/upstream decoder carries a durable REORG_HALT/) }) + // The decoder's marker probe is fail-soft: a DB fault keeps the last known + // state, which starts at false with checked_at null. So "never managed to + // look" and "looked, clean" publish the identical boolean, and only the + // timestamp tells them apart. + it('REFUSES a decoder that reports not-halted having never completed a probe', function () { + const gate = loadGate() + const reasons = gate.evaluateStatusPayload( + { status: 'healthy', lag_blocks: 0, reorg_halted: false, reorg_halt_checked_at: null }) + expect(reasons).to.have.lengthOf(1) + expect(reasons[0]).to.match(/never completed a REORG_HALT marker probe/) + }) + + it('passes the same decoder once a probe has actually completed', function () { + const gate = loadGate() + expect(gate.evaluateStatusPayload( + { status: 'healthy', lag_blocks: 0, reorg_halted: false, reorg_halt_checked_at: 1756000000000 } + )).to.deep.equal([]) + }) + + // The indexer publishes no companion timestamp for decoderReorgHalted, so the + // proof rule must not reach it: keying on the wrong field would refuse every + // indexer bootstrap in the fleet. + it('does not demand a probe timestamp from an indexer payload', function () { + const gate = loadGate() + expect(gate.evaluateStatusPayload({ status: 'healthy', lag: 0, decoderReorgHalted: false })) + .to.deep.equal([]) + }) + + it('REFUSES an indexer reporting its block counter wedged', function () { + const gate = loadGate() + const reasons = gate.evaluateStatusPayload( + { status: 'healthy', lag: 3, stallClass: 'wedged', stallReason: 'vm_executor_host_fault' }) + expect(reasons).to.have.lengthOf(1) + expect(reasons[0]).to.match(/WEDGED/) + expect(reasons[0]).to.match(/vm_executor_host_fault/) + }) + + // The negative control that keeps the wedge check from becoming a fleet-wide + // refusal: on testnet4 the future-stamped-block wait is the PERMANENT steady + // state, and it carries degraded:true with a named stallReason the whole time. + it('passes the healthy future-block wait and an in-grace barrier defer', function () { + const gate = loadGate() + expect(gate.evaluateStatusPayload({ + status: 'healthy', lag: 6, stallClass: 'future_block_wait', degraded: true, + stallReason: 'price_sync_barrier', waitingOnFutureBlock: true + })).to.deep.equal([]) + expect(gate.evaluateStatusPayload({ + status: 'healthy', lag: 3, stallClass: 'barrier_defer', degraded: true, + stallReason: 'match_barrier' + })).to.deep.equal([]) + expect(gate.evaluateStatusPayload({ status: 'healthy', lag: 0, stallClass: 'none' })) + .to.deep.equal([]) + }) + it('refuses when the node tip is stale, since the lag is then unknowable', function () { const gate = loadGate() const reasons = gate.evaluateStatusPayload({ status: 'healthy', lag: 0, node_height_stale: true }) diff --git a/test/unit/BootstrapService.test.js b/test/unit/BootstrapService.test.js index 386c962..61fe1b6 100644 --- a/test/unit/BootstrapService.test.js +++ b/test/unit/BootstrapService.test.js @@ -1355,8 +1355,14 @@ describe('BootstrapService', function () { expect.fail() } catch (err) { expect(err.message).to.include('docker tar restore exited with code 1') + // Post-wipe abort (uuid:7edc76f3): the volume was already cleared, + // so the error is tagged and the container is NOT restarted. A + // restart here boots a fresh tracker with halted=false over an + // incomplete store, which then reports itself caught up. + expect(err.postWipe).to.be.true } - expect(stubs.dockerService.startContainer.called).to.be.true + expect(stubs.dockerService.stopContainer.called).to.be.true + expect(stubs.dockerService.startContainer.called).to.be.false }) it('throws when malformed archive (inner archive missing after extract)', async function () { diff --git a/test/unit/DatabaseService.test.js b/test/unit/DatabaseService.test.js index e12a616..65118d8 100644 --- a/test/unit/DatabaseService.test.js +++ b/test/unit/DatabaseService.test.js @@ -87,6 +87,9 @@ function makeStubs(overrides = {}) { statusChanged: sinon.stub().resolves(), getStatusFromContainer: sinon.stub().resolves({ State: { Status: 'running' } }), forceRemoveContainerByName: sinon.stub().resolves(true), + // Default 'gone': docker positively reports the DB container absent, the + // only state the install branch is allowed to force-remove from. + probeContainerPresenceByName: sinon.stub().resolves('gone'), addContainerToNetwork: sinon.stub().resolves(true), getDockerNetworkInspect: sinon.stub().resolves({ IPAM: { Config: [{ Gateway: '172.18.0.1' }] } @@ -168,7 +171,8 @@ function loadDatabaseService(stubs, constants = {}, configValues = {}) { getStatusFromContainer: stubs.getStatusFromContainer, getDockerNetworkInspect: stubs.getDockerNetworkInspect, addContainerToNetwork: stubs.addContainerToNetwork, - forceRemoveContainerByName: stubs.forceRemoveContainerByName + forceRemoveContainerByName: stubs.forceRemoveContainerByName, + probeContainerPresenceByName: stubs.probeContainerPresenceByName }, // buildDatabaseModule lazy-requires this for the multi-stack host-port // pre-flight; stub it so the install branch doesn't load the real @@ -387,6 +391,45 @@ describe('DatabaseService', function () { expect(stubs.execFileAsync.called).to.be.true }) + // uuid:8a3e5182. The install branch is entered when checkIfDatabaseModuleExists + // returns null, and that helper swallows EVERY error and also answers null on + // any inspect output it cannot parse. So "we are installing" is not evidence + // the container is absent, while the force-remove that follows is `docker rm -f` + // against the stack's only persistent data store. These pin that the delete now + // needs docker's own "no such container", not merely a falsy lookup. + for (const presence of ['exists', 'unknown']) { + it(`refuses to force-remove the MariaDB container when the probe says '${presence}'`, async function () { + const stubs = makeStubs() + stubs.probeContainerPresenceByName = sinon.stub().resolves(presence) + stubs.execFileAsync.onFirstCall().rejects(new Error('No such container')) + stubs.execFileAsync.resolves({ stdout: VALID_CONTAINER_ID + '\n' }) + const ds = loadDatabaseService(stubs) + + let threw = null + try { + await ds.buildDatabaseModule('bitcoin', 'mainnet') + } catch (err) { threw = err } + + expect(threw, 'an ambiguous probe must abort, not delete').to.be.an.instanceOf(Error) + expect(threw.message).to.include(presence) + expect(threw.message).to.include('xchain-node-database') + expect(stubs.forceRemoveContainerByName.called, 'docker rm -f must not run').to.be.false + expect(findDockerRunArgs(stubs.execFileAsync), 'docker run must not run').to.be.null + }) + } + + it('force-removes only after docker positively reports the container gone', async function () { + const stubs = makeStubs() + stubs.execFileAsync.onFirstCall().rejects(new Error('No such container')) + stubs.execFileAsync.resolves({ stdout: VALID_CONTAINER_ID + '\n' }) + const ds = loadDatabaseService(stubs) + await ds.buildDatabaseModule('bitcoin', 'mainnet') + + expect(stubs.probeContainerPresenceByName.calledWith('xchain-node-database')).to.be.true + expect(stubs.forceRemoveContainerByName.calledOnce).to.be.true + expect(findDockerRunArgs(stubs.execFileAsync)).to.not.be.null + }) + it('throws instead of returning undefined when docker run output is not a 64-hex id', async function () { // uuid:fb0c275d: a mismatched id (e.g. a warning line ahead of the id) // means the container IS running but unregistered; falling through @@ -897,6 +940,44 @@ describe('DatabaseService', function () { expect(result).to.equal('1') }) + // The real mariadb client clusters short flags, and the docker sibling + // hands this string to it verbatim, so '-BN' has to mean batch mode here + // too. It did not: the token regexes demanded a whitespace-delimited + // '-B', so every '-BN' caller (the halt-marker probe in the bootstrap + // health gate, the external-DB freshness check that gates a DROP/restore) + // read '' and parsed it into a NaN that loses every comparison. + it('recognizes the clustered short flag -BN as batch mode', async function () { + const stubs = makeStubs() + stubs.mariadb._fakeConn.query.resolves([['3'], ['5']]) + const ds = loadDatabaseService(stubs) + const result = await ds.executeNativeMariaDbCommand(extCfg, 'SELECT id FROM tbl', '-BN') + expect(result).to.equal('3\n5') + }) + + it('recognizes the clustered short flag in either order (-NB)', async function () { + const stubs = makeStubs() + stubs.mariadb._fakeConn.query.resolves([['7']]) + const ds = loadDatabaseService(stubs) + const result = await ds.executeNativeMariaDbCommand(extCfg, 'SELECT id FROM tbl', '-NB') + expect(result).to.equal('7') + }) + + it('asks the driver for array rows when a clustered flag grants batch mode', async function () { + const stubs = makeStubs() + stubs.mariadb._fakeConn.query.resolves([['1']]) + const ds = loadDatabaseService(stubs) + await ds.executeNativeMariaDbCommand(extCfg, 'SELECT 1', '-BN') + expect(stubs.mariadb.createConnection.lastCall.args[0].rowsAsArray).to.equal(true) + }) + + it('still returns empty for an option string carrying no batch flag', async function () { + const stubs = makeStubs() + stubs.mariadb._fakeConn.query.resolves([['1']]) + const ds = loadDatabaseService(stubs) + const result = await ds.executeNativeMariaDbCommand(extCfg, 'SELECT 1', '-N') + expect(result).to.equal('') + }) + it('closes connection even when query throws', async function () { const stubs = makeStubs() stubs.mariadb._fakeConn.query.rejects(new Error('query error')) diff --git a/test/unit/DockerService.test.js b/test/unit/DockerService.test.js index 759e236..272bae0 100644 --- a/test/unit/DockerService.test.js +++ b/test/unit/DockerService.test.js @@ -373,6 +373,54 @@ describe('DockerService', function () { }) }) + // uuid:8a3e5182. A caller about to DELETE a stateful container needs positive + // evidence of absence, and every other lookup here answers falsy for "absent", + // "daemon hiccup" and "unparseable payload" alike. Only docker's own + // "no such container" may read as gone. + describe('probeContainerPresenceByName()', function () { + function probeWith(handler) { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, ...rest) => { + const cb = typeof rest[0] === 'function' ? rest[0] : rest[1] + expect(cmd).to.equal('docker') + expect(args).to.deep.equal(['inspect', '--type', 'container', '--format', '{{.Id}}', 'xchain-node-database']) + handler(cb) + }) + return loadDockerService(stubs).probeContainerPresenceByName('xchain-node-database') + } + + const ID = 'a'.repeat(64) + + it("reports 'exists' on a clean 64-hex id", async function () { + expect(await probeWith(cb => cb(null, ID + '\n', ''))).to.equal('exists') + }) + + it("reports 'gone' only when docker says no such container", async function () { + expect(await probeWith(cb => { + cb(new Error('Command failed'), '', 'Error: No such object: xchain-node-database\n') + })).to.equal('gone') + }) + + it("reports 'unknown' when the daemon is unreachable", async function () { + expect(await probeWith(cb => { + cb(new Error('Cannot connect to the Docker daemon at unix:///var/run/docker.sock'), '', + 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock') + })).to.equal('unknown') + }) + + it("reports 'unknown' on a timeout, which carries no absence evidence", async function () { + expect(await probeWith(cb => { + const err = new Error('spawn ETIMEDOUT') + err.killed = true + cb(err, '', '') + })).to.equal('unknown') + }) + + it("reports 'unknown' on a clean exit that yielded no id", async function () { + expect(await probeWith(cb => cb(null, 'Warning: something\n', ''))).to.equal('unknown') + }) + }) + describe('getStatusFromContainer()', function () { it('runs docker inspect and returns parsed JSON', async function () { const stubs = makeStubs() From 95b1b7dffe61e210065886a0410eca826a846281 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 25 Aug 2026 09:37:48 -0700 Subject: [PATCH 11/12] fix(node): vendored coin corrections The fee-destination comment in the vendored coin bundles said the override was env-overridable after it had been restricted to regtest. Those three files are vendored byte-identically across the fleet behind conformance guards, so the correction is propagated rather than applied in one place. --- src/coins/BTC.js | 2 +- src/coins/DOGE.js | 2 +- src/coins/LTC.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coins/BTC.js b/src/coins/BTC.js index dc3bcb7..f36c86f 100644 --- a/src/coins/BTC.js +++ b/src/coins/BTC.js @@ -121,7 +121,7 @@ module.exports = { GAS: '1XChain3M4uRwcHqt4XuhVBUQ8cL4qQsA', DONATE1: '1Donate1GERVKPW6GFQcnGeTa8dgL6Abyp', // Protocol Development DONATE2: '1Donate2LkbBrsanwCVRPWZCXAqQcvcqGz', // Community Development - FEE_DESTINATION: '1FeesxM9LTEjBYVTkynK6jfDBgvksuh2WL', // native-fee destination (env-overridable) + FEE_DESTINATION: '1FeesxM9LTEjBYVTkynK6jfDBgvksuh2WL', // native-fee destination (regtest-only env override; ignored on mainnet/testnet) REWARD: '1RewardsRQTXMAytLt4bBQvPEscKsSEXt', // validator reward pool (COLLECT) EXPLORER: '1Donate3GBGSZzzrS9U9gUgURYKscAE6Yn', // display-only donation; not read by indexer }, diff --git a/src/coins/DOGE.js b/src/coins/DOGE.js index 424394c..71bd5d5 100644 --- a/src/coins/DOGE.js +++ b/src/coins/DOGE.js @@ -72,7 +72,7 @@ module.exports = { GAS: 'DGasfpttCnTijuuoAdiJ9sXJjG7vQ5pMkW', DONATE1: 'DDonate1RBcwGnCRNnVtwuCmQyWW1Gn25f', // Protocol Development DONATE2: 'DDonate2o3Sg4phybp92oFpkmv8S9ZhGSV', // Community Development - FEE_DESTINATION: 'DFeesjvoMoVqd9UDuwDSAxzHMF5xZFgeG9', // native-fee destination (env-overridable) + FEE_DESTINATION: 'DFeesjvoMoVqd9UDuwDSAxzHMF5xZFgeG9', // native-fee destination (regtest-only env override; ignored on mainnet/testnet) REWARD: 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', // structural only; COLLECT/XCHAIN are BTC-only EXPLORER: 'DDonate3FCoUgi1bxW5r9c2p75uKTLw9qE', // display-only donation }, diff --git a/src/coins/LTC.js b/src/coins/LTC.js index 02a23fd..f02d1e3 100644 --- a/src/coins/LTC.js +++ b/src/coins/LTC.js @@ -72,7 +72,7 @@ module.exports = { GAS: 'LXChainCN6yjHVqqS9tYzYVYZ8CCZcSx72', DONATE1: 'Ldonate18tNZcVThKm5MX33EjvhaanJ6Mg', // Protocol Development DONATE2: 'Ldonate2io846q2e7q8dUArh3TNnaq9ENb', // Community Development - FEE_DESTINATION: 'Lfees7tszAx5Gqam2fuqf6biaX3LXafM4H', // native-fee destination (env-overridable) + FEE_DESTINATION: 'Lfees7tszAx5Gqam2fuqf6biaX3LXafM4H', // native-fee destination (regtest-only env override; ignored on mainnet/testnet) REWARD: 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', // structural only; COLLECT/XCHAIN are BTC-only EXPLORER: 'Ldonate3FfyqbYQAYxo3qjFLcu28oUdAfn', // display-only donation }, From 841317e78a6dc330d5db733ac539ee1a32666f55 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 25 Aug 2026 20:27:58 -0700 Subject: [PATCH 12/12] chore(release): v0.11.0 Pins the twelve component tags this train cut, bumps the carrier to 0.11.0 and dates the changelog. Every commit in the manifest was read back from its tag on origin rather than from local state, and each was checked three ways before being written: the v0.11.0 tag dereferences to that commit, GitHub reports the tag verified against the release identity, and the commit is that repository's master tip. All twelve are master merge commits whose CI is green. --- CHANGELOG.md | 5 +++- package-lock.json | 4 +-- package.json | 2 +- src/release-manifest.json | 62 +++++++++++++++++++-------------------- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8cff09..84ec0f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.11.0] - 2026-08-26 ### Added - Install and update end with a bootstrap restore summary, so a restore that did not happen is stated rather than left as one warning mid-log. - `XCHAIN_NODE_FORCE_BOOTSTRAP=1` restores a published bootstrap over an already-populated service, for when the install that would have taken it failed. +### Changed +- The release manifest pins the v0.11.0 component set. + ### Fixed - The migration precondition refusal only prints a scoped migrate command when the running build is confirmed to support one, and otherwise names every migration an unscoped run would apply. - `reset xchain-decoder` now refuses while an indexer is installed and names the `--with-indexer` joint form, because resetting one half of the pair leaves the other unable to commit blocks. diff --git a/package-lock.json b/package-lock.json index 452b899..1d0aed3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-node", - "version": "0.10.0", + "version": "0.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-node", - "version": "0.10.0", + "version": "0.11.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", diff --git a/package.json b/package.json index 7f2aa2f..44ffabe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "xchain-node", - "version": "0.10.0", + "version": "0.11.0", "description": "xchain-node allows users to install, configure and run XChain platform nodes.", "license": "AGPL-3.0-or-later", "repository": { diff --git a/src/release-manifest.json b/src/release-manifest.json index f212290..fccdee2 100644 --- a/src/release-manifest.json +++ b/src/release-manifest.json @@ -1,63 +1,63 @@ { "_comment": [ - "Pinned component set for XChain Platform v0.10.0.", + "Pinned component set for XChain Platform v0.11.0.", "Written at ceremony step 6 from the ACTUAL tagged master merge commits.", "xchain-node is the carrier and is not listed: checking out its tag IS this manifest.", - "Every module in constants.js modulesUrls is pinned here. The v0.9.0 manifest", - "carried only 8 of the 12, so a pinned install of that train still cloned", - "xchain-sdk, xchain-e2e-test, xchain-contracts and xchain-regtest-miner at their", - "default branch, which is not reproducible. All 12 are pinned from this train on." + "Every module in constants.js modulesUrls is pinned here, all 12 of them.", + "Each commit below is the master MERGE commit its v0.11.0 tag names, and every", + "one of those tags is GPG-signed by the platform release key and reports", + "verified against the tagger identity releases@xchain.io." ], - "platform_version": "0.10.0", - "released": "2026-08-22", + "platform_version": "0.11.0", + "released": "2026-08-26", "components": { "xchain-vm": { - "tag": "v0.10.0", - "commit": "4db160dd6ebcb1a7c8e8309b45ce00ffb5f7793c" + "tag": "v0.11.0", + "commit": "75736e2808557673c6475f131df1020b68a30c59" }, "xchain-decoder": { - "tag": "v0.10.0", - "commit": "09170e9a32c67484ece203a6cfc6f149f530f0a7" + "tag": "v0.11.0", + "commit": "909672ea64f6715f013d5aec831219cbc245b547" }, "xchain-indexer": { - "tag": "v0.10.0", - "commit": "698be256cb062ba3f05bd5a4e69d9d2f1cb206e7" + "tag": "v0.11.0", + "commit": "cef32e403e5751afaac9f1d4a61858ab581914f4" }, "xchain-hub": { - "tag": "v0.10.0", - "commit": "cbb97b97e1b670a8b28b40d4abe9f664d8eb284e" + "tag": "v0.11.0", + "commit": "3da24db45e810b3abf3c2b03fb456326c5365e5f" }, "xchain-sync": { - "tag": "v0.10.0", - "commit": "70a58033ff3b8468c26053afdfa4b8614e502bc8" + "tag": "v0.11.0", + "commit": "c732266cde381f8310907bd4ec8e9f4c8b468396" }, "xchain-encoder": { - "tag": "v0.10.0", - "commit": "27a2a87f892aed2b9602361ceb78f24a639050bb" + "tag": "v0.11.0", + "commit": "9efd49e708d25a1e8cc949a4b4a5d1e01b53a61e" }, "xchain-utxo-tracker": { - "tag": "v0.10.0", - "commit": "c48b87be04a7115f78b2ec7d3031d25104be1706" + "tag": "v0.11.0", + "commit": "5285ef2a389c786894d895b8d9f6afbf9ff31c38" }, "xchain-explorer": { - "tag": "v0.10.0", - "commit": "e4ca98a6af0f153d4ec928576e1f24568c3dfa8b" + "tag": "v0.11.0", + "commit": "d9d8f59ae0f56ebe20252f1b065fad4eec67c7ec" }, "xchain-sdk": { - "tag": "v0.10.0", - "commit": "7bcefac0c28544f7fcb742629fc553e3d6e2c49e" + "tag": "v0.11.0", + "commit": "70c44da101e39467ec17b6d7c6076bb01d23e277" }, "xchain-e2e-test": { - "tag": "v0.10.0", - "commit": "5db34052213ac847218844eef3c1fd2690f2c2b4" + "tag": "v0.11.0", + "commit": "852055dab552af9d6bc93cf83a4ebe7c2ce95186" }, "xchain-contracts": { - "tag": "v0.10.0", - "commit": "da194411bdaa43add28fb1af1d345f92c89c0d2e" + "tag": "v0.11.0", + "commit": "f1ddcb311aa33e0bc609127e8771b3847009b3a8" }, "xchain-regtest-miner": { - "tag": "v0.10.0", - "commit": "ba73838eeaab95c942ef3cbbae9f6efccb9726bb" + "tag": "v0.11.0", + "commit": "2a4468471e07bed00f9c22a3a5237d1e51fe2c97" } } }