From d311f3fe1db091352e24e55e2a0d9160e743cc95 Mon Sep 17 00:00:00 2001 From: Marty Alcala Date: Wed, 29 Jul 2026 17:25:24 -0400 Subject: [PATCH] fix ERC-20 effect materialization --- packages/bitcore-node/docs/erc20-effects.md | 104 ++ .../scripts/backfillErc20Effects.ts | 543 ++++++ .../src/providers/chain-state/evm/api/csp.ts | 161 +- .../providers/chain-state/evm/api/gnosis.ts | 34 +- .../providers/chain-state/evm/erc20Effects.ts | 526 ++++++ .../chain-state/evm/models/transaction.ts | 249 ++- .../evm/p2p/EVMVerificationPeer.ts | 8 + .../src/providers/chain-state/evm/p2p/p2p.ts | 7 + .../chain-state/evm/p2p/syncWorker.ts | 7 + .../src/providers/chain-state/evm/types.ts | 15 + packages/bitcore-node/src/types/Config.ts | 4 + .../integration/models/transaction.test.ts | 338 ++++ .../evm/backfillErc20Effects.test.ts | 398 ++++ .../chain-state/evm/erc20Effects.test.ts | 1597 +++++++++++++++++ 14 files changed, 3946 insertions(+), 45 deletions(-) create mode 100644 packages/bitcore-node/docs/erc20-effects.md create mode 100755 packages/bitcore-node/scripts/backfillErc20Effects.ts create mode 100644 packages/bitcore-node/src/providers/chain-state/evm/erc20Effects.ts create mode 100644 packages/bitcore-node/test/unit/providers/chain-state/evm/backfillErc20Effects.test.ts create mode 100644 packages/bitcore-node/test/unit/providers/chain-state/evm/erc20Effects.test.ts diff --git a/packages/bitcore-node/docs/erc20-effects.md b/packages/bitcore-node/docs/erc20-effects.md new file mode 100644 index 00000000000..8301ca5eb93 --- /dev/null +++ b/packages/bitcore-node/docs/erc20-effects.md @@ -0,0 +1,104 @@ +# Canonical ERC-20 effects + +## Stored field + +Confirmed EVM transactions may contain an additive, inclusion-bound field: + +```json +{ + "erc20Effects": { + "blockHash": "0x...", + "version": 1, + "items": [] + } +} +``` + +`items: []` is an authoritative successful result with no supported ERC-20 +`Transfer(address,address,uint256)` logs. Absence means materialization has not +successfully completed for that inclusion. The existing `effects` field is not +rewritten; local readers compose their established public `effects` response +from legacy non-ERC-20 entries plus valid canonical items. + +Version 1 accepts exactly three-topic `Transfer` logs with a 32-byte amount. +Four-topic ERC-721-shaped logs and other nonstandard Transfer-shaped logs are +counted and ignored. Malformed RPC data, removed logs, duplicate log identities, +block-hash mismatches, and logs for transactions outside the prepared block fail +the block before persistence. Raw logs and receipts are not stored by this +feature. + +## Configuration and rollout + +Configuration is per EVM chain/network: + +```json +{ + "erc20Effects": { + "materializationEnabled": true, + "strictReadActivationHeight": 22000000 + } +} +``` + +`materializationEnabled` makes every confirmed block processed by an upgraded +writer receive a version-1 result, including replayed blocks below the read +boundary. `strictReadActivationHeight` controls reads only: at or above that +height, a missing, malformed, unsupported-version, or inclusion-mismatched +result suppresses legacy heuristic ERC-20 entries while preserving all +non-ERC-20 effects. + +Deploy materialization-aware readers/writers to every EVM writer before enabling +materialization. Mixed old/new canonical writers are not supported. During a +coordinated rollback, confirmed persistence preserves a numerically newer stored +generation while continuing ordinary transaction updates and wallet merges. A +typical rollout enables live materialization, chooses an activation height at or +ahead of the deployment tip, backfills lower heights, validates coverage, and +later moves or removes the compatibility boundary. + +## Online backfill + +Build `@bitpay-labs/bitcore-node`, then run: + +```sh +node build/scripts/backfillErc20Effects.js \ + --chain ETH \ + --network mainnet \ + --start-height 18000000 \ + --end-height 18099999 \ + --concurrency 4 \ + --delay-ms 100 \ + --dry-run +``` + +Remove `--dry-run` to publish. `--concurrency` bounds transaction updates within +one block; blocks are processed in deterministic ascending-height order. +`--delay-ms` throttles block-to-block RPC and MongoDB load. The script emits the +last completed height for explicit-start resumability. `--force-current-version` +re-publishes version 1, but no mode overwrites a version newer than the running +binary. + +Each transaction update is conditional on `_id`, chain/network, txid, observed +block height/hash, the observed prior `erc20Effects` value, and a parser-version +predicate. Writes are limited to `erc20Effects` plus wallet-ID `$addToSet` +merges. The script also verifies that the exact local processed block hash and +ordered transaction membership still match the RPC's canonical block before +publication. A changed inclusion or concurrent non-identical publication loses the +conditional update and aborts the block without repairing shared block state. +After changed canonical ERC-20 effects are published, cache invalidation is limited to the +canonical token/address participants; authoritative empty results do not churn +native-balance cache entries. + +## Address-history indexes + +Two partial multikey indexes preserve existing local address-history parity: + +- `{ chain, network, "erc20Effects.items.to", blockTimeNormalized }` +- `{ chain, network, "erc20Effects.items.from", blockTimeNormalized }` + +Each canonical transfer contributes one entry to each index. A practical +order-of-magnitude estimate is roughly 200–400 bytes of combined index storage +per materialized transfer after address, chain/network, time, record identifier, +and B-tree overhead, but actual size depends on MongoDB version, prefix +compression, chain/network string lengths, and storage engine settings. Operators +should measure representative blocks before a large backfill and provision +index build I/O and disk headroom accordingly. diff --git a/packages/bitcore-node/scripts/backfillErc20Effects.ts b/packages/bitcore-node/scripts/backfillErc20Effects.ts new file mode 100755 index 00000000000..12321beceac --- /dev/null +++ b/packages/bitcore-node/scripts/backfillErc20Effects.ts @@ -0,0 +1,543 @@ +#!/usr/bin/env node + +import { CryptoRpc } from '@bitpay-labs/crypto-rpc'; +import { WalletAddressStorage } from '../src/models/walletAddress'; +import { + ERC20_EFFECTS_VERSION, + attachErc20EffectsToTransactions, + erc20EffectsEqual, + fetchTransferLogsForBlock, + getCanonicalErc20ParticipantAddresses, + isValidErc20EffectsForTransaction, + semanticHashEquals +} from '../src/providers/chain-state/evm/erc20Effects'; +import { EVMBlockStorage } from '../src/providers/chain-state/evm/models/block'; +import { EVMTransactionStorage } from '../src/providers/chain-state/evm/models/transaction'; +import { Rpcs } from '../src/providers/chain-state/evm/p2p/rpcs'; +import { Config } from '../src/services/config'; +import { Storage } from '../src/services/storage'; +import type { MongoBound } from '../src/models/base'; +import type { IRpc } from '../src/providers/chain-state/evm/p2p/rpcs'; +import type { Erc20Effects, IEVMBlock, IEVMTransaction } from '../src/providers/chain-state/evm/types'; +import type { IEVMNetworkConfig, IProvider } from '../src/types/Config'; + +type StoredEvmTransaction = MongoBound & Required, '_id'>>; +const HEX_TRANSACTION_HASH = /^0x[0-9a-fA-F]{64}$/; + +export interface BackfillOptions { + chain: string; + network: string; + startHeight: number; + endHeight: number; + dryRun: boolean; + concurrency: number; + delayMs: number; + forceCurrentVersion: boolean; +} + +interface BlockBackfillStats { + height: number; + transactions: number; + updated: number; + skippedCurrent: number; + skippedNewer: number; + dryRunWouldUpdate: number; + racedConverged: number; + supportedTransferLogs: number; + unsupportedTransferLogs: number; +} + +let shuttingDown = false; + +function registerShutdownHandlers() { + const requestShutdown = () => { + shuttingDown = true; + console.error('Stopping after the current bounded batch...'); + }; + process.on('SIGINT', requestShutdown); + process.on('SIGTERM', requestShutdown); +} + +function usage(message?: string): never { + if (message) { + console.error(`ERROR: ${message}\n`); + } + console.error('Usage: node build/scripts/backfillErc20Effects.js [options]'); + console.error(' --chain REQUIRED (for example ETH or MATIC)'); + console.error(' --network REQUIRED'); + console.error(' --start-height REQUIRED, inclusive'); + console.error(' --end-height REQUIRED, inclusive'); + console.error(' --dry-run Validate and report without writes'); + console.error(' --concurrency Max transaction updates in flight per block (default 4)'); + console.error(' --delay-ms Delay between blocks (default 100)'); + console.error(' --force-current-version Re-publish valid version-1 rows'); + process.exit(1); +} + +function optionValue(args: string[], name: string) { + const index = args.indexOf(name); + return index === -1 ? undefined : args[index + 1]; +} + +export function parseBackfillOptions(args: string[]): BackfillOptions { + const chain = optionValue(args, '--chain')?.toUpperCase(); + const network = optionValue(args, '--network')?.toLowerCase(); + const startHeight = Number(optionValue(args, '--start-height')); + const endHeight = Number(optionValue(args, '--end-height')); + const concurrency = optionValue(args, '--concurrency') === undefined ? 4 : Number(optionValue(args, '--concurrency')); + const delayMs = optionValue(args, '--delay-ms') === undefined ? 100 : Number(optionValue(args, '--delay-ms')); + + if (!chain || !network) { + throw new Error('chain and network are required'); + } + if (!Number.isSafeInteger(startHeight) || startHeight < 0) { + throw new Error('start-height must be a non-negative safe integer'); + } + if (!Number.isSafeInteger(endHeight) || endHeight < startHeight) { + throw new Error('end-height must be a safe integer greater than or equal to start-height'); + } + if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 100) { + throw new Error('concurrency must be an integer from 1 through 100'); + } + if (!Number.isSafeInteger(delayMs) || delayMs < 0 || delayMs > 60000) { + throw new Error('delay-ms must be an integer from 0 through 60000'); + } + + return { + chain, + network, + startHeight, + endHeight, + dryRun: args.includes('--dry-run'), + concurrency, + delayMs, + forceCurrentVersion: args.includes('--force-current-version') + }; +} + + +function getBackfillChainConfig(options: Pick): IEVMNetworkConfig { + const chains = Config.get().chains as Record | undefined>; + const config = chains[options.chain]?.[options.network]; + if (!config) { + throw new Error(`No configuration found for ${options.chain}:${options.network}`); + } + return config; +} + +function providerForBackfill(config: IEVMNetworkConfig): IProvider { + if (config.provider && !config.provider.disabled) { + return config.provider; + } + const providers = (config.providers || []).filter(provider => !provider.disabled); + const historical = providers.find(provider => provider.dataType === 'historical' || provider.dataType === 'combined'); + if (historical) { + return historical; + } + if (providers[0]) { + return providers[0]; + } + throw new Error('No enabled EVM RPC provider is configured'); +} + +async function connectRpc(options: BackfillOptions, config: IEVMNetworkConfig): Promise { + const providerConfig = providerForBackfill(config); + const rpcConfig = { ...providerConfig, chain: options.chain, isEVM: true, currencyConfig: {} }; + const web3 = new CryptoRpc(rpcConfig as any).get(options.chain).web3; + let client = config.client; + if (!client) { + const nodeInfo = await web3.eth.getNodeInfo(); + const detected = nodeInfo.split('/')[0].toLowerCase(); + client = detected === 'erigon' ? 'erigon' : 'geth'; + } + return new Rpcs[client](web3); +} + +export function buildBackfillTransactionFilter(params: { + tx: StoredEvmTransaction; + observedErc20Effects: Erc20Effects | undefined; +}) { + const { tx, observedErc20Effects } = params; + const observedPredicate = observedErc20Effects === undefined + ? { erc20Effects: { $exists: false } } + : { erc20Effects: observedErc20Effects }; + + return { + _id: tx._id, + chain: tx.chain, + network: tx.network, + txid: tx.txid, + blockHeight: tx.blockHeight, + blockHash: tx.blockHash, + $and: [ + { + $or: [ + { 'erc20Effects.version': { $exists: false } }, + { 'erc20Effects.version': { $lte: ERC20_EFFECTS_VERSION } } + ] + }, + observedPredicate + ] + }; +} + +function cloneErc20Effects(value: Erc20Effects | undefined) { + return value === undefined ? undefined : JSON.parse(JSON.stringify(value)) as Erc20Effects; +} + +async function loadCanonicalBlock(options: BackfillOptions, height: number) { + const blocks = await EVMBlockStorage.collection + .find({ chain: options.chain, network: options.network, height, processed: true }) + .limit(2) + .toArray(); + if (blocks.length !== 1) { + throw new Error(`Expected exactly one processed local block at height ${height}; found ${blocks.length}`); + } + return blocks[0] as MongoBound; +} + +export function validateLocalBlockTransactions( + block: Pick, + transactions: Array +) { + if (transactions.length !== block.transactionCount) { + throw new Error( + `Local transaction count mismatch at height ${block.height}: block=${block.transactionCount}, rows=${transactions.length}` + ); + } + const txids = new Set(); + const transactionIndexes = new Set(); + for (const tx of transactions) { + if (tx.blockHash?.toLowerCase() !== block.hash.toLowerCase()) { + throw new Error(`Transaction ${tx.txid} has inconsistent inclusion at height ${block.height}`); + } + const txid = tx.txid.toLowerCase(); + if (txids.has(txid)) { + throw new Error(`Duplicate local transaction ${tx.txid} at height ${block.height}`); + } + txids.add(txid); + + const transactionIndex = typeof tx.transactionIndex === 'number' + ? tx.transactionIndex + : Number(tx.transactionIndex); + if ( + !Number.isSafeInteger(transactionIndex) || + transactionIndex < 0 || + transactionIndex >= block.transactionCount || + transactionIndexes.has(transactionIndex) + ) { + throw new Error(`Transaction ${tx.txid} has invalid transactionIndex at height ${block.height}`); + } + transactionIndexes.add(transactionIndex); + } + transactions.sort((left, right) => Number(left.transactionIndex) - Number(right.transactionIndex)); + return transactions; +} + +function rpcTransactionHash(value: unknown, height: number, transactionIndex: number) { + const hash = typeof value === 'string' + ? value + : value && typeof value === 'object' + ? (value as any).hash + : undefined; + if (typeof hash !== 'string' || !HEX_TRANSACTION_HASH.test(hash)) { + throw new Error(`RPC block at height ${height} has a missing or malformed transaction hash at index ${transactionIndex}`); + } + return hash.toLowerCase(); +} + +export function validateRpcBlockTransactions( + block: Pick, + transactions: Array, + rpcBlock: unknown +) { + if (!rpcBlock || typeof rpcBlock !== 'object' || !semanticHashEquals((rpcBlock as any).hash, block.hash)) { + throw new Error(`RPC block identity changed before persistence at height ${block.height}`); + } + const rpcTransactions = (rpcBlock as any).transactions; + if (!Array.isArray(rpcTransactions)) { + throw new Error(`RPC block at height ${block.height} is missing transaction membership`); + } + if (rpcTransactions.length !== block.transactionCount || rpcTransactions.length !== transactions.length) { + throw new Error( + `RPC transaction count mismatch at height ${block.height}: block=${block.transactionCount}, rpc=${rpcTransactions.length}, local=${transactions.length}` + ); + } + + const seenRpcTransactionHashes = new Set(); + for (let transactionIndex = 0; transactionIndex < rpcTransactions.length; transactionIndex++) { + const rpcTxid = rpcTransactionHash(rpcTransactions[transactionIndex], block.height, transactionIndex); + if (seenRpcTransactionHashes.has(rpcTxid)) { + throw new Error(`Duplicate RPC transaction ${rpcTxid} at height ${block.height}`); + } + seenRpcTransactionHashes.add(rpcTxid); + + const local = transactions[transactionIndex]; + if ( + Number(local.transactionIndex) !== transactionIndex || + typeof local.txid !== 'string' || + !HEX_TRANSACTION_HASH.test(local.txid) || + !semanticHashEquals(local.txid, rpcTxid) + ) { + throw new Error( + `RPC transaction membership mismatch at height ${block.height}, index ${transactionIndex}: local=${local.txid}, rpc=${rpcTxid}` + ); + } + } +} + +export async function assertRpcBlockTransactions( + rpc: IRpc, + block: Pick, + transactions: Array +) { + const rpcBlock = await rpc.getBlock(block.height); + validateRpcBlockTransactions(block, transactions, rpcBlock); +} + +export function materializeBackfillTransactions(params: { + block: Pick; + transactions: Array; + logs: unknown; +}) { + return attachErc20EffectsToTransactions(params); +} + +async function loadAndValidateBlockTransactions(options: BackfillOptions, block: MongoBound) { + const transactions = await EVMTransactionStorage.collection + .find({ chain: options.chain, network: options.network, blockHeight: block.height }) + .sort({ transactionIndex: 1, txid: 1 }) + .toArray() as Array; + return validateLocalBlockTransactions(block, transactions); +} + +async function walletIdsByAddress(options: BackfillOptions, transactions: Array) { + const addresses = new Set(); + for (const tx of transactions) { + for (const address of getCanonicalErc20ParticipantAddresses(tx)) { + addresses.add(address); + addresses.add(address.toLowerCase()); + } + } + if (!addresses.size) { + return new Map(); + } + + const rows = await WalletAddressStorage.collection.find({ + chain: options.chain, + network: options.network, + address: { $in: Array.from(addresses) } + }).toArray(); + const byAddress = new Map(); + for (const row of rows) { + const key = row.address.toLowerCase(); + const wallets = byAddress.get(key) || []; + if (!wallets.some(wallet => wallet.toHexString() === row.wallet.toHexString())) { + wallets.push(row.wallet); + } + byAddress.set(key, wallets); + } + return byAddress; +} + +function walletsForTransaction(tx: StoredEvmTransaction, byAddress: Map) { + const wallets = new Map(); + for (const address of getCanonicalErc20ParticipantAddresses(tx)) { + for (const wallet of byAddress.get(address.toLowerCase()) || []) { + wallets.set(wallet.toHexString(), wallet); + } + } + return Array.from(wallets.values()); +} + +export function getBackfillDisposition(params: { + tx: StoredEvmTransaction; + observedErc20Effects: Erc20Effects | undefined; + forceCurrentVersion: boolean; +}) { + const { tx, observedErc20Effects, forceCurrentVersion } = params; + const observedVersion = observedErc20Effects?.version; + if (typeof observedVersion === 'number' && observedVersion > ERC20_EFFECTS_VERSION) { + return 'skipped-newer' as const; + } + if (!forceCurrentVersion && observedErc20Effects !== undefined && isValidErc20EffectsForTransaction(tx, observedErc20Effects)) { + return 'skipped-current' as const; + } + return 'write' as const; +} + +export async function updateBackfillTransaction(params: { + options: BackfillOptions; + tx: StoredEvmTransaction; + observedErc20Effects: Erc20Effects | undefined; + wallets: any[]; +}) { + const { options, tx, observedErc20Effects, wallets } = params; + const newErc20Effects = tx.erc20Effects!; + const disposition = getBackfillDisposition({ + tx, + observedErc20Effects, + forceCurrentVersion: options.forceCurrentVersion + }); + if (disposition !== 'write') { + return disposition; + } + if (options.dryRun) { + return 'dry-run' as const; + } + + const update: any = { $set: { erc20Effects: newErc20Effects } }; + if (wallets.length) { + update.$addToSet = { wallets: { $each: wallets } }; + } + const result = await EVMTransactionStorage.collection.updateOne( + buildBackfillTransactionFilter({ tx, observedErc20Effects }), + update + ); + const matchedCount = result.matchedCount ?? result.result?.n ?? 0; + const modifiedCount = result.modifiedCount ?? result.result?.nModified ?? 0; + if (!matchedCount) { + const current = await EVMTransactionStorage.collection.findOne({ _id: tx._id }); + if ( + current && + current.chain === tx.chain && + current.network === tx.network && + current.txid?.toLowerCase() === tx.txid?.toLowerCase() && + current.blockHeight === tx.blockHeight && + current.blockHash?.toLowerCase() === tx.blockHash?.toLowerCase() && + isValidErc20EffectsForTransaction(current) && + erc20EffectsEqual(current.erc20Effects, newErc20Effects) + ) { + if (wallets.length) { + const walletResult = await EVMTransactionStorage.collection.updateOne( + { + _id: current._id, + chain: current.chain, + network: current.network, + txid: current.txid, + blockHeight: current.blockHeight, + blockHash: current.blockHash, + erc20Effects: current.erc20Effects + }, + { $addToSet: { wallets: { $each: wallets } } } + ); + const walletMatchedCount = walletResult.matchedCount ?? walletResult.result?.n ?? 0; + if (!walletMatchedCount) { + throw new Error(`Inclusion/version CAS lost while merging wallets for transaction ${tx.txid}`); + } + } + return 'raced-converged' as const; + } + throw new Error(`Inclusion/version CAS lost for transaction ${tx.txid}`); + } + + if (modifiedCount) { + await EVMTransactionStorage.expireErc20BalanceCacheForTransaction({ + chain: options.chain, + network: options.network, + tx + }); + } + return 'updated' as const; +} + +export async function processBlock(options: BackfillOptions, rpc: IRpc, height: number): Promise { + const block = await loadCanonicalBlock(options, height); + const transactions = await loadAndValidateBlockTransactions(options, block); + const observed = new Map(); + for (const tx of transactions) { + observed.set(tx._id.toHexString(), cloneErc20Effects(tx.erc20Effects)); + } + + const fetched = await fetchTransferLogsForBlock(rpc, block); + const attached = materializeBackfillTransactions({ block, transactions, logs: fetched.logs }); + const walletsByAddress = await walletIdsByAddress(options, transactions); + // Historical local state must still identify the RPC's canonical block, + // and its ordered transaction membership, including when an exact blockHash + // log filter returns an empty result. + // Keep this check immediately before the bounded publication loop. + await assertRpcBlockTransactions(rpc, block, transactions); + + const stats: BlockBackfillStats = { + height, + transactions: transactions.length, + updated: 0, + skippedCurrent: 0, + skippedNewer: 0, + dryRunWouldUpdate: 0, + racedConverged: 0, + supportedTransferLogs: attached.supportedTransferLogs, + unsupportedTransferLogs: attached.unsupportedTransferLogs + }; + + for (let offset = 0; offset < transactions.length; offset += options.concurrency) { + if (shuttingDown) { + throw new Error(`Interrupted while processing height ${height}`); + } + const batch = transactions.slice(offset, offset + options.concurrency); + const outcomes = await Promise.all(batch.map(tx => updateBackfillTransaction({ + options, + tx, + observedErc20Effects: observed.get(tx._id.toHexString()), + wallets: walletsForTransaction(tx, walletsByAddress) + }))); + for (const outcome of outcomes) { + if (outcome === 'updated') stats.updated++; + if (outcome === 'skipped-current') stats.skippedCurrent++; + if (outcome === 'skipped-newer') stats.skippedNewer++; + if (outcome === 'dry-run') stats.dryRunWouldUpdate++; + if (outcome === 'raced-converged') stats.racedConverged++; + } + } + + return stats; +} + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export async function runBackfill(options: BackfillOptions) { + const chainConfig = getBackfillChainConfig(options); + const rpc = await connectRpc(options, chainConfig); + let lastCompletedHeight = options.startHeight - 1; + + for (let height = options.startHeight; height <= options.endHeight; height++) { + if (shuttingDown) { + break; + } + const stats = await processBlock(options, rpc, height); + lastCompletedHeight = height; + console.log(JSON.stringify({ ...stats, lastCompletedHeight })); + if (height < options.endHeight && options.delayMs) { + await sleep(options.delayMs); + } + } + + console.log(JSON.stringify({ + complete: lastCompletedHeight === options.endHeight, + lastCompletedHeight, + requestedEndHeight: options.endHeight + })); +} + +if (require.main === module) { + registerShutdownHandlers(); + let options: BackfillOptions; + try { + if (process.argv.includes('--help') || process.argv.includes('-h')) { + usage(); + } + options = parseBackfillOptions(process.argv.slice(2)); + } catch (err) { + usage(err instanceof Error ? err.message : String(err)); + } + + Storage.start() + .then(() => runBackfill(options)) + .catch(err => { + console.error(err?.stack || err); + process.exitCode = 1; + }) + .finally(() => Storage.stop()); +} diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts index 8fdbecbd52b..a97f6ce11c3 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts @@ -23,6 +23,7 @@ import { AavePoolAbi } from '../abi/aavePool'; import { AavePoolAbiV2 } from '../abi/aavePoolV2'; import { ERC20Abi } from '../abi/erc20'; import { MultisendAbi } from '../abi/multisend'; +import { getEffectiveEvmEffects, isTransactionRelevantToAddresses } from '../erc20Effects'; import { EVMBlockStorage } from '../models/block'; import { EVMTransactionStorage } from '../models/transaction'; import { EVMTransactionJSON, IEVMBlock, IEVMTransaction, IEVMTransactionInProcess } from '../types'; @@ -453,18 +454,80 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai return tx; } - populateEffects(tx: MongoBound) { - if (!tx.effects || (tx.effects && tx.effects.length == 0)) { + populateEffects>(tx: T) { + // Canonical materialization applies only to local Mongo-backed rows. An + // external provider transaction has no BSON ObjectID and must retain the + // provider/current-master effects semantics. + const mongoId = (tx as any)._id; + const isLocalMongoRow = mongoId && typeof mongoId.toHexString === 'function'; + if (isLocalMongoRow) { + tx.effects = EVMTransactionStorage.getEffectiveEffects(tx as IEVMTransactionInProcess); + } else if (!tx.effects || tx.effects.length === 0) { tx.effects = EVMTransactionStorage.getEffects(tx as IEVMTransactionInProcess); } return tx; } - populateEffectsForAddresses(tx: MongoBound, addresses: Array) { - tx.effects = EVMTransactionStorage.getEffectsForAddresses(tx as IEVMTransactionInProcess, addresses); + populateEffectsForAddresses>(tx: T, addresses: Array) { + const mongoId = (tx as any)._id; + const isLocalMongoRow = mongoId && typeof mongoId.toHexString === 'function'; + if (isLocalMongoRow) { + tx.effects = EVMTransactionStorage.getEffectsForAddresses(tx as IEVMTransactionInProcess, addresses); + } else { + const effects = tx.effects?.length + ? tx.effects + : EVMTransactionStorage.getEffects(tx as IEVMTransactionInProcess); + const addressSet = new Set(addresses.map(address => address.toLowerCase())); + tx.effects = effects.filter(effect => + addressSet.has(effect.to.toLowerCase()) || addressSet.has(effect.from.toLowerCase()) + ); + } return tx; } + private filterTransactionsForAddresses( + addresses: string[], + options: { includeInternalRecipients?: boolean } = {} + ) { + return new TransformWithEventPipe({ + objectMode: true, + transform: (tx, _, done) => { + if (isTransactionRelevantToAddresses(tx, addresses, options)) { + return done(null, tx); + } + return done(); + } + }); + } + + private normalizeAddressHistoryLimit(limit: unknown) { + const requestedLimit = Number(limit); + // Preserve master behavior for absent/zero values and values that master + // converted to NaN before handing them to the MongoDB driver. + if (requestedLimit === 0 || Number.isNaN(requestedLimit)) { + return undefined; + } + // Do not silently turn numeric values that the driver would reject into an + // unbounded post-filter stream. + if (!Number.isFinite(requestedLimit) || !Number.isSafeInteger(requestedLimit)) { + throw new Error('Address transaction limit must be a finite safe integer'); + } + // MongoDB negative limits cap by magnitude and close the cursor. The + // post-normalization limiter must preserve that bounded behavior. + return Math.abs(requestedLimit); + } + + private addressVariants(address: string) { + const variants = new Set([address, address.toLowerCase()]); + try { + variants.add(Web3.utils.toChecksumAddress(address)); + } catch { + // Preserve the endpoint's existing validation behavior. A malformed + // address simply has no additional checksum query variant. + } + return Array.from(variants); + } + async getTransaction(params: StreamTransactionParams) { try { params.network = params.network.toLowerCase(); @@ -546,18 +609,91 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai const { limit, /* since,*/ tokenAddress } = args; if (!args.tokenAddress) { + const requestedLimit = this.normalizeAddressHistoryLimit(limit); + const canonicalAddressVariants = this.addressVariants(address); const query = { $or: [ { chain, network, from: address }, { chain, network, to: address }, { chain, network, 'internal.action.to': address }, // Retained for old db entries - { chain, network, 'effects.to': address } + { chain, network, 'effects.to': address }, + { chain, network, 'erc20Effects.items.to': { $in: canonicalAddressVariants } }, + { chain, network, 'erc20Effects.items.from': { $in: canonicalAddressVariants } } ] }; // NOTE: commented out since and paging for now b/c they were causing extra long query times on insight. - // The case where an address has >1000 txns is an edge case ATM and can be addressed later - Storage.apiStreamingFind(EVMTransactionStorage, query, { limit /* since, paging: '_id'*/ }, req!, res!); + // The case where an address has >1000 txns is an edge case ATM and can be addressed later. + // Canonical normalization can discard rows that matched only through a + // stale heuristic ERC-20 effect. Apply the API limit after that filter so + // discarded rows cannot consume result slots ahead of canonical-only + // matches. + const cursor = EVMTransactionStorage.getTransactions({ query, options: { /* since, paging: '_id'*/ } }); + const cursorStream = new TransformWithEventPipe({ objectMode: true, passThrough: true }); + let cursorClosed = false; + const closeCursor = (): void => { + if (!cursorClosed) { + cursorClosed = true; + void cursor.close(); + } + }; + const stopCursor = () => { + cursor.unpipe(cursorStream); + closeCursor(); + if (!cursorStream.writableEnded) { + cursorStream.end(); + } + }; + cursor.on('error', err => cursorStream.emit('error', err)); + let transactionStream = cursor.pipe(cursorStream); + transactionStream = transactionStream.eventPipe(new TransformWithEventPipe({ + objectMode: true, + transform: (tx, _, done) => { + let config: IEVMNetworkConfig | undefined; + if (tx.chain && tx.network) { + config = Config.chainConfig({ chain: tx.chain, network: tx.network }) as IEVMNetworkConfig; + } + // This endpoint historically exposed the stored effects array. Compose + // canonical ERC-20 items over that exact fallback instead of deriving + // new native/internal effects for older rows. + tx.effects = getEffectiveEvmEffects( + tx, + config, + Array.isArray(tx.effects) ? tx.effects : [] + ); + return done(null, tx); + } + })); + transactionStream = transactionStream.eventPipe(this.filterTransactionsForAddresses( + [address], + { includeInternalRecipients: true } + )); + + if (requestedLimit !== undefined) { + let accepted = 0; + transactionStream = transactionStream.eventPipe(new TransformWithEventPipe({ + objectMode: true, + transform: (tx, _, done) => { + if (accepted >= requestedLimit) { + return done(); + } + accepted += 1; + if (accepted >= requestedLimit) { + stopCursor(); + } + return done(null, tx); + } + })); + } + transactionStream = transactionStream.eventPipe(new TransformWithEventPipe({ + objectMode: true, + transform: (tx, _, done) => done(null, EVMTransactionStorage._apiTransform(tx)) + })); + + // Keep the established local API streaming lifecycle while allowing the + // normalization pipeline to close its underlying MongoDB cursor. + (transactionStream as any).close = closeCursor; + return Storage.apiStream(transactionStream as any, req!, res!); } else { try { const tokenTransfers = await this.getErc20Transfers(network, address, tokenAddress, args); @@ -595,10 +731,7 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai if (t.blockHeight !== undefined && t.blockHeight >= 0) { confirmations = tipHeight - t.blockHeight + 1; } - // Add effects to old db entries - if (!t.effects || (t.effects && t.effects.length == 0)) { - t.effects = EVMTransactionStorage.getEffects(t as IEVMTransactionInProcess); - } + t = this.populateEffects(t); const convertedTx = EVMTransactionStorage._apiTransform(t, { object: true }) as Partial; return JSON.stringify({ ...convertedTx, confirmations }); }); @@ -752,6 +885,9 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai transactionStream = cursor.pipe(new TransformWithEventPipe({ objectMode: true, passThrough: true })); transactionStream = transactionStream.eventPipe(populateEffects); // For old db entries + transactionStream = transactionStream.eventPipe( + this.filterTransactionsForAddresses(streamParams.walletAddresses) + ); if (params.args.tokenAddress) { const erc20Transform = new Erc20RelatedFilterTransform(params.args.tokenAddress); @@ -960,6 +1096,7 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai } const addressBatchLC = addressBatch.map(address => address.toLowerCase()); + const canonicalAddressBatch = Array.from(new Set(addressBatch.flatMap(address => this.addressVariants(address)))); await EVMTransactionStorage.collection.updateMany( { @@ -978,6 +1115,8 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai }, { chain, network, 'effects.to': { $in: addressBatch } }, { chain, network, 'effects.from': { $in: addressBatch } }, + { chain, network, 'erc20Effects.items.to': { $in: canonicalAddressBatch } }, + { chain, network, 'erc20Effects.items.from': { $in: canonicalAddressBatch } }, ] }, { $addToSet: { wallets: params.wallet._id } } diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts index e55b1e1926d..0c80945de7c 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts @@ -1,4 +1,4 @@ -import { Readable } from 'stream'; +import { Readable, Transform, type TransformCallback } from 'stream'; import { Utils, Web3 } from '@bitpay-labs/crypto-wallet-core'; import { ChainStateProvider } from '../../'; import { Config } from '../../../../services/config'; @@ -187,6 +187,24 @@ export class GnosisApi { ...transactionQuery, 'effects.contractAddress': eitherCase(tokenAddress), 'effects.to': eitherCase(normalizedMultisigContractAddress) + }, + { + ...transactionQuery, + 'erc20Effects.items': { + $elemMatch: { + contractAddress: eitherCase(tokenAddress), + from: eitherCase(normalizedMultisigContractAddress) + } + } + }, + { + ...transactionQuery, + 'erc20Effects.items': { + $elemMatch: { + contractAddress: eitherCase(tokenAddress), + to: eitherCase(normalizedMultisigContractAddress) + } + } } ] }; @@ -230,6 +248,20 @@ export class GnosisApi { res.on('close', cleanupCursor); transactionStream = cursor.pipe(populateEffects); // For old db entries + if (tokenAddress) { + const tokenAddressLower = tokenAddress.toLowerCase(); + const multisigAddressLower = normalizedMultisigContractAddress.toLowerCase(); + transactionStream = transactionStream.pipe(new Transform({ + objectMode: true, + transform(tx: any, _: BufferEncoding, done: TransformCallback) { + const relevant = tx.effects?.some((effect: any) => + effect.contractAddress?.toLowerCase() === tokenAddressLower && + [effect.from, effect.to].some((address: string | undefined) => address?.toLowerCase() === multisigAddressLower) + ); + return relevant ? done(null, tx) : done(); + } + })); + } if (multisigContractAddress) { const multisigTransform = new MultisigRelatedFilterTransform(normalizedMultisigContractAddress, tokenAddress); diff --git a/packages/bitcore-node/src/providers/chain-state/evm/erc20Effects.ts b/packages/bitcore-node/src/providers/chain-state/evm/erc20Effects.ts new file mode 100644 index 00000000000..94c9e50300f --- /dev/null +++ b/packages/bitcore-node/src/providers/chain-state/evm/erc20Effects.ts @@ -0,0 +1,526 @@ +import { Web3 } from '@bitpay-labs/crypto-wallet-core'; +import logger from '../../../logger'; +import type { IRpc } from './p2p/rpcs'; +import type { + CanonicalErc20Effect, + Effect, + Erc20Effects, + IEVMBlock, + IEVMTransaction +} from './types'; +import type { IEVMNetworkConfig } from '../../../types/Config'; + +export const ERC20_EFFECTS_VERSION = 1; +export const ERC20_TRANSFER_TOPIC = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'; + +export interface RpcEvmLog { + address: unknown; + blockHash: unknown; + blockNumber: unknown; + data: unknown; + logIndex: unknown; + removed?: unknown; + topics: unknown; + transactionHash: unknown; + transactionIndex?: unknown; +} + +export interface ParsedErc20BlockLogs { + itemsByTransaction: Map; + unsupportedTransferLogs: number; +} + +export interface Erc20MaterializationResult { + enabled: boolean; + logCount: number; + supportedTransferLogs: number; + unsupportedTransferLogs: number; + usedHeightFallback: boolean; +} + +const HEX_HASH = /^0x[0-9a-fA-F]{64}$/; +const HEX_ADDRESS = /^0x[0-9a-fA-F]{40}$/; +const HEX_WORD = /^0x[0-9a-fA-F]{64}$/; +const HEX_BYTES = /^0x(?:[0-9a-fA-F]{2})*$/; +const DECIMAL_UINT = /^(0|[1-9][0-9]*)$/; +const MAX_UINT256 = 2n ** 256n - 1n; + +function rpcErrorText(err: unknown) { + if (err instanceof Error) { + return err.message; + } + try { + return JSON.stringify(err); + } catch { + return String(err); + } +} + +function isUnsupportedBlockHashFilterError(err: unknown) { + const message = rpcErrorText(err).toLowerCase(); + const code = err && typeof err === 'object' ? (err as any).code : undefined; + const unsupportedWording = + message.includes('unsupported') || + message.includes('not support') || + message.includes('invalid argument') || + message.includes('invalid params') || + message.includes('unknown field') || + message.includes('cannot specify'); + return (message.includes('blockhash') && unsupportedWording) || code === -32602; +} + +function sendRpc(rpc: IRpc, data: { jsonrpc: string; method: string; params: any[]; id: number }) { + return rpc.send(data); +} + +function normalizeHash(value: unknown, fieldName: string) { + if (typeof value !== 'string' || !HEX_HASH.test(value)) { + throw new Error(`Invalid ${fieldName}`); + } + return value.toLowerCase(); +} + +function normalizeAddress(value: unknown, fieldName: string) { + if (typeof value !== 'string' || !HEX_ADDRESS.test(value)) { + throw new Error(`Invalid ${fieldName}`); + } + return Web3.utils.toChecksumAddress(value); +} + +function parseRpcQuantity(value: unknown, fieldName: string) { + let parsed: bigint; + if (typeof value === 'bigint') { + parsed = value; + } else if (typeof value === 'number' && Number.isSafeInteger(value)) { + parsed = BigInt(value); + } else if (typeof value === 'string' && (/^0x[0-9a-fA-F]+$/.test(value) || /^(0|[1-9][0-9]*)$/.test(value))) { + parsed = BigInt(value); + } else { + throw new Error(`Invalid ${fieldName}`); + } + if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new Error(`Invalid ${fieldName}`); + } + return Number(parsed); +} + +export function semanticHashEquals(left: unknown, right: unknown) { + return typeof left === 'string' && typeof right === 'string' && left.toLowerCase() === right.toLowerCase(); +} + +/** + * Pure parser/validator for a block-scoped log response. It performs no RPC, + * database, cache, or API work. + */ +export function parseErc20TransferLogs(params: { + blockHash: string; + blockHeight: number; + transactionHashes: string[]; + logs: unknown; +}): ParsedErc20BlockLogs { + const expectedBlockHash = normalizeHash(params.blockHash, 'prepared block hash'); + if (!Number.isSafeInteger(params.blockHeight) || params.blockHeight < 0) { + throw new Error('Invalid prepared block height'); + } + if (!Array.isArray(params.logs)) { + throw new Error('Malformed eth_getLogs response'); + } + + const orderedTransactionHashes = params.transactionHashes.map((txid, index) => + normalizeHash(txid, `transaction hash at index ${index}`) + ); + const transactionHashes = new Set(orderedTransactionHashes); + if (transactionHashes.size !== params.transactionHashes.length) { + throw new Error('Duplicate transaction hash in prepared block'); + } + + const seenLogIdentities = new Set(); + const itemsByTransaction = new Map(); + let unsupportedTransferLogs = 0; + + for (let i = 0; i < params.logs.length; i++) { + const log = params.logs[i] as RpcEvmLog; + if (!log || typeof log !== 'object') { + throw new Error(`Malformed log at response index ${i}`); + } + if (log.removed !== undefined && typeof log.removed !== 'boolean') { + throw new Error(`Invalid removed flag at response index ${i}`); + } + if (log.removed === true) { + throw new Error(`Removed log returned for prepared block at response index ${i}`); + } + + const blockHash = normalizeHash(log.blockHash, `log blockHash at response index ${i}`); + if (blockHash !== expectedBlockHash) { + throw new Error(`Log block hash mismatch at response index ${i}`); + } + const blockNumber = parseRpcQuantity(log.blockNumber, `log blockNumber at response index ${i}`); + if (blockNumber !== params.blockHeight) { + throw new Error(`Log block height mismatch at response index ${i}`); + } + + const transactionHash = normalizeHash(log.transactionHash, `log transactionHash at response index ${i}`); + if (!transactionHashes.has(transactionHash)) { + throw new Error(`Log transaction is not a member of the prepared block at response index ${i}`); + } + const logIndex = parseRpcQuantity(log.logIndex, `logIndex at response index ${i}`); + if (log.transactionIndex !== undefined) { + const transactionIndex = parseRpcQuantity(log.transactionIndex, `transactionIndex at response index ${i}`); + if (orderedTransactionHashes[transactionIndex] !== transactionHash) { + throw new Error(`Log transactionIndex does not match transactionHash at response index ${i}`); + } + } + + const identity = `${expectedBlockHash}:${logIndex}`; + if (seenLogIdentities.has(identity)) { + throw new Error(`Duplicate log identity ${identity}`); + } + seenLogIdentities.add(identity); + + const contractAddress = normalizeAddress(log.address, `log address at response index ${i}`); + if (typeof log.data !== 'string' || !HEX_BYTES.test(log.data)) { + throw new Error(`Malformed log data at response index ${i}`); + } + if (!Array.isArray(log.topics) || log.topics.some(topic => typeof topic !== 'string' || !HEX_WORD.test(topic))) { + throw new Error(`Malformed topics at response index ${i}`); + } + const topics = log.topics as string[]; + if (topics[0]?.toLowerCase() !== ERC20_TRANSFER_TOPIC) { + continue; + } + + // Four indexed topics are ERC-721-shaped. Other non-standard Transfer + // signatures are counted and ignored rather than reinterpreted as ERC-20. + if (topics.length !== 3) { + unsupportedTransferLogs++; + continue; + } + + if (!HEX_WORD.test(log.data)) { + unsupportedTransferLogs++; + continue; + } + + // Standard ABI encoding left-pads indexed addresses with 12 zero bytes. + if (!/^0x0{24}[0-9a-fA-F]{40}$/.test(topics[1]) || !/^0x0{24}[0-9a-fA-F]{40}$/.test(topics[2])) { + unsupportedTransferLogs++; + continue; + } + + const item: CanonicalErc20Effect = { + type: 'ERC20:transfer', + from: Web3.utils.toChecksumAddress(`0x${topics[1].slice(-40)}`), + to: Web3.utils.toChecksumAddress(`0x${topics[2].slice(-40)}`), + amount: BigInt(log.data).toString(), + contractAddress, + logIndex, + callStack: `log:${logIndex}` + }; + + const items = itemsByTransaction.get(transactionHash) || []; + items.push(item); + itemsByTransaction.set(transactionHash, items); + } + + for (const items of itemsByTransaction.values()) { + items.sort((left, right) => left.logIndex - right.logIndex); + } + + return { itemsByTransaction, unsupportedTransferLogs }; +} + +export function attachErc20EffectsToTransactions(params: { + block: Pick; + transactions: Array; + logs: unknown; +}) { + if (!Number.isSafeInteger(params.block.transactionCount) || params.block.transactionCount < 0) { + throw new Error('Invalid prepared block transaction count'); + } + if (params.transactions.length !== params.block.transactionCount) { + throw new Error( + `Prepared transaction count ${params.transactions.length} does not match block transaction count ${params.block.transactionCount}` + ); + } + const transactionHashes = params.transactions.map(tx => tx.txid); + for (let transactionIndex = 0; transactionIndex < params.transactions.length; transactionIndex++) { + const tx = params.transactions[transactionIndex]; + if (tx.blockHeight !== params.block.height || !semanticHashEquals(tx.blockHash, params.block.hash)) { + throw new Error(`Transaction ${tx.txid} does not match prepared block inclusion`); + } + const storedTransactionIndex = parseRpcQuantity( + tx.transactionIndex, + `prepared transactionIndex for ${tx.txid}` + ); + if (storedTransactionIndex !== transactionIndex) { + throw new Error(`Transaction ${tx.txid} is out of order in the prepared block`); + } + } + + const parsed = parseErc20TransferLogs({ + blockHash: params.block.hash, + blockHeight: params.block.height, + transactionHashes, + logs: params.logs + }); + + let supportedTransferLogs = 0; + for (const tx of params.transactions) { + const items = parsed.itemsByTransaction.get(tx.txid.toLowerCase()) || []; + supportedTransferLogs += items.length; + tx.erc20Effects = { + blockHash: params.block.hash, + version: ERC20_EFFECTS_VERSION, + items: items.map(item => ({ ...item })) + }; + } + + return { + supportedTransferLogs, + unsupportedTransferLogs: parsed.unsupportedTransferLogs + }; +} + +export async function fetchTransferLogsForBlock(rpc: IRpc, block: Pick) { + const requestBase = { + method: 'eth_getLogs', + jsonrpc: '2.0', + id: Date.now() + Math.round(Math.random() * 1000) + }; + + try { + const logs = await sendRpc(rpc, { + ...requestBase, + params: [{ blockHash: block.hash, topics: [ERC20_TRANSFER_TOPIC] }] + }); + if (logs !== undefined) { + return { logs, usedHeightFallback: false }; + } + // Some existing RPC adapters discard a JSON-RPC error object and resolve + // undefined. eth_getLogs never has a successful undefined result, so retry + // through the strictly validated exact-height path. + } catch (err) { + if (!isUnsupportedBlockHashFilterError(err)) { + throw err; + } + } + + const blockQuantity = `0x${block.height.toString(16)}`; + const logs = await sendRpc(rpc, { + ...requestBase, + id: requestBase.id + 1, + params: [{ fromBlock: blockQuantity, toBlock: blockQuantity, topics: [ERC20_TRANSFER_TOPIC] }] + }); + return { logs, usedHeightFallback: true }; +} + +export async function assertRpcBlockIdentity(rpc: IRpc, block: Pick) { + const currentBlock = await rpc.getBlock(block.height); + if (!currentBlock || !semanticHashEquals(currentBlock.hash, block.hash)) { + throw new Error(`RPC block identity changed before persistence at height ${block.height}`); + } +} + +export function isErc20MaterializationEnabled(config?: IEVMNetworkConfig) { + return config?.erc20Effects?.materializationEnabled === true; +} + +/** + * Runs after master conversion/tracing/effect derivation and immediately before + * the existing persistence call. The only mutation is transaction.erc20Effects. + */ +export async function prepareErc20EffectsForPersistence(params: { + rpc: IRpc; + config?: IEVMNetworkConfig; + block: IEVMBlock; + transactions: Array; +}): Promise { + if (!isErc20MaterializationEnabled(params.config)) { + return { + enabled: false, + logCount: 0, + supportedTransferLogs: 0, + unsupportedTransferLogs: 0, + usedHeightFallback: false + }; + } + + const fetched = await fetchTransferLogsForBlock(params.rpc, params.block); + const attached = attachErc20EffectsToTransactions({ + block: params.block, + transactions: params.transactions, + logs: fetched.logs + }); + + if (attached.unsupportedTransferLogs > 0) { + logger.warn('Ignored unsupported Transfer-shaped logs while materializing ERC-20 effects: %o', { + chain: params.block.chain, + network: params.block.network, + height: params.block.height, + blockHash: params.block.hash, + count: attached.unsupportedTransferLogs + }); + } + + // Height-range fallback is safe only if the exact local block identity is + // revalidated after all enrichment and immediately before persistence. + if (fetched.usedHeightFallback) { + await assertRpcBlockIdentity(params.rpc, params.block); + } + + return { + enabled: true, + logCount: Array.isArray(fetched.logs) ? fetched.logs.length : 0, + supportedTransferLogs: attached.supportedTransferLogs, + unsupportedTransferLogs: attached.unsupportedTransferLogs, + usedHeightFallback: fetched.usedHeightFallback + }; +} + +function isUint256Decimal(value: unknown) { + if (typeof value !== 'string' || !DECIMAL_UINT.test(value)) { + return false; + } + try { + return BigInt(value) <= MAX_UINT256; + } catch { + return false; + } +} + +function isCanonicalItem(value: unknown): value is CanonicalErc20Effect { + if (!value || typeof value !== 'object') { + return false; + } + const item = value as CanonicalErc20Effect; + return ( + item.type === 'ERC20:transfer' && + typeof item.from === 'string' && HEX_ADDRESS.test(item.from) && + typeof item.to === 'string' && HEX_ADDRESS.test(item.to) && + typeof item.contractAddress === 'string' && HEX_ADDRESS.test(item.contractAddress) && + isUint256Decimal(item.amount) && + Number.isSafeInteger(item.logIndex) && item.logIndex >= 0 && + item.callStack === `log:${item.logIndex}` + ); +} + +export function isValidErc20EffectsForTransaction(tx: Partial, value: unknown = tx.erc20Effects): value is Erc20Effects { + if (!value || typeof value !== 'object' || !Array.isArray((value as Erc20Effects).items)) { + return false; + } + const erc20Effects = value as Erc20Effects; + if ( + erc20Effects.version !== ERC20_EFFECTS_VERSION || + typeof erc20Effects.blockHash !== 'string' || + !HEX_HASH.test(erc20Effects.blockHash) || + !semanticHashEquals(erc20Effects.blockHash, tx.blockHash) || + !Number.isSafeInteger(tx.blockHeight) || + (tx.blockHeight as number) < 0 + ) { + return false; + } + + let previousLogIndex = -1; + for (const item of erc20Effects.items) { + if (!isCanonicalItem(item) || item.logIndex <= previousLogIndex) { + return false; + } + previousLogIndex = item.logIndex; + } + return true; +} + +export function strictReadActivationHeight(config?: IEVMNetworkConfig) { + const height = config?.erc20Effects?.strictReadActivationHeight; + if (height === undefined) { + return undefined; + } + if (!Number.isSafeInteger(height) || height < 0) { + throw new Error('erc20Effects.strictReadActivationHeight must be a non-negative safe integer'); + } + return height; +} + +export function getEffectiveEvmEffects( + tx: Partial, + config?: IEVMNetworkConfig, + legacyEffects: Effect[] = Array.isArray(tx.effects) ? tx.effects : [] +): Effect[] { + const confirmed = Number.isSafeInteger(tx.blockHeight) && (tx.blockHeight as number) >= 0; + if (!confirmed) { + return legacyEffects; + } + + const nonErc20 = legacyEffects.filter(effect => effect?.type !== 'ERC20:transfer'); + const canonical = tx.erc20Effects; + if (isValidErc20EffectsForTransaction(tx, canonical)) { + return nonErc20.concat(canonical.items.map(item => ({ ...item }))); + } + + const activationHeight = strictReadActivationHeight(config); + if (activationHeight !== undefined && (tx.blockHeight as number) >= activationHeight) { + return nonErc20; + } + return legacyEffects; +} + +export interface TransactionAddressRelevanceOptions { + includeInternalRecipients?: boolean; +} + +export function isTransactionRelevantToAddresses( + tx: Partial, + addresses: string[], + options: TransactionAddressRelevanceOptions = {} +) { + const addressSet = new Set(addresses.map(address => address.toLowerCase())); + const isRelevant = (address?: unknown) => typeof address === 'string' && addressSet.has(address.toLowerCase()); + if (isRelevant(tx.from) || isRelevant(tx.to)) { + return true; + } + if ( + options.includeInternalRecipients && + Array.isArray((tx as any).internal) && + (tx as any).internal.some(internalTx => isRelevant(internalTx?.action?.to)) + ) { + return true; + } + return (tx.effects || []).some(effect => isRelevant(effect.from) || isRelevant(effect.to)); +} + +export function getCanonicalErc20ParticipantAddresses(tx: Partial) { + const canonical = tx.erc20Effects; + if (!isValidErc20EffectsForTransaction(tx, canonical)) { + return []; + } + const addresses = new Set(); + for (const item of canonical.items) { + addresses.add(item.from); + addresses.add(item.to); + } + return Array.from(addresses); +} + +export function erc20EffectsEqual(left: Erc20Effects | undefined, right: Erc20Effects | undefined) { + if (!left || !right) { + return left === right; + } + if ( + left.version !== right.version || + !semanticHashEquals(left.blockHash, right.blockHash) || + left.items.length !== right.items.length + ) { + return false; + } + return left.items.every((item, index) => { + const other = right.items[index]; + return !!other && + item.type === other.type && + item.from === other.from && + item.to === other.to && + item.amount === other.amount && + item.contractAddress === other.contractAddress && + item.logIndex === other.logIndex && + item.callStack === other.callStack; + }); +} diff --git a/packages/bitcore-node/src/providers/chain-state/evm/models/transaction.ts b/packages/bitcore-node/src/providers/chain-state/evm/models/transaction.ts index d9bb55d9612..1f6894eee9e 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/models/transaction.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/models/transaction.ts @@ -16,12 +16,25 @@ import { ERC721Abi } from '../abi/erc721'; import { InvoiceAbi } from '../abi/invoice'; import { MultisendAbi } from '../abi/multisend'; import { MultisigAbi } from '../abi/multisig'; +import { + getCanonicalErc20ParticipantAddresses, + getEffectiveEvmEffects, + isValidErc20EffectsForTransaction +} from '../erc20Effects'; import type { IEVMNetworkConfig } from '../../../../types/Config'; import type { StreamingFindOptions } from '../../../../types/Query'; import type { TransformOptions } from '../../../../types/TransformOptions'; import type { EVMTransactionJSON, Effect, IAbiDecodeResponse, IAbiDecodedData, IEVMBlock, IEVMCachedAddress, IEVMTransaction, IEVMTransactionInProcess, ParsedAbiParams } from '../types'; import type { Web3Types } from '@bitpay-labs/crypto-wallet-core'; +interface EVMTransactionWriteOperation { + updateOne: { + filter: any; + update: any; + upsert: boolean; + forceServerObjectId?: boolean; + }; +} function requireUncached(module) { delete require.cache[require.resolve(module)]; @@ -112,6 +125,20 @@ export class EVMTransactionModel extends BaseTransaction { partialFilterExpression: { 'effects.from': { $exists: true } } } ); + this.collection.createIndex( + { chain: 1, network: 1, 'erc20Effects.items.to': 1, blockTimeNormalized: 1 }, + { + background: true, + partialFilterExpression: { 'erc20Effects.items.to': { $exists: true } } + } + ); + this.collection.createIndex( + { chain: 1, network: 1, 'erc20Effects.items.from': 1, blockTimeNormalized: 1 }, + { + background: true, + partialFilterExpression: { 'erc20Effects.items.from': { $exists: true } } + } + ); } async batchImport(params: { @@ -143,6 +170,9 @@ export class EVMTransactionModel extends BaseTransaction { if (params.initialSyncComplete) { await this.expireBalanceCache(txOps); + for (const tx of params.txs) { + await this.expireErc20BalanceCacheForTransaction({ chain: params.chain, network: params.network, tx }); + } } // Create events for mempool txs @@ -160,7 +190,8 @@ export class EVMTransactionModel extends BaseTransaction { } getAllTouchedAddresses(tx: Partial): { tos: IEVMCachedAddress[]; froms: IEVMCachedAddress[] } { - const { to, from, effects } = tx; + const { to, from } = tx; + const effects = getEffectiveEvmEffects(tx, undefined, Array.isArray(tx.effects) ? tx.effects : []); const toBatch = new Set(); const fromBatch = new Set(); const addToBatch = (batch: Set, obj: IEVMCachedAddress) => { @@ -190,19 +221,46 @@ export class EVMTransactionModel extends BaseTransaction { return { tos, froms }; } + async expireBalanceCacheForTransaction(params: { chain: string; network: string; tx: Partial }) { + const { chain, network, tx } = params; + const { tos, froms } = this.getAllTouchedAddresses(tx); + for (const payload of tos.concat(froms)) { + const lowerAddress = payload.address.toLowerCase(); + const cacheKey = payload.tokenAddress + ? `getBalanceForAddress-${chain}-${network}-${lowerAddress}-${payload.tokenAddress.toLowerCase()}` + : `getBalanceForAddress-${chain}-${network}-${lowerAddress}`; + await CacheStorage.expire(cacheKey); + } + } + + async expireErc20BalanceCacheForTransaction(params: { chain: string; network: string; tx: Partial }) { + const { chain, network, tx } = params; + const canonical = tx.erc20Effects; + if (!isValidErc20EffectsForTransaction(tx, canonical)) { + return; + } + const cacheKeys = new Set(); + for (const item of canonical.items) { + const tokenAddress = item.contractAddress.toLowerCase(); + for (const address of [item.from, item.to]) { + cacheKeys.add( + `getBalanceForAddress-${chain}-${network}-${address.toLowerCase()}-${tokenAddress}` + ); + } + } + for (const cacheKey of cacheKeys) { + await CacheStorage.expire(cacheKey); + } + } + async expireBalanceCache(txOps: Array) { for (const op of txOps) { const { chain, network } = op.updateOne.filter; - - const { tos, froms } = this.getAllTouchedAddresses(op.updateOne.update.$set); - const uniqueBatch = tos.concat(froms); - for (const payload of uniqueBatch) { - const lowerAddress = payload.address.toLowerCase(); - const cacheKey = payload.tokenAddress - ? `getBalanceForAddress-${chain}-${network}-${lowerAddress}-${payload.tokenAddress.toLowerCase()}` - : `getBalanceForAddress-${chain}-${network}-${lowerAddress}`; - await CacheStorage.expire(cacheKey); - } + await this.expireBalanceCacheForTransaction({ + chain, + network, + tx: op.updateOne.update.$set + }); } } @@ -218,37 +276,138 @@ export class EVMTransactionModel extends BaseTransaction { chain: string; network: string; mempoolTime?: Date; - }) { + }): Promise { const { blockTimeNormalized, chain, height, network, parentChain, forkHeight } = params; + const findStoredErc20Effects = async (txids: string[]) => new Map(txids.length + ? (await this.collection.find({ chain, network, txid: { $in: Array.from(new Set(txids)) }, erc20Effects: { $exists: true } }) + .project({ txid: 1, 'erc20Effects.version': 1 }).toArray()).map(tx => [tx.txid, tx]) + : []); + const findWallets = async (addresses: string[]) => uniqBy((addresses.length + ? await WalletAddressStorage.collection.find({ chain, network, address: { $in: Array.from(new Set(addresses)) } }).toArray() + : []).map(w => w.wallet), wallet => wallet.toHexString()); if (parentChain && forkHeight && height < forkHeight) { const parentTxs = await EVMTransactionStorage.collection .find({ blockHeight: height, chain: parentChain, network }) .toArray(); - return parentTxs.map(parentTx => { + const hasPreparedMaterialization = params.txs.some(tx => + isValidErc20EffectsForTransaction(tx, tx.erc20Effects) + ); + const toChildSnapshot = (parentTx: any) => { + const snapshot = { ...parentTx }; + for (const key of ['_id', 'wallets', 'chain', 'network', 'txid', 'erc20Effects']) delete snapshot[key]; + return { ...snapshot, chain, network }; + }; + if (!hasPreparedMaterialization) { + const findPreparedTx = (txid: string) => params.txs.find(tx => + typeof tx.txid === 'string' && tx.txid.toLowerCase() === txid.toLowerCase()); + const storedErc20Effects = await findStoredErc20Effects(parentTxs.flatMap(parentTx => { + const preparedTx = findPreparedTx(parentTx.txid); + return preparedTx ? [parentTx.txid, preparedTx.txid] : [parentTx.txid]; + })); + return Promise.all(parentTxs.map(async parentTx => { + const preparedTx = findPreparedTx(parentTx.txid); + const variants = preparedTx ? [preparedTx.txid, parentTx.txid] : [parentTx.txid]; + const storedChild = variants.map(txid => storedErc20Effects.get(txid)).find(Boolean); + if (!storedChild) { + const parentSnapshot = { ...parentTx }; + delete parentSnapshot.erc20Effects; + return { updateOne: { filter: { txid: parentTx.txid, chain, network }, + update: { $set: { ...parentSnapshot, wallets: new Array() } }, + upsert: true, forceServerObjectId: true } }; + } + const touched = preparedTx ? this.getAllTouchedAddresses(preparedTx) : { tos: [], froms: [] }; + const wallets = await findWallets([...touched.froms, ...touched.tos].map(({ address }) => address)); + const update: any = { $set: toChildSnapshot(parentTx) }; + if (wallets.length) update.$addToSet = { wallets: { $each: wallets } }; + return { updateOne: { filter: { _id: storedChild._id, txid: storedChild.txid, chain, network }, + update, upsert: false } }; + })); + } + + const normalizeTxid = (txid: unknown, source: string) => { + if (typeof txid !== 'string' || !txid.length) { + throw new Error(`Invalid ${source} transaction id in pre-fork persistence`); + } + return txid.toLowerCase(); + }; + const preparedByTxid = new Map(); + for (const tx of params.txs) { + if (!isValidErc20EffectsForTransaction(tx, tx.erc20Effects)) { + throw new Error(`Missing valid ERC-20 materialization for pre-fork transaction ${tx.txid}`); + } + const normalizedTxid = normalizeTxid(tx.txid, 'prepared child'); + if (preparedByTxid.has(normalizedTxid)) { + throw new Error(`Duplicate prepared child transaction ${tx.txid} in pre-fork persistence`); + } + preparedByTxid.set(normalizedTxid, tx); + } + if (parentTxs.length !== preparedByTxid.size) { + throw new Error('Pre-fork parent and prepared child transaction counts do not match'); + } + const seenParentTxids = new Set(); + const targetTxsByTxid = new Map(); + for (const parentTx of parentTxs) { + const normalizedTxid = normalizeTxid(parentTx.txid, 'parent'); + if (seenParentTxids.has(normalizedTxid)) { + throw new Error(`Duplicate parent transaction ${parentTx.txid} in pre-fork persistence`); + } + if (!preparedByTxid.has(normalizedTxid)) { + throw new Error(`No prepared child transaction matches pre-fork parent transaction ${parentTx.txid}`); + } + const preparedTx = preparedByTxid.get(normalizedTxid)!; + if (!isValidErc20EffectsForTransaction(parentTx, preparedTx.erc20Effects)) { + throw new Error(`Prepared ERC-20 materialization does not match pre-fork parent inclusion ${parentTx.txid}`); + } + targetTxsByTxid.set(parentTx.txid, preparedTx).set(preparedTx.txid, preparedTx); + seenParentTxids.add(normalizedTxid); + } + const storedErc20Effects = await findStoredErc20Effects(Array.from(targetTxsByTxid.keys())); + return Promise.all(parentTxs.map(async parentTx => { + const preparedTx = preparedByTxid.get(normalizeTxid(parentTx.txid, 'parent'))!; + const canonicalAddresses = getCanonicalErc20ParticipantAddresses(preparedTx); + const wallets = await findWallets(canonicalAddresses.flatMap(address => [address, address.toLowerCase()])); + const txidVariants = Array.from(new Set([parentTx.txid, preparedTx.txid])); + const update: any = { + $set: toChildSnapshot(parentTx), + $setOnInsert: { txid: preparedTx.txid } + }; + if (!txidVariants.some(txid => { + const storedVersion = storedErc20Effects.get(txid)?.erc20Effects?.version; + return typeof storedVersion === 'number' && storedVersion > preparedTx.erc20Effects!.version; + })) { + update.$set.erc20Effects = preparedTx.erc20Effects; + } + if (wallets.length) { + update.$addToSet = { wallets: { $each: wallets } }; + } else { + update.$setOnInsert.wallets = []; + } return { updateOne: { - filter: { txid: parentTx.txid, chain, network }, - update: { - $set: { - ...parentTx, - wallets: new Array() - } - }, + filter: { txid: { $in: txidVariants }, chain, network }, + update, upsert: true, forceServerObjectId: true } }; - }); + })); } else { + const materializedByTxid = new Map(params.txs.filter(tx => + height >= SpentHeightIndicators.minimum && isValidErc20EffectsForTransaction(tx) + ).map(tx => [tx.txid, tx])); + const storedErc20Effects = height >= SpentHeightIndicators.minimum + ? await findStoredErc20Effects(params.txs.map(tx => tx.txid)) + : new Map(); return Promise.all( - // Get all "to" and "from" addresses so we can add the any corresponding wallets params.txs.map(async (tx: IEVMTransactionInProcess) => { const { tos, froms } = this.getAllTouchedAddresses(tx); - const toAddresses = tos.map(a => a.address); - const fromAddresses = froms.map(a => a.address); + const hasValidCanonicalMaterialization = materializedByTxid.has(tx.txid); + const touchedAddresses = [...froms, ...tos].map(({ address }) => address); + const canonicalAddressVariants = hasValidCanonicalMaterialization ? getCanonicalErc20ParticipantAddresses(tx).map(address => address.toLowerCase()) : []; + const walletLookupAddresses = Array.from(new Set(touchedAddresses.concat(canonicalAddressVariants))); const walletsAddys = await WalletAddressStorage.collection - .find({ chain, network, address: { $in: [...fromAddresses, ...toAddresses] } }) + .find({ chain, network, address: { $in: walletLookupAddresses } }) .toArray(); const wallets = uniqBy( walletsAddys.map(w => w.wallet), @@ -260,16 +419,31 @@ export class EVMTransactionModel extends BaseTransaction { if ((Config.chainConfig({ chain, network }) as IEVMNetworkConfig).leanTransactionStorage) { leanTx = EVMTransactionStorage.toLeanTransaction(tx); } + const update: any = { + $set: { + ...leanTx, + blockTimeNormalized + } + }; + delete update.$set.wallets; + const storedVersion = storedErc20Effects.get(tx.txid)?.erc20Effects?.version; + if (!hasValidCanonicalMaterialization || + (typeof storedVersion === 'number' && storedVersion > tx.erc20Effects!.version)) { + delete update.$set.erc20Effects; + } + if (hasValidCanonicalMaterialization || storedErc20Effects.has(tx.txid)) { + if (wallets.length) { + update.$addToSet = { wallets: { $each: wallets } }; + } else { + update.$setOnInsert = { wallets: [] }; + } + } else { + update.$set.wallets = wallets; + } return { updateOne: { filter: { txid: tx.txid, chain, network }, - update: { - $set: { - ...leanTx, - blockTimeNormalized, - wallets - } - }, + update, upsert: true, forceServerObjectId: true } @@ -456,8 +630,17 @@ export class EVMTransactionModel extends BaseTransaction { * @param {IEVMTransactionInProcess} tx * @param {Array} addresses */ + getEffectiveEffects(tx: Partial): Effect[] { + const legacyEffects = tx.effects?.length ? tx.effects : this.getEffects(tx as IEVMTransactionInProcess); + let config: IEVMNetworkConfig | undefined; + if (tx.chain && tx.network) { + config = Config.chainConfig({ chain: tx.chain, network: tx.network }) as IEVMNetworkConfig; + } + return getEffectiveEvmEffects(tx, config, legacyEffects); + } + getEffectsForAddresses(tx: IEVMTransactionInProcess, addresses: Array): Effect[] { - const effects = tx.effects?.length ? tx.effects : this.getEffects(tx); + const effects = this.getEffectiveEffects(tx); const addySet = new Set(addresses.map(a => a.toLowerCase())); return effects.filter(effect => addySet.has(effect.to.toLowerCase()) || addySet.has(effect.from.toLowerCase())); } diff --git a/packages/bitcore-node/src/providers/chain-state/evm/p2p/EVMVerificationPeer.ts b/packages/bitcore-node/src/providers/chain-state/evm/p2p/EVMVerificationPeer.ts index 9a8376de4b8..2cf9ee444b9 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/p2p/EVMVerificationPeer.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/p2p/EVMVerificationPeer.ts @@ -1,6 +1,7 @@ import logger from '../../../../logger'; import { ITransaction } from '../../../../models/baseTransaction'; import { ErrorType, IVerificationPeer } from '../../../../services/verification'; +import { prepareErc20EffectsForPersistence } from '../erc20Effects'; import { EVMBlockStorage } from '../models/block'; import { EVMP2pWorker } from './p2p'; @@ -42,6 +43,13 @@ export class EVMVerificationPeer extends EVMP2pWorker implements IVerificationPe convertedBlock.nextBlockHash = nextBlock.hash; } + await prepareErc20EffectsForPersistence({ + rpc: this.rpc!, + config: this.chainConfig, + block: convertedBlock, + transactions: convertedTxs + }); + await this.blockModel.processBlock({ chain: this.chain, network: this.network, diff --git a/packages/bitcore-node/src/providers/chain-state/evm/p2p/p2p.ts b/packages/bitcore-node/src/providers/chain-state/evm/p2p/p2p.ts index c056baa06ee..2f1dbf11df4 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/p2p/p2p.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/p2p/p2p.ts @@ -7,6 +7,7 @@ import { StateStorage } from '../../../../models/state'; import { BaseP2PWorker } from '../../../../services/p2p'; import { wait } from '../../../../utils'; import { BaseEVMStateProvider } from '../api/csp'; +import { prepareErc20EffectsForPersistence } from '../erc20Effects'; import { EVMBlockModel, EVMBlockStorage } from '../models/block'; import { EVMTransactionModel, EVMTransactionStorage } from '../models/transaction'; import { type IRpc, Rpcs } from './rpcs'; @@ -212,6 +213,12 @@ export class EVMP2pWorker extends BaseP2PWorker { } async processBlock(block: IEVMBlock, transactions: IEVMTransactionInProcess[]): Promise { + await prepareErc20EffectsForPersistence({ + rpc: this.rpc!, + config: this.chainConfig, + block, + transactions + }); await this.blockModel.addBlock({ chain: this.chain, network: this.network, diff --git a/packages/bitcore-node/src/providers/chain-state/evm/p2p/syncWorker.ts b/packages/bitcore-node/src/providers/chain-state/evm/p2p/syncWorker.ts index 23660afd5cf..714a38b6052 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/p2p/syncWorker.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/p2p/syncWorker.ts @@ -4,6 +4,7 @@ import logger from '../../../../logger'; import { Config } from '../../../../services/config'; import { Storage } from '../../../../services/storage'; import { wait } from '../../../../utils'; +import { prepareErc20EffectsForPersistence } from '../erc20Effects'; import { EVMBlockStorage } from '../models/block'; import { EVMTransactionStorage } from '../models/transaction'; import { type IRpc, Rpcs } from './rpcs'; @@ -116,6 +117,12 @@ export class SyncWorker { } async processBlock(block: IEVMBlock, transactions: IEVMTransactionInProcess[]): Promise { + await prepareErc20EffectsForPersistence({ + rpc: this.rpc!, + config: this.chainConfig, + block, + transactions + }); await EVMBlockStorage.addBlock({ chain: this.chain, network: this.network, diff --git a/packages/bitcore-node/src/providers/chain-state/evm/types.ts b/packages/bitcore-node/src/providers/chain-state/evm/types.ts index e4e6e6d9d75..d846d40b671 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/types.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/types.ts @@ -161,6 +161,7 @@ export type IEVMTransaction = ITransaction & { error?: string; receipt?: TxReceipt; effects?: Effect[]; // Meant to replace abiType, internal, calls and data on stored txs + erc20Effects?: Erc20Effects; confirmations?: number; }; @@ -171,6 +172,20 @@ export interface Effect { type?: 'ERC20:transfer' | 'MULTISIG:submitTransaction' | 'MULTISIG:confirmTransaction'; // These are the only txs types we care about contractAddress?: string; callStack?: string; + logIndex?: number; +} + +export interface CanonicalErc20Effect extends Effect { + type: 'ERC20:transfer'; + contractAddress: string; + callStack: string; + logIndex: number; +} + +export interface Erc20Effects { + blockHash: string; + version: number; + items: CanonicalErc20Effect[]; } export type IEVMTransactionInProcess = IEVMTransaction & { diff --git a/packages/bitcore-node/src/types/Config.ts b/packages/bitcore-node/src/types/Config.ts index 0b3e012c7fe..07d540354c4 100644 --- a/packages/bitcore-node/src/types/Config.ts +++ b/packages/bitcore-node/src/types/Config.ts @@ -67,6 +67,10 @@ export interface IEVMNetworkConfig extends INetworkConfig { leanTransactionStorage?: boolean; // Removes data, abiType, internal and calls before saving a transaction to the databases needsL1Fee?: boolean; // Does this chain require a layer-1 fee to be added to a transaction (e.g. OP-stack chains)? indexedProviderRouting?: IMultiProviderConfig[]; // Per-network indexed API routing order; provider credentials live under config.externalProviders + erc20Effects?: { + materializationEnabled?: boolean; // Populate inclusion-bound canonical ERC-20 effects on confirmed block writes + strictReadActivationHeight?: number; // At/above this height, missing or stale canonical ERC-20 effects fail closed + }; } export interface IXrpNetworkConfig extends INetworkConfig { diff --git a/packages/bitcore-node/test/integration/models/transaction.test.ts b/packages/bitcore-node/test/integration/models/transaction.test.ts index 1800e77bd45..c19c96e4052 100644 --- a/packages/bitcore-node/test/integration/models/transaction.test.ts +++ b/packages/bitcore-node/test/integration/models/transaction.test.ts @@ -240,6 +240,344 @@ describe('Transaction Model', function() { const walletTxs = await EVMTransactionStorage.collection.find({ chain, network, wallets: wallet }).toArray(); expect(walletTxs.length).eq(1); }); + + describe('erc20Effects persistence ownership', () => { + const height = 300; + const blockHash = `0x${'12'.repeat(32)}`; + const from = `0x${'23'.repeat(20)}`; + const to = `0x${'34'.repeat(20)}`; + const canonicalFrom = `0x${'45'.repeat(20)}`; + const canonicalTo = `0x${'56'.repeat(20)}`; + const token = `0x${'67'.repeat(20)}`; + const time = new Date('2026-07-02T00:00:00.000Z'); + + function transaction(txid: string, overrides: Record = {}) { + return { + chain, + network, + txid, + blockHeight: height, + blockHash, + blockTime: time, + blockTimeNormalized: time, + fee: 1, + value: 0, + wallets: [], + gasLimit: 21000, + gasPrice: 1, + nonce: 1, + transactionIndex: 0, + to, + from, + data: '0x', + internal: [], + calls: [], + effects: [], + ...overrides + }; + } + + function materialization(version: number, amount: string) { + return { + blockHash, + version, + items: [{ + type: 'ERC20:transfer', + from: canonicalFrom, + to: canonicalTo, + amount, + contractAddress: token, + logIndex: 1, + callStack: 'log:1' + }] + }; + } + + it('preserves an already stored newer generation while updating fields and canonical wallets', async () => { + const txid = `0x${'78'.repeat(32)}`; + const canonicalWallet = new ObjectId(); + const version1 = materialization(1, '10'); + const version2 = materialization(2, '20'); + await WalletAddressStorage.collection.insertOne({ + chain, + network, + wallet: canonicalWallet, + address: canonicalTo, + processed: true + }); + await EVMTransactionStorage.collection.insertOne(transaction(txid, { erc20Effects: version2 }) as any); + + await EVMTransactionStorage.batchImport({ + txs: [transaction(txid, { fee: 99, erc20Effects: version1 }) as any], + height, + blockHash, + blockTime: time, + blockTimeNormalized: time, + chain, + network, + initialSyncComplete: false + }); + + const stored = await EVMTransactionStorage.collection.findOne({ chain, network, txid }); + expect(stored!.fee).to.equal(99); + expect(stored!.erc20Effects).to.deep.equal(version2); + expect(stored!.wallets.map(walletId => walletId.toString())).to.deep.equal([canonicalWallet.toString()]); + }); + + it('preserves canonical wallet ownership only on rows that already own erc20Effects', async () => { + const materializedTxid = `0x${'89'.repeat(32)}`; + const unmaterializedTxid = `0x${'9a'.repeat(32)}`; + const canonicalWallet = new ObjectId(); + const replacedWallet = new ObjectId(); + const masterWallet = new ObjectId(); + const storedMaterialization = materialization(1, '10'); + await WalletAddressStorage.collection.insertOne({ chain, network, wallet: masterWallet, address: from, processed: true }); + await EVMTransactionStorage.collection.insertMany([ + transaction(materializedTxid, { wallets: [canonicalWallet], erc20Effects: storedMaterialization }), + transaction(unmaterializedTxid, { wallets: [replacedWallet], transactionIndex: 1 }) + ] as any); + + await EVMTransactionStorage.batchImport({ + txs: [ + transaction(materializedTxid, { fee: 10, erc20Effects: undefined }) as any, + transaction(unmaterializedTxid, { fee: 20, transactionIndex: 1, erc20Effects: undefined }) as any + ], + height, + blockHash, + blockTime: time, + blockTimeNormalized: time, + chain, + network, + initialSyncComplete: false + }); + + const materialized = await EVMTransactionStorage.collection.findOne({ chain, network, txid: materializedTxid }); + const unmaterialized = await EVMTransactionStorage.collection.findOne({ chain, network, txid: unmaterializedTxid }); + expect(materialized!.erc20Effects).to.deep.equal(storedMaterialization); + expect(materialized!.wallets.map(walletId => walletId.toString()).sort()).to.deep.equal( + [canonicalWallet.toString(), masterWallet.toString()].sort() + ); + expect(unmaterialized).not.to.have.property('erc20Effects'); + expect(unmaterialized!.wallets.map(walletId => walletId.toString())).to.deep.equal([masterWallet.toString()]); + }); + }); + + describe('pre-fork materialized parent copies', () => { + const parentChain = 'ETH'; + const childChain = 'ARB'; + const height = 100; + const forkHeight = 200; + const childTxid = `0x${'ab'.repeat(32)}`; + const parentTxid = `0x${childTxid.slice(2).toUpperCase()}`; + const blockHash = `0x${'22'.repeat(32)}`; + const parentFrom = `0x${'33'.repeat(20)}`; + const parentTo = `0x${'44'.repeat(20)}`; + const canonicalFrom = `0x${'55'.repeat(20)}`; + const canonicalTo = `0x${'66'.repeat(20)}`; + const token = `0x${'77'.repeat(20)}`; + const time = new Date('2026-07-01T00:00:00.000Z'); + + const canonicalWallet = new ObjectId(); + + function parentTransaction(wallets: ObjectId[] = []) { + return { + chain: parentChain, + network, + txid: parentTxid, + blockHeight: height, + blockHash, + blockTime: time, + blockTimeNormalized: time, + fee: 1, + value: 0, + wallets, + gasLimit: 21000, + gasPrice: 1, + nonce: 1, + transactionIndex: 0, + to: parentTo, + from: parentFrom, + data: '0x', + internal: [], + calls: [], + effects: [] + }; + } + + function preparedChildTransaction() { + return { + ...parentTransaction(), + chain: childChain, + txid: childTxid, + erc20Effects: { + blockHash, + version: 1, + items: [{ + type: 'ERC20:transfer', + from: canonicalFrom, + to: canonicalTo, + amount: '10', + contractAddress: token, + logIndex: 1, + callStack: 'log:1' + }] + } + }; + } + + beforeEach(async () => { + await WalletAddressStorage.collection.insertOne({ + chain: childChain, + network, + wallet: canonicalWallet, + address: canonicalTo, + processed: true + }); + }); + + it('inserts a distinct child row without copying the parent Mongo identity or route', async () => { + const parentInsert = await EVMTransactionStorage.collection.insertOne(parentTransaction() as any); + const preparedTx = preparedChildTransaction(); + + await EVMTransactionStorage.batchImport({ + txs: [preparedTx as any], + height, + blockTime: time, + blockHash, + blockTimeNormalized: time, + parentChain, + forkHeight, + chain: childChain, + network, + initialSyncComplete: false + }); + + const parent = await EVMTransactionStorage.collection.findOne({ _id: parentInsert.insertedId }); + const child = await EVMTransactionStorage.collection.findOne({ txid: childTxid, chain: childChain, network }); + + expect(parent).to.exist; + expect(parent).not.to.have.property('erc20Effects'); + expect(child).to.exist; + expect(child!._id!.toString()).not.to.equal(parentInsert.insertedId.toString()); + expect(child!.chain).to.equal(childChain); + expect(child!.network).to.equal(network); + expect(child!.txid).to.equal(childTxid); + expect(child!.from).to.equal(parentFrom); + expect(child!.erc20Effects).to.deep.equal(preparedTx.erc20Effects); + expect(child!.wallets.map(walletId => walletId.toString())).to.deep.equal([canonicalWallet.toString()]); + }); + + + it('updates a materialized child from an omitted-field pre-fork write without replacing its identity', async () => { + const masterWallet = new ObjectId(); + await WalletAddressStorage.collection.insertOne({ + chain: childChain, + network, + wallet: masterWallet, + address: parentTo, + processed: true + }); + const parentInsert = await EVMTransactionStorage.collection.insertOne(parentTransaction() as any); + const preparedTx = preparedChildTransaction(); + await EVMTransactionStorage.batchImport({ + txs: [preparedTx as any], + height, + blockTime: time, + blockHash, + blockTimeNormalized: time, + parentChain, + forkHeight, + chain: childChain, + network, + initialSyncComplete: false + }); + const originalChild = await EVMTransactionStorage.collection.findOne({ txid: childTxid, chain: childChain, network }); + await EVMTransactionStorage.collection.updateOne({ _id: parentInsert.insertedId }, { $set: { fee: 9 } }); + const omittedTx: any = preparedChildTransaction(); + delete omittedTx.erc20Effects; + + await EVMTransactionStorage.batchImport({ + txs: [omittedTx], + height, + blockTime: time, + blockHash, + blockTimeNormalized: time, + parentChain, + forkHeight, + chain: childChain, + network, + initialSyncComplete: false + }); + + const children = await EVMTransactionStorage.collection.find({ + chain: childChain, + network, + txid: { $in: [parentTxid, childTxid] } + }).toArray(); + const child = children[0]; + expect(children).to.have.length(1); + expect(child!._id!.toString()).to.equal(originalChild!._id!.toString()); + expect(child!.chain).to.equal(childChain); + expect(child!.network).to.equal(network); + expect(child!.txid).to.equal(childTxid); + expect(child!.erc20Effects).to.deep.equal(originalChild!.erc20Effects); + expect(child!.wallets.map(walletId => walletId.toString()).sort()).to.deep.equal( + [canonicalWallet.toString(), masterWallet.toString()].sort() + ); + expect(child!.fee).to.equal(9); + }); + + it('preserves an existing child identity, wallet, and newer materialization while adding the canonical wallet', async () => { + const existingWallet = new ObjectId(); + const futureErc20Effects = { + ...preparedChildTransaction().erc20Effects, + version: 2, + items: [{ ...preparedChildTransaction().erc20Effects.items[0], amount: '20' }] + }; + const parentInsert = await EVMTransactionStorage.collection.insertOne(parentTransaction() as any); + const childInsert = await EVMTransactionStorage.collection.insertOne({ + ...parentTransaction([existingWallet]), + chain: childChain, + txid: childTxid, + erc20Effects: futureErc20Effects + } as any); + const preparedTx = preparedChildTransaction(); + + await EVMTransactionStorage.batchImport({ + txs: [preparedTx as any], + height, + blockTime: time, + blockHash, + blockTimeNormalized: time, + parentChain, + forkHeight, + chain: childChain, + network, + initialSyncComplete: false + }); + + const parent = await EVMTransactionStorage.collection.findOne({ _id: parentInsert.insertedId }); + const children = await EVMTransactionStorage.collection.find({ + chain: childChain, + network, + txid: { $in: [parentTxid, childTxid] } + }).toArray(); + const child = children[0]; + + expect(parent).to.exist; + expect(parent).not.to.have.property('erc20Effects'); + expect(children).to.have.length(1); + expect(child).to.exist; + expect(child!._id!.toString()).to.equal(childInsert.insertedId.toString()); + expect(child!.txid).to.equal(childTxid); + expect(child!.chain).to.equal(childChain); + expect(child!.network).to.equal(network); + expect(child!.erc20Effects).to.deep.equal(futureErc20Effects); + expect(child!.wallets.map(walletId => walletId.toString()).sort()).to.deep.equal( + [existingWallet.toString(), canonicalWallet.toString()].sort() + ); + }); + }); }); }); diff --git a/packages/bitcore-node/test/unit/providers/chain-state/evm/backfillErc20Effects.test.ts b/packages/bitcore-node/test/unit/providers/chain-state/evm/backfillErc20Effects.test.ts new file mode 100644 index 00000000000..1fc69402610 --- /dev/null +++ b/packages/bitcore-node/test/unit/providers/chain-state/evm/backfillErc20Effects.test.ts @@ -0,0 +1,398 @@ +import { ObjectId } from 'bson'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import { + buildBackfillTransactionFilter, + getBackfillDisposition, + materializeBackfillTransactions, + parseBackfillOptions, + processBlock, + updateBackfillTransaction, + validateLocalBlockTransactions +} from '../../../../../scripts/backfillErc20Effects'; +import { + ERC20_TRANSFER_TOPIC, + attachErc20EffectsToTransactions +} from '../../../../../src/providers/chain-state/evm/erc20Effects'; +import { EVMBlockStorage } from '../../../../../src/providers/chain-state/evm/models/block'; +import { EVMTransactionStorage } from '../../../../../src/providers/chain-state/evm/models/transaction'; +import type { Erc20Effects, IEVMTransaction } from '../../../../../src/providers/chain-state/evm/types'; +import type { BackfillOptions } from '../../../../../scripts/backfillErc20Effects'; +import type { MongoBound } from '../../../../../src/models/base'; + +type StoredEvmTransaction = MongoBound & Required, '_id'>>; + +const hash = (byte: string) => `0x${byte.repeat(32)}`; +const address = (byte: string) => `0x${byte.repeat(20)}`; +const BLOCK_HASH = hash('11'); +const OTHER_BLOCK_HASH = hash('22'); +const TXID = hash('33'); +const topicAddress = (value: string) => `0x${'0'.repeat(24)}${value.slice(2).toLowerCase()}`; +const uintWord = (value: bigint | number) => `0x${BigInt(value).toString(16).padStart(64, '0')}`; + +function erc20Effects(overrides: Partial = {}): Erc20Effects { + return { + blockHash: BLOCK_HASH, + version: 1, + items: [{ + type: 'ERC20:transfer', + from: address('aa'), + to: address('bb'), + amount: '10', + contractAddress: address('cc'), + logIndex: 2, + callStack: 'log:2' + }], + ...overrides + }; +} + +function tx(overrides: Record = {}): StoredEvmTransaction { + return { + _id: new ObjectId(), + chain: 'ETH', + network: 'mainnet', + txid: TXID, + blockHeight: 100, + blockHash: BLOCK_HASH, + blockTime: new Date('2026-07-01T00:00:00.000Z'), + blockTimeNormalized: new Date('2026-07-01T00:00:00.000Z'), + fee: 1, + value: 0, + wallets: [], + gasLimit: 21000, + gasPrice: 1, + nonce: 1, + transactionIndex: 0, + to: address('dd'), + from: address('ee'), + effects: [], + erc20Effects: erc20Effects(), + ...overrides + } as StoredEvmTransaction; +} + +function options(overrides: Partial = {}): BackfillOptions { + return { + chain: 'ETH', + network: 'mainnet', + startHeight: 100, + endHeight: 200, + dryRun: false, + concurrency: 4, + delayMs: 100, + forceCurrentVersion: false, + ...overrides + }; +} + +async function expectRejected(promise: Promise, expected: RegExp) { + try { + await promise; + expect.fail('Expected promise to reject'); + } catch (err: any) { + expect(err.message || String(err)).to.match(expected); + } +} + +describe('ERC-20 effects historical backfill', function() { + const sandbox = sinon.createSandbox(); + + afterEach(function() { + sandbox.restore(); + }); + + it('requires an explicit bounded range and validates throttle controls', function() { + expect(parseBackfillOptions([ + '--chain', 'eth', + '--network', 'MAINNET', + '--start-height', '100', + '--end-height', '110', + '--dry-run', + '--concurrency', '8', + '--delay-ms', '25', + '--force-current-version' + ])).to.deep.equal({ + chain: 'ETH', + network: 'mainnet', + startHeight: 100, + endHeight: 110, + dryRun: true, + concurrency: 8, + delayMs: 25, + forceCurrentVersion: true + }); + + expect(() => parseBackfillOptions([ + '--chain', 'ETH', + '--network', 'mainnet', + '--start-height', '110', + '--end-height', '100' + ])).to.throw(/end-height/i); + }); + + it('builds an inclusion-bound, observed-value, monotonic-version CAS filter', function() { + const row = tx(); + const observed = erc20Effects({ items: [] }); + const filter = buildBackfillTransactionFilter({ tx: row, observedErc20Effects: observed }); + + expect(filter).to.include({ + _id: row._id, + chain: 'ETH', + network: 'mainnet', + txid: TXID, + blockHeight: 100, + blockHash: BLOCK_HASH + }); + expect(filter.$and[0]).to.deep.equal({ + $or: [ + { 'erc20Effects.version': { $exists: false } }, + { 'erc20Effects.version': { $lte: 1 } } + ] + }); + expect(filter.$and[1]).to.deep.equal({ erc20Effects: observed }); + }); + + it('distinguishes current-version skip, force-current write, and newer-version protection', function() { + const row = tx(); + const current = erc20Effects(); + + expect(getBackfillDisposition({ + tx: row, + observedErc20Effects: current, + forceCurrentVersion: false + })).to.equal('skipped-current'); + + expect(getBackfillDisposition({ + tx: row, + observedErc20Effects: current, + forceCurrentVersion: true + })).to.equal('write'); + + expect(getBackfillDisposition({ + tx: row, + observedErc20Effects: erc20Effects({ version: 2 }), + forceCurrentVersion: true + })).to.equal('skipped-newer'); + }); + + it('rejects inconsistent local block/transaction state without attempting repair', function() { + const block = { hash: BLOCK_HASH, height: 100, transactionCount: 1 }; + expect(() => validateLocalBlockTransactions(block, [])).to.throw(/transaction count mismatch/i); + expect(() => validateLocalBlockTransactions(block, [tx({ blockHash: OTHER_BLOCK_HASH })])).to.throw(/inconsistent inclusion/i); + expect(() => validateLocalBlockTransactions( + { ...block, transactionCount: 2 }, + [tx(), tx({ _id: new ObjectId() })] + )).to.throw(/duplicate local transaction/i); + expect(() => validateLocalBlockTransactions( + { ...block, transactionCount: 2 }, + [tx(), tx({ _id: new ObjectId(), txid: hash('44'), transactionIndex: 0 })] + )).to.throw(/invalid transactionIndex/i); + }); + + it('aborts before transaction updates when RPC block membership differs from local inclusion', async function() { + const row = tx({ erc20Effects: undefined }); + const localBlock = { + chain: 'ETH', + network: 'mainnet', + hash: BLOCK_HASH, + height: 100, + transactionCount: 1, + processed: true + }; + const blockCursor = { + limit: sandbox.stub().returnsThis(), + toArray: sandbox.stub().resolves([localBlock]) + }; + sandbox.stub(EVMBlockStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(blockCursor) + } as any)); + + const transactionCursor = { + sort: sandbox.stub().returnsThis(), + toArray: sandbox.stub().resolves([row]) + }; + const updateOne = sandbox.stub(); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(transactionCursor), + updateOne + } as any)); + + const rpc = { + send: sandbox.stub().resolves([]), + getBlock: sandbox.stub().resolves({ + hash: BLOCK_HASH, + transactions: [{ hash: hash('44') }] + }) + } as any; + + await expectRejected( + processBlock(options({ startHeight: 100, endHeight: 100, delayMs: 0 }), rpc, 100), + /RPC transaction membership mismatch/i + ); + + expect(rpc.getBlock.calledOnce).to.equal(true); + expect(updateOne.called).to.equal(false); + }); + + it('uses the same materializer as the live path and emits authoritative empty results', function() { + const block = { hash: BLOCK_HASH, height: 100, transactionCount: 1 }; + const log = { + address: address('cc'), + blockHash: BLOCK_HASH, + blockNumber: '0x64', + data: uintWord(10), + logIndex: '0x2', + removed: false, + topics: [ERC20_TRANSFER_TOPIC, topicAddress(address('aa')), topicAddress(address('bb'))], + transactionHash: TXID, + transactionIndex: '0x0' + }; + const liveRow = tx({ erc20Effects: undefined }); + const backfillRow = tx({ _id: new ObjectId(), erc20Effects: undefined }); + + attachErc20EffectsToTransactions({ block, transactions: [liveRow], logs: [log] }); + materializeBackfillTransactions({ block, transactions: [backfillRow], logs: [log] }); + + expect(backfillRow.erc20Effects).to.deep.equal(liveRow.erc20Effects); + + const emptyRow = tx({ _id: new ObjectId(), erc20Effects: undefined }); + materializeBackfillTransactions({ block, transactions: [emptyRow], logs: [] }); + expect(emptyRow.erc20Effects).to.deep.equal({ blockHash: BLOCK_HASH, version: 1, items: [] }); + }); + + it('writes only erc20Effects plus wallet merge semantics and invalidates relevant caches after change', async function() { + const updateOne = sandbox.stub().resolves({ matchedCount: 1, modifiedCount: 1 }); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ updateOne } as any)); + const expire = sandbox.stub(EVMTransactionStorage, 'expireErc20BalanceCacheForTransaction').resolves(); + const row = tx(); + const wallet = new ObjectId(); + + const result = await updateBackfillTransaction({ + options: options(), + tx: row, + observedErc20Effects: undefined, + wallets: [wallet] + }); + + expect(result).to.equal('updated'); + expect(updateOne.calledOnce).to.equal(true); + expect(updateOne.firstCall.args[1]).to.deep.equal({ + $set: { erc20Effects: row.erc20Effects }, + $addToSet: { wallets: { $each: [wallet] } } + }); + expect(expire.calledOnce).to.equal(true); + }); + + it('is dry-run safe and does not issue a MongoDB update', async function() { + const updateOne = sandbox.stub(); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ updateOne } as any)); + + const result = await updateBackfillTransaction({ + options: options({ dryRun: true }), + tx: tx(), + observedErc20Effects: undefined, + wallets: [] + }); + + expect(result).to.equal('dry-run'); + expect(updateOne.called).to.equal(false); + }); + + it('skips a valid current row on rerun and never downgrades a newer parser version', async function() { + const updateOne = sandbox.stub(); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ updateOne } as any)); + const row = tx(); + + expect(await updateBackfillTransaction({ + options: options(), + tx: row, + observedErc20Effects: erc20Effects(), + wallets: [] + })).to.equal('skipped-current'); + + expect(await updateBackfillTransaction({ + options: options({ forceCurrentVersion: true }), + tx: row, + observedErc20Effects: erc20Effects({ version: 2 }), + wallets: [] + })).to.equal('skipped-newer'); + + expect(updateOne.called).to.equal(false); + }); + + it('accepts a same-version live-write race only when it converged to identical inclusion-bound output', async function() { + const row = tx(); + const updateOne = sandbox.stub().resolves({ matchedCount: 0, modifiedCount: 0 }); + const findOne = sandbox.stub().resolves({ ...row, erc20Effects: erc20Effects() }); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ updateOne, findOne } as any)); + + const result = await updateBackfillTransaction({ + options: options(), + tx: row, + observedErc20Effects: undefined, + wallets: [] + }); + + expect(result).to.equal('raced-converged'); + }); + + it('binds a post-convergence wallet merge to the exact current MongoDB snapshot', async function() { + const lowerBlockHash = hash('ab'); + const upperBlockHash = `0x${lowerBlockHash.slice(2).toUpperCase()}`; + const lowerTxid = hash('cd'); + const upperTxid = `0x${lowerTxid.slice(2).toUpperCase()}`; + const row = tx({ + txid: upperTxid, + blockHash: upperBlockHash, + erc20Effects: erc20Effects({ blockHash: upperBlockHash }) + }); + const wallet = new ObjectId(); + const updateOne = sandbox.stub(); + updateOne.onFirstCall().resolves({ matchedCount: 0, modifiedCount: 0 }); + updateOne.onSecondCall().resolves({ matchedCount: 1, modifiedCount: 1 }); + const current = { + ...row, + txid: lowerTxid, + blockHash: lowerBlockHash, + erc20Effects: erc20Effects({ blockHash: lowerBlockHash }) + }; + const findOne = sandbox.stub().resolves(current); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ updateOne, findOne } as any)); + + const result = await updateBackfillTransaction({ + options: options(), + tx: row, + observedErc20Effects: undefined, + wallets: [wallet] + }); + + expect(result).to.equal('raced-converged'); + expect(updateOne.secondCall.args[0]).to.deep.equal({ + _id: current._id, + chain: current.chain, + network: current.network, + txid: current.txid, + blockHeight: current.blockHeight, + blockHash: current.blockHash, + erc20Effects: current.erc20Effects + }); + expect(updateOne.secondCall.args[1]).to.deep.equal({ + $addToSet: { wallets: { $each: [wallet] } } + }); + }); + + it('fails safely when inclusion changes before publication', async function() { + const row = tx(); + const updateOne = sandbox.stub().resolves({ matchedCount: 0, modifiedCount: 0 }); + const findOne = sandbox.stub().resolves({ ...row, blockHash: OTHER_BLOCK_HASH, erc20Effects: erc20Effects({ blockHash: OTHER_BLOCK_HASH }) }); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ updateOne, findOne } as any)); + + await expectRejected(updateBackfillTransaction({ + options: options(), + tx: row, + observedErc20Effects: undefined, + wallets: [] + }), /CAS lost/i); + }); +}); diff --git a/packages/bitcore-node/test/unit/providers/chain-state/evm/erc20Effects.test.ts b/packages/bitcore-node/test/unit/providers/chain-state/evm/erc20Effects.test.ts new file mode 100644 index 00000000000..7046e52f642 --- /dev/null +++ b/packages/bitcore-node/test/unit/providers/chain-state/evm/erc20Effects.test.ts @@ -0,0 +1,1597 @@ +import { Web3 } from '@bitpay-labs/crypto-wallet-core'; +import { ObjectId } from 'bson'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import { Readable, Writable } from 'stream'; +import { CacheStorage } from '../../../../../src/models/cache'; +import { WalletAddressStorage } from '../../../../../src/models/walletAddress'; +import { ChainStateProvider } from '../../../../../src/providers/chain-state'; +import { BaseEVMStateProvider } from '../../../../../src/providers/chain-state/evm/api/csp'; +import { Gnosis } from '../../../../../src/providers/chain-state/evm/api/gnosis'; +import { PopulateEffectsForAddressTransform, PopulateEffectsTransform } from '../../../../../src/providers/chain-state/evm/api/populateEffectsTransform'; +import { Erc20RelatedFilterTransform } from '../../../../../src/providers/chain-state/evm/api/erc20Transform'; +import { + ERC20_EFFECTS_VERSION, + ERC20_TRANSFER_TOPIC, + attachErc20EffectsToTransactions, + getEffectiveEvmEffects, + parseErc20TransferLogs, + prepareErc20EffectsForPersistence +} from '../../../../../src/providers/chain-state/evm/erc20Effects'; +import { EVMTransactionStorage } from '../../../../../src/providers/chain-state/evm/models/transaction'; +import { EVMP2pWorker } from '../../../../../src/providers/chain-state/evm/p2p/p2p'; +import { Config } from '../../../../../src/services/config'; +import { Storage } from '../../../../../src/services/storage'; +import type { Effect, IEVMTransactionInProcess } from '../../../../../src/providers/chain-state/evm/types'; +import type { IEVMNetworkConfig } from '../../../../../src/types/Config'; + +const hash = (byte: string) => `0x${byte.repeat(32)}`; +const address = (byte: string) => `0x${byte.repeat(20)}`; +const topicAddress = (value: string) => `0x${'0'.repeat(24)}${value.slice(2).toLowerCase()}`; +const uintWord = (value: bigint | number) => `0x${BigInt(value).toString(16).padStart(64, '0')}`; + +const BLOCK_HASH = hash('11'); +const OTHER_BLOCK_HASH = hash('22'); +const TX1 = hash('33'); +const TX2 = hash('44'); +const FROM = Web3.utils.toChecksumAddress(address('aa')); +const TO = Web3.utils.toChecksumAddress(address('bb')); +const OTHER = Web3.utils.toChecksumAddress(address('cc')); +const TOKEN = Web3.utils.toChecksumAddress(address('dd')); +const OTHER_TOKEN = Web3.utils.toChecksumAddress(address('ee')); +const ZERO = Web3.utils.toChecksumAddress(address('00')); + +const nativeEffect: Effect = { from: FROM, to: TO, amount: '7', callStack: '0' }; +const heuristicEffect: Effect = { + type: 'ERC20:transfer', + from: FROM, + to: OTHER, + amount: '999', + contractAddress: OTHER_TOKEN, + callStack: 'legacy' +}; +const heuristicSafeTokenEffect: Effect = { + type: 'ERC20:transfer', + from: FROM, + to: TO, + amount: '999', + contractAddress: TOKEN, + callStack: 'legacy-safe' +}; + +function makeLog(overrides: Record = {}) { + return { + address: TOKEN, + blockHash: BLOCK_HASH, + blockNumber: '0x64', + data: uintWord(123), + logIndex: '0x1', + removed: false, + topics: [ERC20_TRANSFER_TOPIC, topicAddress(FROM), topicAddress(TO)], + transactionHash: TX1, + transactionIndex: '0x0', + ...overrides + }; +} + +function makeTx(overrides: Record = {}): IEVMTransactionInProcess { + const time = new Date('2026-07-01T00:00:00.000Z'); + return { + chain: 'ETH', + network: 'mainnet', + txid: TX1, + blockHeight: 100, + blockHash: BLOCK_HASH, + blockTime: time, + blockTimeNormalized: time, + fee: 1, + value: 0, + wallets: [], + gasLimit: 21000, + gasPrice: 1, + nonce: 1, + transactionIndex: 0, + to: TOKEN, + from: FROM, + data: '0x', + internal: [], + calls: [], + effects: [nativeEffect, heuristicEffect], + ...overrides + } as IEVMTransactionInProcess; +} + +function makeLocalTx(overrides: Record = {}) { + return Object.assign(makeTx(overrides), { _id: new ObjectId() }); +} + +function makeBlock(overrides: Record = {}) { + return { + chain: 'ETH', + network: 'mainnet', + hash: BLOCK_HASH, + height: 100, + transactionCount: 1, + ...overrides + } as any; +} + +function validCanonicalEffect(overrides: Record = {}) { + return { + type: 'ERC20:transfer' as const, + from: FROM, + to: TO, + amount: '123', + contractAddress: TOKEN, + logIndex: 1, + callStack: 'log:1', + ...overrides + }; +} + +function enableConfig(strictReadActivationHeight?: number): IEVMNetworkConfig { + return { + erc20Effects: { + materializationEnabled: true, + strictReadActivationHeight + } + } as IEVMNetworkConfig; +} + +async function expectRejected(promise: Promise, expected: RegExp) { + try { + await promise; + expect.fail('Expected promise to reject'); + } catch (err: any) { + expect(err.message || String(err)).to.match(expected); + } +} + +async function collectObjects(stream: NodeJS.ReadableStream) { + const rows: any[] = []; + await new Promise((resolve, reject) => { + stream.on('data', row => rows.push(row)); + stream.on('error', reject); + stream.on('end', resolve); + }); + return rows; +} + +function cursorWithRows(rows: any[]) { + const cursor: any = { toArray: sinon.stub().resolves(rows) }; + cursor.project = sinon.stub().returns(cursor); + return cursor; +} + +function makeRpc(sendImplementation: (request: any, callback: (err: any, response?: any) => void) => void) { + return { + web3: { eth: {} }, + send: (request: any) => new Promise((resolve, reject) => { + sendImplementation(request, (err, response) => { + if (err || response?.error) { + return reject(err || response.error); + } + resolve(response?.result); + }); + }), + getBlock: sinon.stub().resolves({ hash: BLOCK_HASH }) + } as any; +} + +describe('Canonical ERC-20 effects', function() { + const sandbox = sinon.createSandbox(); + + afterEach(function() { + sandbox.restore(); + }); + + describe('pure Transfer-log parser', function() { + it('parses one standard Transfer event into the Effect-compatible shape', function() { + const parsed = parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1], + logs: [makeLog()] + }); + + expect(parsed.unsupportedTransferLogs).to.equal(0); + expect(parsed.itemsByTransaction.get(TX1)).to.deep.equal([validCanonicalEffect()]); + }); + + it('preserves identical-looking sibling events and orders them by logIndex', function() { + const parsed = parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1], + logs: [ + makeLog({ logIndex: '0x8', data: uintWord(5) }), + makeLog({ logIndex: '0x2', data: uintWord(5) }) + ] + }); + + expect(parsed.itemsByTransaction.get(TX1)!.map(item => item.logIndex)).to.deep.equal([2, 8]); + expect(parsed.itemsByTransaction.get(TX1)!.map(item => item.callStack)).to.deep.equal(['log:2', 'log:8']); + }); + + it('retains standard zero-address mint and burn events', function() { + const parsed = parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1], + logs: [ + makeLog({ + logIndex: '0x1', + topics: [ERC20_TRANSFER_TOPIC, topicAddress(ZERO), topicAddress(TO)] + }), + makeLog({ + logIndex: '0x2', + topics: [ERC20_TRANSFER_TOPIC, topicAddress(FROM), topicAddress(ZERO)] + }) + ] + }); + + const items = parsed.itemsByTransaction.get(TX1)!; + expect(items[0].from).to.equal(ZERO); + expect(items[1].to).to.equal(ZERO); + }); + + it('ignores and counts ERC-721-shaped four-topic Transfer logs', function() { + const parsed = parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1], + logs: [makeLog({ topics: [ERC20_TRANSFER_TOPIC, topicAddress(FROM), topicAddress(TO), uintWord(7)], data: '0x' })] + }); + + expect(parsed.unsupportedTransferLogs).to.equal(1); + expect(parsed.itemsByTransaction.size).to.equal(0); + }); + + it('ignores unrelated logs and counts nonstandard Transfer-shaped encodings', function() { + const parsed = parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1], + logs: [ + makeLog({ logIndex: '0x1', topics: [hash('99')] }), + makeLog({ logIndex: '0x2', topics: [ERC20_TRANSFER_TOPIC, hash('12'), topicAddress(TO)] }), + makeLog({ logIndex: '0x3', data: '0x01' }) + ] + }); + + expect(parsed.unsupportedTransferLogs).to.equal(2); + expect(parsed.itemsByTransaction.size).to.equal(0); + }); + + it('rejects malformed or inclusion-inconsistent log responses', function() { + expect(() => parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1], + logs: [makeLog({ blockHash: OTHER_BLOCK_HASH })] + })).to.throw(/block hash mismatch/i); + + expect(() => parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1], + logs: [makeLog({ data: '0x0g' })] + })).to.throw(/log data/i); + }); + + it('rejects duplicate block log identity even when transaction hashes differ', function() { + expect(() => parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1, TX2], + logs: [ + makeLog({ transactionHash: TX1, transactionIndex: '0x0', logIndex: '0x3' }), + makeLog({ transactionHash: TX2, transactionIndex: '0x1', logIndex: '0x3' }) + ] + })).to.throw(/duplicate log identity/i); + }); + + it('rejects a transactionIndex that contradicts the returned transactionHash', function() { + expect(() => parseErc20TransferLogs({ + blockHash: BLOCK_HASH, + blockHeight: 100, + transactionHashes: [TX1, TX2], + logs: [makeLog({ transactionHash: TX2, transactionIndex: '0x0' })] + })).to.throw(/transactionIndex does not match transactionHash/i); + }); + }); + + describe('materialization and live confirmed-block path', function() { + it('attaches a result to every confirmed transaction, including authoritative empty and failed rows', function() { + const failed = makeTx({ txid: TX1, transactionIndex: 0, receipt: { status: false } }); + const indirect = makeTx({ txid: TX2, transactionIndex: 1, effects: [nativeEffect] }); + const result = attachErc20EffectsToTransactions({ + block: makeBlock({ transactionCount: 2 }), + transactions: [failed, indirect], + logs: [makeLog({ transactionHash: TX2, transactionIndex: '0x1', logIndex: '0x4' })] + }); + + expect(result.supportedTransferLogs).to.equal(1); + expect(failed.effects).to.deep.equal([nativeEffect, heuristicEffect]); + expect(failed.erc20Effects).to.deep.equal({ blockHash: BLOCK_HASH, version: 1, items: [] }); + expect(indirect.erc20Effects!.items[0]).to.include({ logIndex: 4, amount: '123' }); + }); + + it('produces deterministic identical output when rerun for live or backfill use', function() { + const left = makeTx(); + const right = makeTx(); + const logs = [makeLog({ logIndex: '0x9' }), makeLog({ logIndex: '0x2', data: uintWord(8) })]; + + attachErc20EffectsToTransactions({ block: makeBlock(), transactions: [left], logs }); + attachErc20EffectsToTransactions({ block: makeBlock(), transactions: [right], logs: [...logs].reverse() }); + + expect(left.erc20Effects).to.deep.equal(right.erc20Effects); + }); + + it('rejects an incomplete prepared transaction set before attaching results', function() { + expect(() => attachErc20EffectsToTransactions({ + block: makeBlock({ transactionCount: 2 }), + transactions: [makeTx()], + logs: [] + })).to.throw(/transaction count/i); + }); + + it('rejects prepared transactions whose stored indexes do not match block order', function() { + expect(() => attachErc20EffectsToTransactions({ + block: makeBlock({ transactionCount: 2 }), + transactions: [ + makeTx({ txid: TX1, transactionIndex: 1 }), + makeTx({ txid: TX2, transactionIndex: 0 }) + ], + logs: [] + })).to.throw(/out of order/i); + }); + + it('uses an exact blockHash log filter where supported', async function() { + const send = sandbox.spy((_request: any, callback: (err: any, response?: any) => void) => { + callback(null, { result: [makeLog()] }); + }); + const tx = makeTx(); + await prepareErc20EffectsForPersistence({ + rpc: makeRpc(send), + config: enableConfig(1000), + block: makeBlock(), + transactions: [tx] + }); + + expect(send.calledOnce).to.equal(true); + expect(send.firstCall.args[0].params[0]).to.deep.equal({ + blockHash: BLOCK_HASH, + topics: [ERC20_TRANSFER_TOPIC] + }); + expect(tx.erc20Effects!.items).to.have.length(1); + }); + + it('uses the validated height fallback when an existing RPC adapter resolves an unsupported query as undefined', async function() { + const send = sandbox.stub(); + send.onFirstCall().callsFake((_request, callback) => callback(null, {})); + send.onSecondCall().callsFake((_request, callback) => callback(null, { result: [] })); + const rpc = makeRpc(send); + + const result = await prepareErc20EffectsForPersistence({ + rpc, + config: enableConfig(), + block: makeBlock(), + transactions: [makeTx()] + }); + + expect(result.usedHeightFallback).to.equal(true); + expect(send.callCount).to.equal(2); + expect(rpc.getBlock.calledOnceWithExactly(100)).to.equal(true); + }); + + it('falls back to an exact height and revalidates block identity when blockHash filters are unsupported', async function() { + const send = sandbox.stub(); + send.onFirstCall().callsFake((_request, callback) => callback(null, { error: { message: 'blockHash filter unsupported' } })); + send.onSecondCall().callsFake((_request, callback) => callback(null, { result: [] })); + const rpc = makeRpc(send); + const tx = makeTx(); + + const result = await prepareErc20EffectsForPersistence({ + rpc, + config: enableConfig(), + block: makeBlock(), + transactions: [tx] + }); + + expect(result.usedHeightFallback).to.equal(true); + expect(send.secondCall.args[0].params[0]).to.deep.equal({ + fromBlock: '0x64', + toBlock: '0x64', + topics: [ERC20_TRANSFER_TOPIC] + }); + expect(rpc.getBlock.calledOnceWithExactly(100)).to.equal(true); + expect(tx.erc20Effects).to.deep.equal({ blockHash: BLOCK_HASH, version: 1, items: [] }); + }); + + it('preserves master behavior and performs no log RPC when the feature is disabled', async function() { + const send = sandbox.spy((_request: any, callback: (err: any, response?: any) => void) => callback(null, { result: [] })); + const tx = makeTx(); + const result = await prepareErc20EffectsForPersistence({ + rpc: makeRpc(send), + config: {} as IEVMNetworkConfig, + block: makeBlock(), + transactions: [tx] + }); + + expect(result.enabled).to.equal(false); + expect(send.called).to.equal(false); + expect(tx.erc20Effects).to.equal(undefined); + }); + + it('keeps pending transaction ingestion on the existing legacy path', function() { + const pending = makeTx({ blockHeight: -1, blockHash: undefined, erc20Effects: undefined }); + const worker = Object.create(EVMP2pWorker.prototype) as any; + worker.chain = 'ETH'; + worker.network = 'mainnet'; + worker.txModel = { + convertRawTx: sandbox.stub().returns(pending), + batchImport: sandbox.stub() + }; + + worker.processTransaction({ hash: TX1 } as any); + + expect(worker.txModel.batchImport.calledOnce).to.equal(true); + expect(worker.txModel.batchImport.firstCall.args[0]).to.include({ height: -1 }); + expect(worker.txModel.batchImport.firstCall.args[0].txs[0].erc20Effects).to.equal(undefined); + }); + + it('does not call the existing persistence boundary when log acquisition fails', async function() { + const worker = Object.create(EVMP2pWorker.prototype) as any; + worker.chain = 'ETH'; + worker.network = 'mainnet'; + worker.chainConfig = enableConfig(); + worker.initialSyncComplete = true; + worker.rpc = makeRpc((_request, callback) => callback(new Error('log acquisition failed'))); + worker.blockModel = { addBlock: sandbox.stub().resolves() }; + const tx = makeTx(); + + await expectRejected(worker.processBlock(makeBlock(), [tx]), /log acquisition failed/i); + expect(worker.blockModel.addBlock.called).to.equal(false); + expect(tx.erc20Effects).to.equal(undefined); + }); + + it('does not persist when returned logs or fallback block identity mismatch the prepared block', async function() { + const logMismatchWorker = Object.create(EVMP2pWorker.prototype) as any; + logMismatchWorker.chain = 'ETH'; + logMismatchWorker.network = 'mainnet'; + logMismatchWorker.chainConfig = enableConfig(); + logMismatchWorker.initialSyncComplete = true; + logMismatchWorker.rpc = makeRpc((_request, callback) => callback(null, { result: [makeLog({ blockHash: OTHER_BLOCK_HASH })] })); + logMismatchWorker.blockModel = { addBlock: sandbox.stub().resolves() }; + + await expectRejected(logMismatchWorker.processBlock(makeBlock(), [makeTx()]), /block hash mismatch/i); + expect(logMismatchWorker.blockModel.addBlock.called).to.equal(false); + + const fallbackSend = sandbox.stub(); + fallbackSend.onFirstCall().callsFake((_request, callback) => callback(null, { error: { message: 'blockHash unsupported' } })); + fallbackSend.onSecondCall().callsFake((_request, callback) => callback(null, { result: [] })); + const fallbackRpc = makeRpc(fallbackSend); + fallbackRpc.getBlock.resolves({ hash: OTHER_BLOCK_HASH }); + const identityWorker = Object.create(EVMP2pWorker.prototype) as any; + identityWorker.chain = 'ETH'; + identityWorker.network = 'mainnet'; + identityWorker.chainConfig = enableConfig(); + identityWorker.initialSyncComplete = true; + identityWorker.rpc = fallbackRpc; + identityWorker.blockModel = { addBlock: sandbox.stub().resolves() }; + + await expectRejected(identityWorker.processBlock(makeBlock(), [makeTx()]), /identity changed/i); + expect(identityWorker.blockModel.addBlock.called).to.equal(false); + }); + + it('attaches canonical effects before invoking the existing single persistence path', async function() { + const worker = Object.create(EVMP2pWorker.prototype) as any; + worker.chain = 'ETH'; + worker.network = 'mainnet'; + worker.chainConfig = enableConfig(); + worker.initialSyncComplete = true; + worker.rpc = makeRpc((_request, callback) => callback(null, { result: [makeLog()] })); + worker.blockModel = { addBlock: sandbox.stub().resolves() }; + const tx = makeTx(); + + await worker.processBlock(makeBlock(), [tx]); + + expect(worker.blockModel.addBlock.calledOnce).to.equal(true); + expect(worker.blockModel.addBlock.firstCall.args[0].transactions[0].erc20Effects.items).to.deep.equal([ + validCanonicalEffect() + ]); + }); + }); + + describe('shared read semantics', function() { + async function streamGnosisTokenRows(transactions: any[]) { + sandbox.stub(Config, 'chainConfig').returns(enableConfig(100)); + const cursor = Readable.from(transactions, { objectMode: true }) as any; + cursor.sort = sandbox.stub().returns(cursor); + cursor.addCursorFlag = sandbox.stub().returns(cursor); + cursor.close = sandbox.stub().resolves(); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursor) + } as any)); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + provider.getWalletTransactionQuery = sandbox.stub().returns({ + chain: 'ETH', network: 'mainnet', wallets: new ObjectId() + }); + sandbox.stub(ChainStateProvider, 'get').returns(provider); + + const chunks: string[] = []; + const req = new Readable({ read() {} }) as any; + const res = new Writable({ + write(chunk, _encoding, done) { + chunks.push(chunk.toString()); + done(); + } + }) as any; + const finished = new Promise((resolve, reject) => { + res.on('finish', resolve); + res.on('error', reject); + }); + + await Gnosis.streamGnosisWalletTransactions({ + chain: 'ETH', + network: 'mainnet', + multisigContractAddress: TO, + wallet: { _id: new ObjectId() }, + req, + res, + args: { tokenAddress: TOKEN } + } as any); + await finished; + return chunks.join('').split('\n').filter(Boolean).map(row => JSON.parse(row)); + } + + it('replaces only legacy ERC-20 entries and preserves native/internal entries', function() { + const tx = makeTx({ + erc20Effects: { + blockHash: BLOCK_HASH, + version: ERC20_EFFECTS_VERSION, + items: [validCanonicalEffect()] + } + }); + + expect(getEffectiveEvmEffects(tx, enableConfig(100))).to.deep.equal([nativeEffect, validCanonicalEffect()]); + }); + + it('treats valid empty items as authoritative zero', function() { + const tx = makeTx({ erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [] } }); + expect(getEffectiveEvmEffects(tx, enableConfig(100))).to.deep.equal([nativeEffect]); + }); + + it('rejects structurally impossible canonical amounts and fails closed after activation', function() { + const tx = makeTx({ + erc20Effects: { + blockHash: BLOCK_HASH, + version: 1, + items: [validCanonicalEffect({ amount: (2n ** 256n).toString() })] + } + }); + expect(getEffectiveEvmEffects(tx, enableConfig(100))).to.deep.equal([nativeEffect]); + }); + + it('uses historical fallback below activation and fails closed at or above activation', function() { + const stale = makeTx({ erc20Effects: { blockHash: OTHER_BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } }); + expect(getEffectiveEvmEffects(stale, enableConfig(101))).to.deep.equal([nativeEffect, heuristicEffect]); + expect(getEffectiveEvmEffects(stale, enableConfig(100))).to.deep.equal([nativeEffect]); + + const missing = makeTx({ erc20Effects: undefined }); + expect(getEffectiveEvmEffects(missing, enableConfig(100))).to.deep.equal([nativeEffect]); + + const inconsistentConfirmed = makeTx({ blockHash: undefined, erc20Effects: undefined }); + expect(getEffectiveEvmEffects(inconsistentConfirmed, enableConfig(100))).to.deep.equal([nativeEffect]); + }); + + it('retains master heuristic behavior for pending transactions', function() { + const pending = makeTx({ blockHeight: -1, blockHash: undefined, erc20Effects: undefined }); + expect(getEffectiveEvmEffects(pending, enableConfig(0))).to.deep.equal([nativeEffect, heuristicEffect]); + }); + + it('does not apply local strict-read normalization to external-provider rows', function() { + sandbox.stub(Config, 'chainConfig').returns(enableConfig(0)); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + const external = makeTx({ _id: 'external-provider-row', erc20Effects: undefined }); + provider.populateEffects(external); + const transformed = EVMTransactionStorage._apiTransform(external, { object: true }) as any; + expect(transformed.effects).to.deep.equal([nativeEffect, heuristicEffect]); + }); + + it('applies the same normalizer to individual and block-stream local reads', async function() { + sandbox.stub(Config, 'chainConfig').returns(enableConfig(100)); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + provider._getTransaction = sandbox.stub().resolves({ tipHeight: 110, found: makeLocalTx() }); + provider.populateReceipt = sandbox.stub().callsFake(async tx => tx); + + const individual = await provider.getTransaction({ chain: 'ETH', network: 'mainnet', txId: TX1 }); + expect(individual.effects).to.deep.equal([nativeEffect]); + expect(individual).not.to.have.property('erc20Effects'); + + let blockRow: any; + provider.getLocalTip = sandbox.stub().resolves({ height: 110 }); + sandbox.stub(Storage, 'apiStreamingFind').callsFake((_model, _query, _args, _req, _res, transform: any) => { + blockRow = JSON.parse(transform(makeLocalTx())); + return undefined as any; + }); + await provider.streamTransactions({ + chain: 'ETH', + network: 'mainnet', + args: { blockHeight: 100 }, + req: {} as any, + res: {} as any + }); + expect(blockRow.effects).to.deep.equal([nativeEffect]); + }); + + it('drops wallet-history rows selected only by discarded heuristic ERC-20 effects', async function() { + sandbox.stub(Config, 'chainConfig').returns(enableConfig(100)); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + const requestedAddress = OTHER; + const cursor = Readable.from([makeLocalTx({ from: FROM, to: TOKEN })], { objectMode: true }) as any; + cursor.sort = sandbox.stub().returns(cursor); + cursor.addCursorFlag = sandbox.stub().returns(cursor); + cursor.close = sandbox.stub().resolves(); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursor) + } as any)); + const req = new Readable({ read() {} }) as any; + const res = new Writable({ objectMode: true, write(_chunk, _encoding, done) { done(); } }) as any; + const populate = new PopulateEffectsForAddressTransform(provider, [requestedAddress]); + + const stream = await provider._buildWalletTransactionsStream({ + chain: 'ETH', + network: 'mainnet', + wallet: { _id: new ObjectId() }, + args: {}, + req, + res + }, { + transactionStream: new PopulateEffectsTransform(provider), + populateEffects: populate, + walletAddresses: [requestedAddress] + }); + + const rows = await collectObjects(stream); + expect(rows).to.deep.equal([]); + }); + + it('preserves arbitrary-address rows selected through legacy internal.action.to', async function() { + sandbox.stub(Config, 'chainConfig').returns({ + erc20Effects: { materializationEnabled: false } + } as IEVMNetworkConfig); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + const internalOnly = makeLocalTx({ + effects: [], + erc20Effects: undefined, + internal: [{ + action: { from: TOKEN, to: OTHER, value: '0' }, + abiType: undefined, + traceAddress: [0] + }] + }); + let capturedQuery: any; + const cursor = Readable.from([internalOnly], { objectMode: true }) as any; + cursor.close = sandbox.stub().callsFake(() => cursor.destroy()); + sandbox.stub(EVMTransactionStorage, 'getTransactions').callsFake(params => { + capturedQuery = params.query; + return cursor; + }); + + const chunks: string[] = []; + const req = new Readable({ read() {} }) as any; + const res = new Writable({ + write(chunk, _encoding, done) { + chunks.push(chunk.toString()); + done(); + } + }) as any; + res.type = sandbox.stub().returns(res); + const finished = new Promise((resolve, reject) => { + res.on('finish', resolve); + res.on('error', reject); + }); + + await provider._buildAddressTransactionsStream({ + chain: 'ETH', + network: 'mainnet', + address: OTHER, + args: {}, + req, + res + }); + await finished; + + const output = JSON.parse(chunks.join('')); + expect(output).to.have.length(1); + expect(output[0].txid).to.equal(TX1); + expect(output[0].effects).to.deep.equal([]); + expect(capturedQuery.$or).to.deep.include({ + chain: 'ETH', + network: 'mainnet', + 'internal.action.to': OTHER + }); + }); + + it('does not synthesize missing legacy effects for arbitrary-address history', async function() { + sandbox.stub(Config, 'chainConfig').returns({ + erc20Effects: { materializationEnabled: false } + } as IEVMNetworkConfig); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + const storedEmptyEffects = makeLocalTx({ + to: OTHER, + effects: [], + erc20Effects: undefined, + internal: [{ + action: { from: TOKEN, to: TO, value: '1' }, + abiType: undefined, + traceAddress: [0] + }] + }); + expect(EVMTransactionStorage.getEffects(storedEmptyEffects)).to.have.length(1); + + const cursor = Readable.from([storedEmptyEffects], { objectMode: true }) as any; + cursor.close = sandbox.stub().callsFake(() => cursor.destroy()); + sandbox.stub(EVMTransactionStorage, 'getTransactions').returns(cursor); + + const chunks: string[] = []; + const req = new Readable({ read() {} }) as any; + const res = new Writable({ + write(chunk, _encoding, done) { + chunks.push(chunk.toString()); + done(); + } + }) as any; + res.type = sandbox.stub().returns(res); + const finished = new Promise((resolve, reject) => { + res.on('finish', resolve); + res.on('error', reject); + }); + + await provider._buildAddressTransactionsStream({ + chain: 'ETH', + network: 'mainnet', + address: OTHER, + args: {}, + req, + res + }); + await finished; + + const output = JSON.parse(chunks.join('')); + expect(output).to.have.length(1); + expect(output[0].txid).to.equal(TX1); + expect(output[0].effects).to.deep.equal([]); + }); + + it('applies arbitrary-address limits after canonical normalization and relevance filtering', async function() { + sandbox.stub(Config, 'chainConfig').returns(enableConfig(100)); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + const canonicalTx = makeLocalTx({ + erc20Effects: { + blockHash: BLOCK_HASH, + version: 1, + items: [validCanonicalEffect({ to: OTHER })] + } + }); + const staleHeuristicOnly = makeLocalTx({ + txid: TX2, + from: FROM, + to: TOKEN, + effects: [heuristicEffect], + erc20Effects: undefined + }); + const secondCanonicalTx = makeLocalTx({ + txid: hash('55'), + erc20Effects: { + blockHash: BLOCK_HASH, + version: 1, + items: [validCanonicalEffect({ to: OTHER, logIndex: 2, callStack: 'log:2' })] + } + }); + const sourceRows = [staleHeuristicOnly, canonicalTx, secondCanonicalTx]; + let cursor: any; + let capturedQuery: any; + let capturedOptions: any; + sandbox.stub(EVMTransactionStorage, 'getTransactions').callsFake(params => { + capturedQuery = params.query; + capturedOptions = params.options; + const databaseLimit = Number(params.options.limit); + const returnedRows = databaseLimit > 0 ? sourceRows.slice(0, databaseLimit) : sourceRows; + cursor = Readable.from(returnedRows, { objectMode: true }) as any; + cursor.close = sandbox.stub().callsFake(() => cursor.destroy()); + return cursor; + }); + + const chunks: string[] = []; + const req = new Readable({ read() {} }) as any; + const res = new Writable({ + write(chunk, _encoding, done) { + chunks.push(chunk.toString()); + done(); + } + }) as any; + res.type = sandbox.stub().returns(res); + const finished = new Promise((resolve, reject) => { + res.on('finish', resolve); + res.on('error', reject); + }); + + await provider._buildAddressTransactionsStream({ + chain: 'ETH', + network: 'mainnet', + address: OTHER.toLowerCase(), + args: { limit: 1 }, + req, + res + }); + await finished; + + const output = JSON.parse(chunks.join('')); + expect(output).to.have.length(1); + expect(output[0].txid).to.equal(TX1); + expect(output[0].effects).to.deep.equal([nativeEffect, validCanonicalEffect({ to: OTHER })]); + expect(capturedOptions).not.to.have.property('limit'); + expect(cursor.close.calledOnce).to.equal(true); + expect(capturedQuery.$or).to.deep.include({ + chain: 'ETH', + network: 'mainnet', + 'erc20Effects.items.to': { $in: [OTHER.toLowerCase(), OTHER] } + }); + }); + + it('preserves the magnitude and cursor-closing behavior of a negative address limit', async function() { + sandbox.stub(Config, 'chainConfig').returns(enableConfig(100)); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + const first = makeLocalTx({ + erc20Effects: { + blockHash: BLOCK_HASH, + version: 1, + items: [validCanonicalEffect({ to: OTHER })] + } + }); + const second = makeLocalTx({ + txid: TX2, + transactionIndex: 1, + erc20Effects: { + blockHash: BLOCK_HASH, + version: 1, + items: [validCanonicalEffect({ to: OTHER, logIndex: 2, callStack: 'log:2' })] + } + }); + let cursor: any; + let capturedOptions: any; + sandbox.stub(EVMTransactionStorage, 'getTransactions').callsFake(params => { + capturedOptions = params.options; + cursor = Readable.from([first, second], { objectMode: true }) as any; + cursor.close = sandbox.stub().callsFake(() => cursor.destroy()); + return cursor; + }); + + const chunks: string[] = []; + const req = new Readable({ read() {} }) as any; + const res = new Writable({ + write(chunk, _encoding, done) { + chunks.push(chunk.toString()); + done(); + } + }) as any; + res.type = sandbox.stub().returns(res); + const finished = new Promise((resolve, reject) => { + res.on('finish', resolve); + res.on('error', reject); + }); + + await provider._buildAddressTransactionsStream({ + chain: 'ETH', + network: 'mainnet', + address: OTHER, + args: { limit: -1 }, + req, + res + }); + await finished; + + const output = JSON.parse(chunks.join('')); + expect(output).to.have.length(1); + expect(output[0].txid).to.equal(TX1); + expect(capturedOptions).not.to.have.property('limit'); + expect(cursor.close.calledOnce).to.equal(true); + }); + + it('rejects exceptional numeric address limits instead of silently streaming without a limit', async function() { + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + const getTransactions = sandbox.stub(EVMTransactionStorage, 'getTransactions'); + + await expectRejected(provider._buildAddressTransactionsStream({ + chain: 'ETH', + network: 'mainnet', + address: OTHER, + args: { limit: '1.5' }, + req: {} as any, + res: {} as any + }), /finite safe integer/i); + + expect(getTransactions.called).to.equal(false); + }); + + it('feeds canonical items into local token-history transforms', async function() { + sandbox.stub(Config, 'chainConfig').returns(enableConfig(100)); + const provider = Object.create(BaseEVMStateProvider.prototype) as any; + const populate = new PopulateEffectsTransform(provider); + const tokenFilter = new Erc20RelatedFilterTransform(TOKEN); + populate.pipe(tokenFilter); + const rowsPromise = collectObjects(tokenFilter); + populate.end(makeLocalTx({ + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + })); + const rows = await rowsPromise; + expect(rows).to.have.length(1); + expect(rows[0]).to.include({ from: FROM, to: TO, value: 123, callStack: 'log:1' }); + }); + + it('extends Gnosis local token-history selection to canonical-only items', async function() { + const cursor = Readable.from([], { objectMode: true }) as any; + cursor.sort = sandbox.stub().returns(cursor); + cursor.addCursorFlag = sandbox.stub().returns(cursor); + cursor.close = sandbox.stub().resolves(); + let capturedQuery: any; + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().callsFake(query => { + capturedQuery = query; + return cursor; + }) + } as any)); + sandbox.stub(ChainStateProvider, 'get').returns({ + getWalletTransactionQuery: sandbox.stub().returns({ chain: 'ETH', network: 'mainnet', wallets: new ObjectId() }), + populateEffects: sandbox.stub().callsFake(tx => tx), + populateReceipt: sandbox.stub().callsFake(tx => tx) + } as any); + const req = new Readable({ read() {} }) as any; + const res = new Writable({ write(_chunk, _encoding, done) { done(); } }) as any; + + await Gnosis.streamGnosisWalletTransactions({ + chain: 'ETH', + network: 'mainnet', + multisigContractAddress: TO, + wallet: { _id: new ObjectId() }, + req, + res, + args: { tokenAddress: TOKEN } + } as any); + + expect(capturedQuery.$or).to.deep.include({ + chain: 'ETH', + network: 'mainnet', + 'erc20Effects.items': { + $elemMatch: { + contractAddress: { $in: [TOKEN, TOKEN.toLowerCase()] }, + to: { $in: [TO, TO.toLowerCase()] } + } + } + }); + }); + + it('drops a Gnosis token row when an authoritative empty result removes its only token relationship', async function() { + const rows = await streamGnosisTokenRows([makeLocalTx({ + to: TO, + effects: [heuristicSafeTokenEffect], + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [] }, + receipt: { status: true } + })]); + + expect(rows).to.deep.equal([]); + }); + + it('drops a Gnosis token row when a stale materialization leaves no valid normalized token relationship', async function() { + const rows = await streamGnosisTokenRows([makeLocalTx({ + to: TO, + effects: [heuristicSafeTokenEffect], + erc20Effects: { + blockHash: OTHER_BLOCK_HASH, + version: 1, + items: [validCanonicalEffect()] + }, + receipt: { status: true } + })]); + + expect(rows).to.deep.equal([]); + }); + + it('emits a valid canonical Gnosis token effect after normalization', async function() { + const rows = await streamGnosisTokenRows([makeLocalTx({ + to: TO, + effects: [heuristicSafeTokenEffect], + erc20Effects: { + blockHash: BLOCK_HASH, + version: 1, + items: [validCanonicalEffect()] + }, + receipt: { status: true } + })]); + + expect(rows).to.have.length(1); + expect(rows[0]).to.include({ category: 'receive', satoshis: '123' }); + expect(rows[0].effects).to.deep.equal([validCanonicalEffect()]); + }); + + it('emits only Safe-related canonical siblings in Gnosis token history', async function() { + const safeReceive = validCanonicalEffect({ amount: '5' }); + const safeSend = validCanonicalEffect({ + from: TO, + to: OTHER, + amount: '7', + logIndex: 2, + callStack: 'log:2' + }); + const unrelated = validCanonicalEffect({ + from: FROM, + to: OTHER, + amount: '9', + logIndex: 3, + callStack: 'log:3' + }); + const rows = await streamGnosisTokenRows([makeLocalTx({ + to: TO, + effects: [heuristicSafeTokenEffect], + erc20Effects: { + blockHash: BLOCK_HASH, + version: 1, + items: [safeReceive, safeSend, unrelated] + }, + receipt: { status: true } + })]); + + expect(rows).to.have.length(2); + expect(rows.map(row => row.effects[0].logIndex)).to.deep.equal([1, 2]); + expect(rows.map(row => row.category)).to.deep.equal(['receive', 'send']); + expect(rows.map(row => row.satoshis)).to.deep.equal(['5', -7]); + }); + }); + + describe('wallet association and balance-cache integration', function() { + it('overlays prepared materialization and merges canonical wallets in the pre-fork parent-copy path', async function() { + const childTxid = hash('ab'); + const parentTxid = `0x${childTxid.slice(2).toUpperCase()}`; + const parentWallet = new ObjectId(); + const canonicalWallet = new ObjectId(); + const parentTx = makeLocalTx({ + chain: 'ETH', + txid: parentTxid, + from: OTHER, + to: OTHER, + wallets: [parentWallet] + }); + const parentFind = sandbox.stub(); + parentFind.onFirstCall().returns(cursorWithRows([parentTx])); + parentFind.onSecondCall().returns(cursorWithRows([makeLocalTx({ + chain: 'ARB', + txid: childTxid, + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + })])); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ find: parentFind } as any)); + const walletFind = sandbox.stub().returns({ + toArray: sandbox.stub().resolves([{ address: TO.toLowerCase(), wallet: canonicalWallet }]) + }); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ find: walletFind } as any)); + const preparedTx = makeTx({ + chain: 'ARB', + txid: childTxid, + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + }); + + const [operation] = await EVMTransactionStorage.addTransactions({ + txs: [preparedTx], + height: 100, + chain: 'ARB', + network: 'mainnet', + parentChain: 'ETH', + forkHeight: 200, + initialSyncComplete: true + }); + + expect(parentFind.firstCall.args[0]).to.deep.equal({ blockHeight: 100, chain: 'ETH', network: 'mainnet' }); + expect(walletFind.firstCall.args[0]).to.include({ chain: 'ARB', network: 'mainnet' }); + expect(walletFind.firstCall.args[0].address.$in).to.include(TO.toLowerCase()); + expect(operation.updateOne.filter).to.deep.equal({ + txid: { $in: [parentTxid, childTxid] }, + chain: 'ARB', + network: 'mainnet' + }); + expect(operation.updateOne.update.$set.from).to.equal(OTHER); + expect(operation.updateOne.update.$set).not.to.have.property('_id'); + expect(operation.updateOne.update.$set).not.to.have.property('txid'); + expect(operation.updateOne.update.$set.chain).to.equal('ARB'); + expect(operation.updateOne.update.$set.network).to.equal('mainnet'); + expect(operation.updateOne.update.$set.erc20Effects).to.deep.equal(preparedTx.erc20Effects); + expect(operation.updateOne.update.$set).not.to.have.property('wallets'); + expect(operation.updateOne.update.$addToSet.wallets.$each).to.deep.equal([canonicalWallet]); + expect(operation.updateOne.update.$setOnInsert).to.deep.equal({ txid: childTxid }); + }); + + it('uses insert-only empty wallets for a pre-fork authoritative empty materialization', async function() { + const parentTx = makeLocalTx({ chain: 'ETH', wallets: [new ObjectId()] }); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursorWithRows([parentTx])) + } as any)); + const walletFind = sandbox.stub(); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ find: walletFind } as any)); + const preparedTx = makeTx({ + chain: 'ARB', + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [] } + }); + + const [operation] = await EVMTransactionStorage.addTransactions({ + txs: [preparedTx], + height: 100, + chain: 'ARB', + network: 'mainnet', + parentChain: 'ETH', + forkHeight: 200, + initialSyncComplete: true + }); + + expect(walletFind.called).to.equal(false); + expect(operation.updateOne.update.$set.erc20Effects).to.deep.equal(preparedTx.erc20Effects); + expect(operation.updateOne.update.$set).not.to.have.property('wallets'); + expect(operation.updateOne.update.$setOnInsert).to.deep.equal({ txid: preparedTx.txid, wallets: [] }); + expect(operation.updateOne.update).not.to.have.property('$addToSet'); + }); + + it('retains pre-fork master behavior without copying a parent-owned materialization', async function() { + const parentTx = makeLocalTx({ + chain: 'ETH', + txid: TX2, + wallets: [new ObjectId()], + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + }); + const txFind = sandbox.stub(); + txFind.onFirstCall().returns(cursorWithRows([parentTx])); + txFind.onSecondCall().returns(cursorWithRows([])); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: txFind + } as any)); + const walletFind = sandbox.stub(); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ find: walletFind } as any)); + + const [operation] = await EVMTransactionStorage.addTransactions({ + txs: [makeTx({ chain: 'ARB', txid: TX1, erc20Effects: undefined })], + height: 100, + chain: 'ARB', + network: 'mainnet', + parentChain: 'ETH', + forkHeight: 200, + initialSyncComplete: true + }); + + expect(walletFind.called).to.equal(false); + expect(operation.updateOne.filter.txid).to.equal(TX2); + expect(operation.updateOne.update.$set.wallets).to.deep.equal([]); + expect(operation.updateOne.update.$set).not.to.have.property('erc20Effects'); + expect(operation.updateOne.update).not.to.have.property('$addToSet'); + expect(operation.updateOne.update).not.to.have.property('$setOnInsert'); + }); + + it('sanitizes the parent copy and preserves a materialized pre-fork child identity and wallets', async function() { + const childTxid = hash('ab'); + const parentTxid = `0x${childTxid.slice(2).toUpperCase()}`; + const existingWallet = new ObjectId(); + const masterWallet = new ObjectId(); + const parentTx = makeLocalTx({ chain: 'ETH', txid: parentTxid, wallets: [new ObjectId()] }); + const childTx = makeLocalTx({ + chain: 'ARB', + txid: childTxid, + wallets: [existingWallet], + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + }); + const txFind = sandbox.stub(); + txFind.onFirstCall().returns(cursorWithRows([parentTx])); + txFind.onSecondCall().returns(cursorWithRows([childTx])); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ find: txFind } as any)); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ + find: sandbox.stub().returns({ toArray: sandbox.stub().resolves([{ wallet: masterWallet }]) }) + } as any)); + + const [operation] = await EVMTransactionStorage.addTransactions({ + txs: [makeTx({ chain: 'ARB', txid: childTxid, erc20Effects: undefined })], + height: 100, + chain: 'ARB', + network: 'mainnet', + parentChain: 'ETH', + forkHeight: 200, + initialSyncComplete: true + }); + + expect(txFind.secondCall.args[0]).to.deep.equal({ + chain: 'ARB', + network: 'mainnet', + txid: { $in: [parentTxid, childTxid] }, + erc20Effects: { $exists: true } + }); + expect(operation.updateOne.filter).to.deep.equal({ + _id: childTx._id, + txid: childTxid, + chain: 'ARB', + network: 'mainnet' + }); + expect(operation.updateOne.upsert).to.equal(false); + expect(operation.updateOne.update.$set.chain).to.equal('ARB'); + expect(operation.updateOne.update.$set.network).to.equal('mainnet'); + for (const field of ['_id', 'txid', 'wallets', 'erc20Effects']) { + expect(operation.updateOne.update.$set).not.to.have.property(field); + } + expect(operation.updateOne.update.$addToSet.wallets.$each).to.deep.equal([masterWallet]); + expect(operation.updateOne.update).not.to.have.property('$setOnInsert'); + }); + + it('rejects pre-fork parent/prepared membership disagreement before wallet lookup', async function() { + const parentTx = makeLocalTx({ chain: 'ETH', txid: TX2 }); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursorWithRows([parentTx])) + } as any)); + const walletFind = sandbox.stub(); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ find: walletFind } as any)); + + await expectRejected(EVMTransactionStorage.addTransactions({ + txs: [makeTx({ + chain: 'ARB', + txid: TX1, + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + })], + height: 100, + chain: 'ARB', + network: 'mainnet', + parentChain: 'ETH', + forkHeight: 200, + initialSyncComplete: true + }), /no prepared child transaction matches/i); + + expect(walletFind.called).to.equal(false); + }); + + it('rejects a prepared materialization bound to a different pre-fork parent inclusion', async function() { + const parentTx = makeLocalTx({ chain: 'ETH', blockHash: OTHER_BLOCK_HASH }); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursorWithRows([parentTx])) + } as any)); + const walletFind = sandbox.stub(); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ find: walletFind } as any)); + + await expectRejected(EVMTransactionStorage.addTransactions({ + txs: [makeTx({ + chain: 'ARB', + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + })], + height: 100, + chain: 'ARB', + network: 'mainnet', + parentChain: 'ETH', + forkHeight: 200, + initialSyncComplete: true + }), /does not match pre-fork parent inclusion/i); + + expect(walletFind.called).to.equal(false); + }); + + it('preserves a newer pre-fork child materialization while updating parent-owned fields and wallets and invalidating canonical caches', async function() { + const childTxid = hash('ab'); + const parentTxid = `0x${childTxid.slice(2).toUpperCase()}`; + const canonicalWallet = new ObjectId(); + const parentTx = makeLocalTx({ chain: 'ETH', txid: parentTxid, from: OTHER, to: OTHER }); + const childTx = makeLocalTx({ + chain: 'ARB', + txid: childTxid, + erc20Effects: { blockHash: BLOCK_HASH, version: 2, items: [validCanonicalEffect({ amount: '222' })] } + }); + const txFind = sandbox.stub(); + txFind.onFirstCall().returns(cursorWithRows([parentTx])); + txFind.onSecondCall().returns(cursorWithRows([childTx])); + const bulkWrite = sandbox.stub().resolves(); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: txFind, + update: sandbox.stub().resolves(), + bulkWrite + } as any)); + sandbox.stub(Config, 'get').returns({ maxPoolSize: 1 } as any); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ + find: sandbox.stub().returns({ toArray: sandbox.stub().resolves([{ wallet: canonicalWallet }]) }) + } as any)); + const expireMaster = sandbox.stub(EVMTransactionStorage, 'expireBalanceCache').resolves(); + const expire = sandbox.stub(CacheStorage, 'expire').resolves(); + const preparedTx = makeTx({ + chain: 'ARB', + txid: childTxid, + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + }); + + await EVMTransactionStorage.batchImport({ + txs: [preparedTx], + height: 100, + blockHash: BLOCK_HASH, + chain: 'ARB', + network: 'mainnet', + parentChain: 'ETH', + forkHeight: 200, + initialSyncComplete: true + }); + const operation = bulkWrite.firstCall.args[0][0]; + + expect(txFind.secondCall.args[0]).to.deep.equal({ + chain: 'ARB', + network: 'mainnet', + txid: { $in: [parentTxid, childTxid] }, + erc20Effects: { $exists: true } + }); + expect(operation.updateOne.update.$set.from).to.equal(OTHER); + expect(operation.updateOne.update.$set).not.to.have.property('erc20Effects'); + expect(operation.updateOne.update.$addToSet.wallets.$each).to.deep.equal([canonicalWallet]); + expect(expireMaster.calledOnce).to.equal(true); + expect(expire.getCalls().map(call => call.args[0])).to.have.members([ + `getBalanceForAddress-ARB-mainnet-${FROM.toLowerCase()}-${TOKEN.toLowerCase()}`, + `getBalanceForAddress-ARB-mainnet-${TO.toLowerCase()}-${TOKEN.toLowerCase()}` + ]); + expect(expire.getCalls().map(call => call.args[0])).not.to.include( + `getBalanceForAddress-ARB-mainnet-${OTHER.toLowerCase()}-${OTHER_TOKEN.toLowerCase()}` + ); + expect(bulkWrite.calledOnce).to.equal(true); + expect(bulkWrite.calledBefore(expire)).to.equal(true); + }); + + it('preserves a newer ordinary materialization while updating master-owned fields and wallets and invalidating canonical caches', async function() { + sandbox.stub(Config, 'chainConfig').returns({ leanTransactionStorage: false } as IEVMNetworkConfig); + sandbox.stub(Config, 'get').returns({ maxPoolSize: 1 } as any); + const canonicalWallet = new ObjectId(); + const versionCursor = cursorWithRows([makeLocalTx({ + blockHash: OTHER_BLOCK_HASH, + erc20Effects: { blockHash: OTHER_BLOCK_HASH, version: 2, items: [validCanonicalEffect({ amount: '222' })] } + })]); + const txFind = sandbox.stub().returns(versionCursor); + const bulkWrite = sandbox.stub().resolves(); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: txFind, + update: sandbox.stub().resolves(), + bulkWrite + } as any)); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ + find: sandbox.stub().returns({ toArray: sandbox.stub().resolves([{ wallet: canonicalWallet }]) }) + } as any)); + const expireMaster = sandbox.stub(EVMTransactionStorage, 'expireBalanceCache').resolves(); + const expire = sandbox.stub(CacheStorage, 'expire').resolves(); + const tx = makeTx({ + fee: 99, + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + }); + + await EVMTransactionStorage.batchImport({ + txs: [tx], + height: 100, + blockHash: BLOCK_HASH, + chain: 'ETH', + network: 'mainnet', + initialSyncComplete: true + }); + const operation = bulkWrite.firstCall.args[0][0]; + + expect(txFind.firstCall.args[0]).to.deep.equal({ + chain: 'ETH', + network: 'mainnet', + txid: { $in: [TX1] }, + erc20Effects: { $exists: true } + }); + expect(versionCursor.project.firstCall.args[0]).to.deep.equal({ txid: 1, 'erc20Effects.version': 1 }); + expect(operation.updateOne.update.$set.fee).to.equal(99); + expect(operation.updateOne.update.$set).not.to.have.property('erc20Effects'); + expect(operation.updateOne.update.$addToSet.wallets.$each).to.deep.equal([canonicalWallet]); + expect(expireMaster.calledOnce).to.equal(true); + expect(expire.getCalls().map(call => call.args[0])).to.have.members([ + `getBalanceForAddress-ETH-mainnet-${FROM.toLowerCase()}-${TOKEN.toLowerCase()}`, + `getBalanceForAddress-ETH-mainnet-${TO.toLowerCase()}-${TOKEN.toLowerCase()}` + ]); + expect(expire.getCalls().map(call => call.args[0])).not.to.include( + `getBalanceForAddress-ETH-mainnet-${OTHER.toLowerCase()}-${OTHER_TOKEN.toLowerCase()}` + ); + expect(bulkWrite.calledOnce).to.equal(true); + expect(bulkWrite.calledBefore(expire)).to.equal(true); + }); + + it('writes the current materialization over the same version and onto an existing row with no field', async function() { + sandbox.stub(Config, 'chainConfig').returns({ leanTransactionStorage: false } as IEVMNetworkConfig); + const current = { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect({ amount: '1' })] }; + const txFind = sandbox.stub().returns(cursorWithRows([ + makeLocalTx({ txid: TX1, erc20Effects: current }), + makeLocalTx({ txid: TX2, erc20Effects: undefined }) + ])); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ find: txFind } as any)); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursorWithRows([])) + } as any)); + const first = makeTx({ txid: TX1, erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } }); + const second = makeTx({ txid: TX2, erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [] } }); + + const operations = await EVMTransactionStorage.addTransactions({ + txs: [first, second], + height: 100, + chain: 'ETH', + network: 'mainnet', + initialSyncComplete: true + }); + + expect(operations[0].updateOne.update.$set.erc20Effects).to.deep.equal(first.erc20Effects); + expect(operations[1].updateOne.update.$set.erc20Effects).to.deep.equal(second.erc20Effects); + }); + + it('merges a canonical recipient wallet without case-normalizing unrelated master-owned addresses', async function() { + sandbox.stub(Config, 'chainConfig').returns({ leanTransactionStorage: false } as IEVMNetworkConfig); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursorWithRows([])) + } as any)); + const canonicalWallet = new ObjectId(); + const unrelatedWallet = new ObjectId(); + const unrelatedTopLevel = OTHER; + let walletQuery: any; + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ + find: sandbox.stub().callsFake(query => { + walletQuery = query; + const rows = [ + { address: TO.toLowerCase(), wallet: canonicalWallet }, + { address: unrelatedTopLevel.toLowerCase(), wallet: unrelatedWallet } + ].filter(row => query.address.$in.includes(row.address)); + return { toArray: sandbox.stub().resolves(rows) }; + }) + } as any)); + const tx = makeTx({ + to: unrelatedTopLevel, + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + }); + + const [operation] = await EVMTransactionStorage.addTransactions({ + txs: [tx], + height: 100, + blockTimeNormalized: tx.blockTimeNormalized, + chain: 'ETH', + network: 'mainnet', + initialSyncComplete: true + }); + + expect(walletQuery.address.$in).to.include(TO.toLowerCase()); + expect(walletQuery.address.$in).to.include(unrelatedTopLevel); + expect(walletQuery.address.$in).not.to.include(unrelatedTopLevel.toLowerCase()); + expect(operation.updateOne.update.$addToSet.wallets.$each).to.deep.equal([canonicalWallet]); + expect(operation.updateOne.update.$set).not.to.have.property('wallets'); + expect(operation.updateOne.update.$set.effects).to.deep.equal([nativeEffect, heuristicEffect]); + expect(operation.updateOne.update.$set.erc20Effects.items).to.deep.equal([validCanonicalEffect()]); + }); + + it('preserves canonical wallet ownership on an ordinary omitted-field write and merges master wallets in the same update', async function() { + sandbox.stub(Config, 'chainConfig').returns({ leanTransactionStorage: false } as IEVMNetworkConfig); + const canonicalWallet = new ObjectId(); + const masterWallet = new ObjectId(); + const storedCursor = cursorWithRows([makeLocalTx({ + wallets: [canonicalWallet], + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + })]); + const txFind = sandbox.stub().returns(storedCursor); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ find: txFind } as any)); + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursorWithRows([{ wallet: masterWallet }])) + } as any)); + const tx = makeTx({ effects: [], erc20Effects: undefined }); + + const [operation] = await EVMTransactionStorage.addTransactions({ + txs: [tx], + height: 100, + chain: 'ETH', + network: 'mainnet', + initialSyncComplete: true + }); + + expect(txFind.firstCall.args[0]).to.deep.equal({ + chain: 'ETH', + network: 'mainnet', + txid: { $in: [TX1] }, + erc20Effects: { $exists: true } + }); + expect(operation.updateOne.update.$set).not.to.have.property('erc20Effects'); + expect(operation.updateOne.update.$set).not.to.have.property('wallets'); + expect(operation.updateOne.update.$addToSet.wallets.$each).to.deep.equal([masterWallet]); + }); + + it('uses exact master wallet lookup for unmaterialized confirmed and pending transactions', async function() { + sandbox.stub(Config, 'chainConfig').returns({ leanTransactionStorage: false } as IEVMNetworkConfig); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => ({ + find: sandbox.stub().returns(cursorWithRows([])) + } as any)); + const exactWallet = new ObjectId(); + const lowerOnlyWallet = new ObjectId(); + const mixedCaseAddress = '0xBbBbBbBbBbBbBbBbBbBbBbBbBbBbBbBbBbBbBbBb'; + const walletQueries: any[] = []; + sandbox.stub(WalletAddressStorage, 'collection').get(() => ({ + find: sandbox.stub().callsFake(query => { + walletQueries.push(query); + const rows = [ + { address: mixedCaseAddress, wallet: exactWallet }, + { address: mixedCaseAddress.toLowerCase(), wallet: lowerOnlyWallet } + ].filter(row => query.address.$in.includes(row.address)); + return { toArray: sandbox.stub().resolves(rows) }; + }) + } as any)); + const unmaterialized = makeTx({ + from: mixedCaseAddress, + to: mixedCaseAddress, + effects: [], + erc20Effects: undefined + }); + const pending = makeTx({ + from: mixedCaseAddress, + to: mixedCaseAddress, + effects: [], + blockHeight: -1, + blockHash: undefined, + erc20Effects: undefined + }); + + const [confirmedOperation] = await EVMTransactionStorage.addTransactions({ + txs: [unmaterialized], + height: 100, + chain: 'ETH', + network: 'mainnet', + initialSyncComplete: true + }); + const [pendingOperation] = await EVMTransactionStorage.addTransactions({ + txs: [pending], + height: -1, + chain: 'ETH', + network: 'mainnet', + initialSyncComplete: true + }); + + expect(walletQueries).to.have.length(2); + for (const query of walletQueries) { + expect(query.address.$in).to.include(mixedCaseAddress); + expect(query.address.$in).not.to.include(mixedCaseAddress.toLowerCase()); + } + for (const operation of [confirmedOperation, pendingOperation]) { + expect(operation.updateOne.update.$set.wallets).to.deep.equal([exactWallet]); + expect(operation.updateOne.update.$set).not.to.have.property('erc20Effects'); + expect(operation.updateOne.update).not.to.have.property('$addToSet'); + } + }); + + it('invalidates token-balance caches for canonical participants rather than discarded heuristic participants', async function() { + const expire = sandbox.stub(CacheStorage, 'expire').resolves(); + const tx = makeTx({ + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + }); + + await EVMTransactionStorage.expireBalanceCacheForTransaction({ chain: 'ETH', network: 'mainnet', tx }); + + const keys = expire.getCalls().map(call => call.args[0]); + expect(keys).to.include(`getBalanceForAddress-ETH-mainnet-${TO.toLowerCase()}-${TOKEN.toLowerCase()}`); + expect(keys).not.to.include(`getBalanceForAddress-ETH-mainnet-${OTHER.toLowerCase()}-${OTHER_TOKEN.toLowerCase()}`); + }); + + it('limits backfill cache invalidation to canonical token participants and makes empty results a no-op', async function() { + const expire = sandbox.stub(CacheStorage, 'expire').resolves(); + const tx = makeTx({ + erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [validCanonicalEffect()] } + }); + + await EVMTransactionStorage.expireErc20BalanceCacheForTransaction({ chain: 'ETH', network: 'mainnet', tx }); + + expect(expire.getCalls().map(call => call.args[0])).to.have.members([ + `getBalanceForAddress-ETH-mainnet-${FROM.toLowerCase()}-${TOKEN.toLowerCase()}`, + `getBalanceForAddress-ETH-mainnet-${TO.toLowerCase()}-${TOKEN.toLowerCase()}` + ]); + expire.resetHistory(); + + await EVMTransactionStorage.expireErc20BalanceCacheForTransaction({ + chain: 'ETH', + network: 'mainnet', + tx: makeTx({ erc20Effects: { blockHash: BLOCK_HASH, version: 1, items: [] } }) + }); + expect(expire.called).to.equal(false); + }); + }); +});