diff --git a/NETWORK.md b/NETWORK.md index f57f3e70..5a5474ee 100644 --- a/NETWORK.md +++ b/NETWORK.md @@ -1,149 +1,144 @@ -### Network Layer Overview +# Network Layer -This document describes how the networking subsystem works in the Atto node, focusing on `NetworkProcessor`, -`NodeConnectionManager`, and the Guardian rate-limiting system. ---- - -### NetworkProcessor - -`NetworkProcessor` is a Spring `@Component` responsible for establishing and accepting peer-to-peer WebSocket -connections with a mutual challenge-based handshake. - -#### Embedded Server - -An embedded Ktor CIO server exposes two endpoints: - -- **`POST /handshakes`** – Receives a `CounterChallengeResponse` from a connecting peer. Rejects requests from banned - addresses (via `BannedMonitor`), calls `Guardian.requestInboundConnection` for rate-limiting, validates the - challenge, verifies the cryptographic signature and genesis hash, signs the counter-challenge, and emits the peer's - `AttoNode` into a `MutableSharedFlow` so the WebSocket handler (or the outbound connector) can proceed. -- **`WS /`** – Accepts inbound WebSocket connections. The connecting client must provide `Atto-Public-Uri` and - `Atto-Http-Challenge` headers. The server rejects banned addresses, validates the challenge prefix, performs a mutual - handshake by calling the client's `/handshakes` endpoint over HTTP, verifies the response signature and `publicUri` - match, and then delegates the live WebSocket session to `NodeConnectionManager.manage()`. - -#### Outbound Connections (Client Side) - -- **`bootstrap()`** – Runs every 1 second (`@Scheduled(fixedRate = 1_000)`). Iterates over configured default nodes and - calls `connectAsync()` for each. -- **`onKeepAlive()`** – Listens for inbound `AttoKeepAlive` messages. If a neighbour URI is present, attempts to connect - to it (peer discovery via gossip). -- **`connectAsync()` / `connection()`** – Opens a WebSocket to the remote peer, sends challenge headers, waits (with a - 5-second timeout) for the handshake flow to complete, verifies the `publicUri` matches, and then hands the session to - `NodeConnectionManager.manage()`. - -#### Handshake Protocol - -1. **Client** opens a WebSocket to the server, sending its `publicUri` and a generated challenge (prefixed with the - server's URI) in headers. -2. **Server** validates the challenge prefix, signs the client's challenge, generates its own counter-challenge, and - POSTs a `CounterChallengeResponse` (including the genesis hash) to the client's `/handshakes` endpoint. -3. **Client's `/handshakes`** handler validates the counter-challenge, verifies the signature and genesis hash, signs - the counter-challenge, and responds with a `ChallengeResponse`. -4. **Server** verifies the client's signature on the counter-challenge and that the `publicUri` matches. If valid, the - WebSocket is promoted to a managed connection. Both sides prove ownership of their private key by signing challenges, - preventing impersonation. The genesis hash exchange ensures both peers are on the same network. - -#### Key Data Structures - -- **`connectingMap`** – A Caffeine cache (`URI → MutableSharedFlow`) that coordinates the async handshake. - Entries expire after 5 seconds. -- **`ChallengeStore`** – Generates and validates one-time challenges prefixed with the target node's URI. - ---- - -### NodeConnectionManager - -`NodeConnectionManager` is a Spring `@Component` that manages the lifecycle of all active peer connections. - -#### Connection Storage - -- **`connectionMap`** – A Caffeine cache (`URI → NodeConnection`) with a 60-second expiry. When an entry is evicted - (timeout, explicit removal, or replacement), the removal listener disconnects the session and publishes a - `NodeDisconnected` event. - -#### Connection Lifecycle (`manage()`) - -1. Wraps the `AttoNode`, socket address, and `WebSocketSession` into a `NodeConnection`. -2. Inserts into `connectionMap` via `put`. If a connection already exists for that URI, the previous one is disconnected - and replaced (last connection wins). -3. Consumes the WebSocket's incoming binary frames as a Kotlin `Flow`. -4. On flow start, publishes a `NodeConnected` event. -5. Each frame is deserialized; invalid messages cause disconnection. `AttoKeepAlive` messages refresh the cache entry - (resetting the 60-second TTL). -6. On flow completion or exception, the connection is removed from the map and the session is disconnected. - -Connection events are delivered by independent asynchronous listener tasks; publishing `NodeConnected` before -`NodeDisconnected` does not guarantee that their listeners finish in that order. The peer projections in -`TransactionNetworkProvider`, `GapDiscoverer`, and `SendDiscoverer` currently use unconditional add/remove operations. -A delayed connect listener can therefore add a stale URI/public-key entry after disconnect cleanup has run. -These projections have no expiry or reconciliation, so that metadata remains until a later removal or process restart. -The authoritative connection map and WebSocket lifecycle are separate from these retained peer entries. - -#### Sending Messages - -- **`send(publicUri, message)`** – Serializes and sends a message to a specific peer. On failure, removes the - connection. -- **`send(DirectNetworkMessage)`** – Event listener that sends a message to a single peer. -- **`send(BroadcastNetworkMessage)`** – Event listener that broadcasts a message to all connected peers matching the - broadcast filter. `BroadcastNetworkMessage` carries a `BroadcastStrategy` (`EVERYONE` or `VOTERS`) and an optional - `exceptions` set of URIs to exclude. The `accepts(target, node)` method determines whether each peer should receive - the message. - -#### Banning - -- **`ban(NodeBanned)`** – Event listener that disconnects all connections whose address matches the banned address. - -#### Keep-Alive - -- **`keepAlive()`** – Runs every 10 seconds. Picks a random connected peer as a "neighbour" sample and broadcasts an - `AttoKeepAlive` message to all peers. This serves two purposes: resetting the 60-second connection TTL and enabling - peer discovery (recipients learn about the sampled neighbour). - ---- - -### Guardian (Rate Limiting) - -The `guardian` package provides automatic rate-limiting and banning of misbehaving peers. - -#### Guardian - -`Guardian` is a Spring `@Service` that monitors inbound message rates and bans peers that exceed a threshold derived -from voter behaviour. - -- **Message Counting** – Tracks per-socket-address message counts in `statisticsMap`. Counting and window rotation - share a lock; `guard()` swaps out the map and retains only the latest completed window as its snapshot. - Outbound `DirectNetworkMessage` events with positive expected-response counts add per-URI credits so that solicited - responses are not counted against the peer. Consuming the final credit removes the entry immediately. Unanswered - credits expire one minute after the latest request's original timestamp; inbound responses do not extend that deadline. - Expired entries are removed when a response is counted or during the next guard sweep, including credits added by - a delayed outbound listener after disconnect. -- **Voter Tracking** – Maintains a `voterMap` of connected voters (peers above minimal rebroadcast weight). Updated on - `NodeConnected` / `NodeDisconnected` events. -- **`guard()`** – Runs every 15 seconds. Drains the current counts, calculates the median rate among connected eligible - voters (including zero for idle voters), and bans any address whose merged rate exceeds `median × toleranceMultiplier` - (configurable via `GuardianProperties`). A `minimalMedian` threshold prevents banning when overall traffic is too low. -- **Connection Requests** – `NetworkProcessor` calls `requestInboundConnection` directly for handshake/WebSocket - admission. Guardian counts non-banned attempts by IP and applies the configured per-minute connection limit. - -#### BannedMonitor - -`BannedMonitor` is a singleton `@Component` that maintains an in-memory `ConcurrentHashSet` of banned `InetAddress` -entries. It listens for `NodeBanned` and `NodeUnbanned` events and exposes `isBanned(address)` for use by -`NetworkProcessor` to reject connections early. - -#### Unbanner - -`Unbanner` is a -`@Component` that automatically lifts bans after 1 hour. It tracks ban timestamps and runs a review every 1 minute, publishing -`NodeUnbanned` events for expired bans. ---- - -### Network Events Summary - -| Event | Published By | Consumed By | -|------------------------------|-------------------------|------------------------------------------------------| -| `NodeConnected` | `NodeConnectionManager` | `Guardian` | -| `NodeDisconnected` | `NodeConnectionManager` | `Guardian` | -| `NodeBanned` | `Guardian` | `BannedMonitor`, `NodeConnectionManager`, `Unbanner` | -| `NodeUnbanned` | `Unbanner` | `BannedMonitor` | +The networking subsystem authenticates peers in [NetworkProcessor][processor], manages their WebSocket sessions in +[NodeConnectionManager][connections], and counts traffic in [Guardian][guardian]. These owners communicate through direct +queries and asynchronous network-message and lifecycle events. + +## Peer Authentication + +`NetworkProcessor` starts an embedded Ktor CIO server with two endpoints: + +- **`POST /handshakes`** accepts a `CounterChallengeResponse`. Guardian admits the remote address, the processor consumes + a one-time challenge, checks the expected public URI, genesis hash, signature and counter-challenge prefix, then signs + a `ChallengeResponse`. Successful authentication completes the waiting outbound attempt before the HTTP response is sent. +- **`WS /`** accepts a WebSocket with `Atto-Public-Uri` and `Atto-Http-Challenge` headers. Guardian admits the remote address, + the processor validates the headers and challenge prefix, then posts a counter-challenge to the peer's `/handshakes` + endpoint. It checks the returned URI and signature before handing the session to `NodeConnectionManager.manage()`. + +An inbound HTTP handshake has a five-second preparation deadline covering admission, body reading, verification and signing. +A separate semaphore limits concurrent HTTP handshakes to 64. The [callback service][callback] applies a five-second deadline +covering URI validation, request construction/signing and the callback exchange. Both handshake payload directions have bounded +body readers in [HandshakePayloadLimit][payload-limit]. Cancellation propagates to ongoing handshake work. + +### Outbound Connections + +`bootstrap()` runs every second and tries the configured default nodes. An inbound `AttoKeepAlive` can also supply a neighbour +URI for discovery. `connectAsync()` reserves the URI in `connectingMap` before starting a connection coroutine; non-default +outbound work also needs a permit from a pool of 64. + +`connection()` validates the peer URI, checks whether it is already connected, and opens a WebSocket with the challenge +headers. It waits for its [OutboundConnectionAttempt][attempt] to produce the authenticated node. A timeout task expires an +unauthenticated attempt and cancels the connection coroutine after five seconds. Authentication and expiration compete through +the same `CompletableDeferred`; expiration cannot win after authentication. After authentication, the timeout is cancelled and +`manage()` owns the established session. The coroutine's `finally` block releases its reservation and any outbound permit. + +### Handshake State + +- `connectingMap` is a Caffeine cache of `URI → OutboundConnectionAttempt`. It reserves pending peer work and lets an HTTP + callback complete the outbound attempt's deferred node. Entries expire after five seconds. +- [ChallengeStore][challenges] generates random one-time challenges prefixed with the target URI. Entries expire after five + seconds; consuming a challenge removes it. +- [PeerUriValidator][uri-validator] owns callback/outbound URI policy, including configured default-node exceptions and + address checks. [NetworkDnsResolver][dns] owns the dispatcher used for blocking DNS calls. + +## Active Connections and Peer Queries + +`NodeConnectionManager` is the sole owner of active connections. Its Caffeine map stores `URI → NodeConnection`, with a +60-second expiry refreshed by keepalives from the current session. Registering another connection for the same URI replaces +and disconnects the previous session: the last registration wins. + +The manager implements [ConnectedPeers][connected-peers], exposing `snapshot()` and `isConnected(publicUri)`. A snapshot +contains detached `ConnectedPeer(node, socketAddress)` metadata, without transport objects. These queries describe current +local state; a peer may disconnect immediately after a query. They do not reserve a connection or promise delivery. + +Consumers query this owner instead of reconstructing membership from connection events: + +| Consumer | Query and feature-owned policy | +| --- | --- | +| `TransactionNetworkProvider` | Checks membership before transaction lookup/streaming and between streamed responses. Historical-node eligibility, concurrent lookup permits, stream serialization and request deadlines remain with the provider. | +| `GapDiscoverer` | Selects connected historical peers and excludes peers with active gap sessions. It owns session budgets and cancellation; a disconnect event releases sessions only when the URI is currently disconnected. | +| `SendDiscoverer` | Finds a connected representative from the supplied votes, preserving vote order; otherwise broadcasts the missing-send request. | +| `VoteKeeper` | Finds a connected historical representative for missing votes. Local vote signing and missing-vote tracking remain with the keeper. | +| `Guardian` | Derives the current voter socket/public-key view, then applies its own current-weight eligibility check. | +| `NetworkInfoContributor` | Builds the `paired` diagnostic output from current connection metadata. | + +### Session Lifecycle + +`manage()` registers the connection and consumes binary frames from its incoming channel. It deserializes each binary +message through [NetworkSerializer][serializer], which also checks protocol validity. Invalid messages remove the connection. Valid messages become `InboundNetworkMessage` events with the peer URI, +socket address and `WEBSOCKET` source. + +A current session's keepalive renews its map entry without replacing a newer session. Flow completion, decoding rejection and +send failures remove entries conditionally using the session identity, so an old session cannot remove its replacement. +The `finally` block initiates transport close. Cache removal also schedules close and then publishes `NodeDisconnected`; +expiry, explicit removal and replacement use this same cleanup path. + +Lifecycle events run through independent asynchronous listener tasks. Their publication order does not guarantee consumer +completion order, and an old session's disconnect event can arrive after its replacement is registered. Current peer queries +avoid relying on that order. `GapDiscoverer` retains a disconnect reaction because cancelling its feature-owned work is a real +side effect; its connectivity check prevents a delayed disconnect from releasing sessions solely because the URI reconnected. +Gap sessions identify peers by URI and retain their existing expiry; this is not connection-generation isolation. + +`clear()` removes connections and waits for already scheduled cleanup jobs. `stop()` clears the map and cancels the manager's +scope. `NetworkProcessor.stop()` separately closes its client, cancels outbound work and stops its embedded server. + +### Sending and Keepalives + +- `send(publicUri, message)` serializes and sends to the current session, returning if the URI is absent. Failure removes + that session conditionally. +- `DirectNetworkMessage` listeners send to one URI. `BroadcastNetworkMessage` selects all connected peers matching its + `EVERYONE` or `VOTERS` strategy and optional URI exceptions. +- `NodeBanned` disconnects active connections using the banned IP address. +- Every ten seconds, `keepAlive()` broadcasts `AttoKeepAlive` with a random connected neighbour URI, if one is available. + Recipients refresh their connection lifetime and may discover that neighbour. + +## Traffic Admission and Banning + +Guardian owns traffic statistics, threshold decisions and inbound connection request counters. [PeerBans][peer-bans] owns +active bans and their expiry; [ExpectedResponses][expected-responses] owns per-URI response credits. `NetworkProcessor` calls +Guardian's `requestInboundConnection()` directly for HTTP handshake and WebSocket admission. Guardian queries current ban state +before counting; other attempts contribute to statistics and the configured per-minute connection limit. + +Message counting and window rotation share a lock, so each counted hit belongs to one window. Every 15 seconds, `guard()` +swaps out the current window and retains it as the latest diagnostic snapshot. It queries connected voters, applies current +minimal rebroadcast weight, and includes zero for idle eligible voters. If the median meets `minimalMedian`, addresses whose +merged count reaches `median × toleranceMultiplier` are banned through `PeerBans`. The owner updates admission state before +publishing `NodeBanned`; its asynchronous consumers cannot delay that state transition. + +`NetworkMessagePublisher` records positive expected-response counts synchronously in `ExpectedResponses` before dispatching +an outbound `DirectNetworkMessage`. A transport listener can therefore send a request and receive a fast response without +racing a separate credit-allocation listener. Guardian consumes credits instead of counting solicited inbound responses; +consuming the final credit immediately removes the entry. Only outbound request timestamps extend credit expiry. Inbound +responses do not, and delayed publication retains the request's original deadline. Expired credits are removed during response +counting or the next guard sweep. Credit lifetime is independent of peer metadata and survives reconnects until consumption +or expiry. Credits retain the existing publication-based contract: publication does not promise successful transport delivery. + +`PeerBans` reviews active bans every minute and removes them after one hour from the latest ban. Renewal, expiry, admission +queries and snapshots share its lock, so an expiry cannot remove a concurrent renewal. `NodeBanned` and `NodeUnbanned` report +completed state changes; they do not update the owner's state. Diagnostic banned output queries this same owner directly, +so delayed lifecycle listeners cannot leave admission and diagnostics with conflicting ban projections. + +## Events + +Network messages use the asynchronous application event dispatcher. Their publisher returns without waiting for every +transport, feature, traffic-accounting or metrics listener to finish. + +| Event | Producer | Current lifecycle consumers | +| --- | --- | --- | +| `NodeDisconnected` | `NodeConnectionManager` | `GapDiscoverer` session cleanup. | +| `NodeBanned` | `PeerBans`, called by `Guardian` | `NodeConnectionManager` disconnects affected sessions. | +| `NodeUnbanned` | `PeerBans` expiry review | No current consumers. | + +[processor]: src/main/kotlin/cash/atto/node/network/NetworkProcessor.kt +[connections]: src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt +[connected-peers]: src/main/kotlin/cash/atto/node/network/ConnectedPeers.kt +[guardian]: src/main/kotlin/cash/atto/node/network/guardian/Guardian.kt +[callback]: src/main/kotlin/cash/atto/node/network/HandshakeCallback.kt +[payload-limit]: src/main/kotlin/cash/atto/node/network/HandshakePayloadLimit.kt +[attempt]: src/main/kotlin/cash/atto/node/network/OutboundConnectionAttempt.kt +[challenges]: src/main/kotlin/cash/atto/node/network/ChallengeStore.kt +[uri-validator]: src/main/kotlin/cash/atto/node/network/PeerUriValidator.kt +[dns]: src/main/kotlin/cash/atto/node/network/NetworkDnsResolver.kt +[serializer]: src/main/kotlin/cash/atto/node/network/NetworkSerializer.kt +[peer-bans]: src/main/kotlin/cash/atto/node/network/guardian/PeerBans.kt +[expected-responses]: src/main/kotlin/cash/atto/node/network/guardian/ExpectedResponses.kt diff --git a/README.md b/README.md index 67e68d41..96416a73 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,9 @@ All streaming endpoints use **NDJSON** (`application/x-ndjson`) and combine: Address-specific account streams subscribe to live updates before reading account snapshots. Live flow emissions wait for that subscriber to finish the snapshot, then continue through the same subscription. Each collection forwards only increasing heights for each requested address. +Transaction and account-entry hash/range streams use the same subscription-before-query handoff. Hash streams +complete after one result; range streams order and deduplicate results by height before completing the requested range. +Slow snapshot reads or clients backpressure the shared live feed; disconnecting cancels that subscriber and its query. Receivable streams subscribe to live updates before reading the database. They scan the database before draining a bounded live-update queue and continuing on the same subscription. Only hashes from the initial @@ -224,8 +227,9 @@ stores it as an **unchecked transaction**: `AttoTransactionStreamRequest/Response`. - Handles out-of-order deliveries with an in-memory buffer. -- **Head/last discovery** (`LastDiscoverer`) - - Historical nodes periodically broadcast a sample of recent “head” transactions (`AttoBootstrapTransactionPush`). +- **Head/last discovery** (`BootstrapHeadBroadcaster`, `LastDiscoverer`) + - `BootstrapHeadBroadcaster` periodically broadcasts a sample of recent “head” transactions + (`AttoBootstrapTransactionPush`) from historical nodes with no unchecked backlog. - If a receiving node is behind, it starts a lightweight vote stream for the head to confirm it’s real, then turns it into a `TransactionDiscovered` event once the votes reach quorum. - Partial vote collections expire without starting a live election. A later head advertisement can retry discovery. diff --git a/TRANSACTION-PROCESSING.md b/TRANSACTION-PROCESSING.md index 209620ba..2ba20092 100644 --- a/TRANSACTION-PROCESSING.md +++ b/TRANSACTION-PROCESSING.md @@ -40,7 +40,7 @@ are listed in [Events and completion](#events-and-completion). ```mermaid flowchart TB - REST["TransactionController
REST submission"] + REST["TransactionPublicationController
REST submission"] P2P["NodeConnectionManager
peer transaction push"] PRIORITY["TransactionPrioritizer
deduplicate and buffer dependencies"] TXWORK["Transaction queue + worker"] @@ -64,7 +64,7 @@ flowchart TB COMMIT -.->|"AccountUpdated after commit"| COMPLETE ``` -1. **Admission:** [TransactionController.publishAndConfirm][transaction-controller] checks transaction structure and +1. **Admission:** [TransactionPublicationController.publishAndConfirm][transaction-publication] checks transaction structure and network, registers a hash-specific pending result, and publishes a REST-sourced message. Peer pushes come through [NodeConnectionManager.manage][connections]. Both reach `TransactionPrioritizer.add(message)`; rebroadcast tracking independently observes the same message. @@ -142,19 +142,19 @@ listener does not mean every event changes its state. Definitions live beside th | Event/message | Producer | Behavior listeners | | --- | --- | --- | -| `InboundNetworkMessage` | [TransactionController][transaction-controller], [NodeConnectionManager][connections] | [TransactionPrioritizer.add][transaction-prioritizer], [TransactionRebroadcaster.process][transaction-rebroadcaster] | +| `InboundNetworkMessage` | [TransactionPublicationController][transaction-publication], [NodeConnectionManager][connections] | [TransactionPrioritizer.add][transaction-prioritizer], [TransactionRebroadcaster.process][transaction-rebroadcaster] | | `TransactionReceived` | [TransactionPrioritizer.drain][transaction-prioritizer] | [TransactionValidationManager.process][validation] | | `TransactionValidated` | [TransactionValidationManager.process][validation] | [Election.start][election], [TransactionRebroadcaster.process][transaction-rebroadcaster] | -| `TransactionRejected` | [TransactionValidationManager.process][validation] | [TransactionPrioritizer][transaction-prioritizer], [TransactionRebroadcaster][transaction-rebroadcaster], [VotePrioritizer][vote-prioritizer], [ElectionVoter][election-voter], [DependencyDiscoverer][dependency-discoverer], [TransactionController][transaction-controller] | -| `TransactionValidationFailed` | [TransactionValidationManager.process][validation] | [TransactionPrioritizer][transaction-prioritizer], [TransactionController][transaction-controller] | -| `TransactionDropped` | [TransactionPrioritizer][transaction-prioritizer], on queue eviction | [TransactionPrioritizer][transaction-prioritizer], [TransactionRebroadcaster][transaction-rebroadcaster], [TransactionController][transaction-controller] | +| `TransactionRejected` | [TransactionValidationManager.process][validation] | [TransactionPrioritizer][transaction-prioritizer], [TransactionRebroadcaster][transaction-rebroadcaster], [VotePrioritizer][vote-prioritizer], [ElectionVoter][election-voter], [DependencyDiscoverer][dependency-discoverer], [TransactionPublicationController][transaction-publication] | +| `TransactionValidationFailed` | [TransactionValidationManager.process][validation] | [TransactionPrioritizer][transaction-prioritizer], [TransactionPublicationController][transaction-publication] | +| `TransactionDropped` | [TransactionPrioritizer][transaction-prioritizer], on queue eviction | [TransactionPrioritizer][transaction-prioritizer], [TransactionRebroadcaster][transaction-rebroadcaster], [TransactionPublicationController][transaction-publication] | | `ElectionStarted` | [Election.start][election] | [VotePrioritizer][vote-prioritizer], [ElectionVoter][election-voter] | -| `ElectionRejected` | [Election.start][election], for pending/completed heights; completion cleanup for historical candidates | [TransactionPrioritizer][transaction-prioritizer] through `TransactionFailure`, [TransactionController][transaction-controller] | +| `ElectionRejected` | [Election.start][election], for pending/completed heights; completion cleanup for historical candidates | [TransactionPrioritizer][transaction-prioritizer] through `TransactionFailure`, [TransactionPublicationController][transaction-publication] | | `ElectionDiscarded` | [ElectionProcessor][election-processor], after reconciliation observes a committed height at or beyond the candidate | [Election][election], [TransactionPrioritizer][transaction-prioritizer] through `TransactionFailure` | | `ElectionConsensusChanged` | [Election.process][election] | [ElectionVoter][election-voter] | | `ElectionConsensusReached` | [Election.process][election] | [ElectionProcessor][election-processor], [ElectionVoter][election-voter], [FinalVoteRecorder][final-votes], [VoteTimestampRecorder][vote-timestamps] | | `ElectionExpiring` | [Election.notifyExpiring][election], scheduled | [ElectionVoter][election-voter], [ElectionProcessor][election-processor] | -| `ElectionExpired` | [Election.expireOld][election], scheduled | [TransactionPrioritizer][transaction-prioritizer], [VotePrioritizer][vote-prioritizer], [ElectionVoter][election-voter], [TransactionController][transaction-controller] | +| `ElectionExpired` | [Election.expireOld][election], scheduled | [TransactionPrioritizer][transaction-prioritizer], [VotePrioritizer][vote-prioritizer], [ElectionVoter][election-voter], [TransactionPublicationController][transaction-publication] | | `ElectionLost` | [Election][election], on consensus, expiry, or account completion | [TransactionPrioritizer][transaction-prioritizer] | | `VoteReceived` | [VoteConverter][vote-converter] | [VotePrioritizer][vote-prioritizer], [VoteRebroadcaster][vote-rebroadcaster] | | `VoteValidated` | [VotePrioritizer.drain][vote-prioritizer], [ElectionVoter][election-voter] | [Election][election], [VoteRebroadcaster][vote-rebroadcaster], [VoteWeighter][vote-weighter] | @@ -163,7 +163,7 @@ listener does not mean every event changes its state. Definitions live beside th | After-commit event | Registered by | Behavior listeners | | --- | --- | --- | -| `AccountUpdated` | [AccountService.add][account-service], for `ELECTION` or `BOOTSTRAP` | [TransactionPrioritizer][transaction-prioritizer], [Election][election], [ElectionProcessor][election-processor], [ElectionVoter][election-voter], [VotePrioritizer][vote-prioritizer], [VoteCleaner][vote-cleaner], [VoteWeighter][vote-weighter], [AccountController][account-controller], [TransactionController][transaction-controller] | +| `AccountUpdated` | [AccountService.add][account-service], for `ELECTION` or `BOOTSTRAP` | [TransactionPrioritizer][transaction-prioritizer], [Election][election], [ElectionProcessor][election-processor], [ElectionVoter][election-voter], [VotePrioritizer][vote-prioritizer], [VoteCleaner][vote-cleaner], [VoteWeighter][vote-weighter], [AccountController][account-controller], [TransactionPublicationController][transaction-publication], [TransactionController][transaction-controller] | | `ReceivableSaved` | [ReceivableService.saveAll][receivable-service], after a successful insert | [ReceivableController][receivable-controller]; each stream buffers live updates before scanning the database, then suppresses notifications matching retained database hashes | | `AccountEntrySaved` | [AccountEntryService.saveAll][entry-service], historical nodes | [AccountEntryController][entry-controller] | @@ -175,27 +175,33 @@ election expiry are recoverable, while transaction rejections derive the flag fr failures default to retaining time-based duplicate suppression. `TransactionDropped` describes queued work and does not remove an active pending registration with the same hash. -`TransactionController` completes pending publication results on confirmation, rejection, validation exceptions, +`TransactionPublicationController` completes pending publication results on confirmation, rejection, validation exceptions, election admission rejection, election expiry, or queue drop; `ALREADY_CONFIRMED` is handled as confirmation. On `ElectionRejected`, a stored exact hash completes successfully; otherwise the request receives HTTP 409. Lookup failures and validation exceptions return a generic HTTP 500 without exposing their cause. It does not listen to `ElectionLost` or `ElectionDiscarded`. A request timeout does not -prove the transaction failed to commit. Historical vote and timestamp recording have their own queues; they are not +prove the transaction failed to commit. The separate [TransactionController][transaction-controller] consumes +`AccountUpdated` independently to emit its live query feeds; publication confirmation does not wait for those streams. +Historical vote and timestamp recording have their own queues; they are not part of the account transaction merely because they consume the consensus event. ## State ownership and recovery | State | Owner and synchronization | Completion/failure behavior to preserve | | --- | --- | --- | +| Pending HTTP publication attempts | [TransactionPublicationController][transaction-publication]; concurrent hash map with per-hash registration/removal of waiter counts and replayed results | Registration precedes publication. Same-hash requests can share one attempt; joining and retiring waiters update membership and counts atomically so a retiring caller cannot detach a new waiter; each caller removes only its own registration on completion or cancellation. A suspended election-rejection lookup completes only the attempt it observed, so it cannot replace a confirmation or complete a newer publication. | | Pending hashes and dependent transactions | [TransactionPrioritizer][transaction-prioritizer]; concurrent map, with atomic operations per dependency hash | The single worker registers each dispatched hash before validation. `computeIfPresent` attaches hash-deduplicated dependents; `AccountUpdated` and processing failures detach only the completed hash, then requeue dependents outside map callbacks. Election owners report stale/obsolete candidates through failure events. The inbound duplicate filter is advisory; queue admission and the pending claim enforce unique processing. | | Transaction priority queue and queued hash membership | [TransactionQueue][transaction-queue]; existing lock per priority group | Queue admission and dequeue claims run under the same group lock, preventing a duplicate from slipping between queue removal and pending registration. The queue deduplicates hashes even when arrival metadata or work differs. Callbacks do not mutate the queue or publish events; deferred candidates leave the queue only after their dependency takes ownership. | -| Candidates, votes, pending consensus, cached confirmed heights | [Election][election]; mutex for keyed elections, pending markers, their per-account height index, and the bounded confirmed-height cache | Consensus moves the public key/height into `pendingConsensus` while retaining its index membership. `AccountUpdated` and `ElectionDiscarded` advance the retained confirmed height and inspect only that account's indexed active/pending heights for retirement. Newer work is preserved. Exact-height competitors receive `ElectionLost`; historical candidates receive `ElectionRejected`. Expiry and clear also remove index membership. Admission rejects pending or cached completed heights. | +| Candidates, votes, pending consensus, cached confirmed heights | [Election][election]; one coroutine mutex for keyed elections, pending markers, their per-account height index, the bounded confirmed-height cache, clear, and detached diagnostic snapshots | Consensus moves the public key/height into `pendingConsensus` while retaining its index membership. `AccountUpdated` and `ElectionDiscarded` advance the retained confirmed height and inspect only that account's indexed active/pending heights for retirement. Newer work is preserved. Exact-height competitors receive `ElectionLost`; historical candidates receive `ElectionRejected`. Expiry and clear also remove index membership. Admission rejects pending or cached completed heights. Synchronous diagnostics and clear acquire the same mutex through a blocking bridge; suspended transitions retain cancellable acquisition. Diagnostics copy nested candidates, votes, weights, and quorum while holding the mutex; readers cannot mutate live elections. | | Votes awaiting an election and priority queue | [VotePrioritizer][vote-prioritizer]; mutex plus concurrent election membership | `ElectionStarted` releases buffered votes; rejection drops buffered votes; account completion and expiry remove active membership. | | Consensus awaiting local persistence | [ElectionProcessor][election-processor]; `bufferLock`, hash-keyed `LinkedHashMap`, one drain worker | Exact duplicates retain their insertion position. A batch stops before repeating a public key; remaining work stays queued. On persistence failure, reuse the account reconciliation read and publish `ElectionDiscarded` before removing obsolete work. Publication occurs outside the buffer lock; failure retains that work. Unresolved entries remain queued for retry. | +| Final votes awaiting persistence | [VoteService][vote-service]; `bufferLock` protects the FIFO buffer and clear generation, with one drain worker | Each batch deduplicates signatures before insertion. Failed batches return ahead of newer votes for the worker's retry; cancellation propagates. Clearing advances the generation so an in-flight failure cannot restore cleared votes. Repository I/O runs outside the lock. | | Account and receivable caches | [AccountRepository][account-repository], [ReceivableRepository][receivable-repository]; reactive transaction resources and completion callbacks | Account cache updates after commit and evicts on rollback/unknown completion. Receivable cache replacement/removal is conditional on the pending entry's identity, preventing a late callback from restoring deleted state. These callbacks are separate from `AccountUpdated` listeners. | | Account stream snapshot and latest heights | [AccountController][account-controller]; registered SharedFlow subscription and one height map per collection | `onSubscription` reads the database after registration. Concurrent live emissions wait until that subscriber finishes its snapshot; the same subscription then delivers live updates. `forwardHeightBy` suppresses equal/older heights per requested address. Cancellation or a failed query detaches the subscriber. | +| Transaction and account-entry stream snapshots | [TransactionController][transaction-controller] and [AccountEntryController][account-entry-controller]; one registered SharedFlow subscription per hash or account range | `onSubscription` reads the database after registration, then the same subscription supplies live updates. Hash streams take one result; range streams sort and deduplicate by height and complete at the requested count. Cancellation detaches the subscription and cancels a pending snapshot read. | | Receivable stream snapshot hashes and live buffer | [ReceivableController][receivable-controller]; one collecting coroutine owns the snapshot hash set, one child collector sends to a bounded live channel | Live subscription starts before the database scan. Database hashes remain for delayed overlapping notifications; live-only hashes are never added. The stream drains database rows before consuming live updates. Backpressure suspends live producers, and cancellation or a failed scan cancels the live collector and channel. | | Local voting/signing jobs | [ElectionVoter][election-voter]; consensus state, mutexes, coroutine jobs | Provisional updates, cancellation, and committed final-vote publication have distinct lifecycles. Trace them before changing cleanup. | +| Bootstrap persistence observations and admission target | [BootstrapLoadMonitor][bootstrap-load]; synchronized recording and polling, with a volatile capacity snapshot | Insert completion records elapsed time, successful discovery persistence records attempted rows after commit, and non-cancelled failure records a reset. Each poll consumes its accumulated observations once. Metric registration and meter values do not drive admission. | The intended admission invariant is **one current election height per public key**, with competing hashes at that height. Validate it through the account snapshot, predecessor checks, and reachable transitions. Do not confuse it with @@ -210,9 +216,11 @@ Bootstrap can reach the same persistence and completion owners without traversin | --- | --- | --- | | Live rejection → dependency discovery | [DependencyDiscoverer][dependency-discoverer] tracks recoverable `TransactionRejected` events and qualifying final `VoteDropped` events; sufficient weight leads to a direct `DiscoveryQueue.queue` call | A rejected live attempt can later enter bootstrap recovery. | | Discovery → unchecked storage | [DiscoveryQueue][discovery-queue] admits to its bounded buffer, then publishes `TransactionDiscovered`. [BootstrapController.run][bootstrap-controller] drives [DiscoveryPersistenceWorker.persist][discovery-persistence] | `TransactionDiscovered` is a notification after admission, not the command that persists the transaction. | +| Unchecked persistence → adaptive admission | [UncheckedTransactionInserter][unchecked-inserter] records insert-only elapsed time in `finally`; [DiscoveryPersistenceWorker][discovery-persistence] records committed attempted rows or non-cancelled failure directly into [BootstrapLoadMonitor][bootstrap-load] | The next controller poll aggregates those observations. SQL affected rows and whole-save duration remain separate measurements; cancellation does not record a persistence failure. | | Unchecked storage → committed account | [UncheckedTransactionProcessor][unchecked-processor] directly calls `TransactionValidationManager.validate`; on success it calls `AccountService.add(BOOTSTRAP)` inside its resolution transaction | This path bypasses `TransactionReceived`, `TransactionValidated`, and normal election admission, while still producing after-commit `AccountUpdated`. | | Missing dependency → send discovery | On validation failure, [UncheckedTransactionProcessor][unchecked-processor] publishes `TransactionStuck` and skips later candidates for that public key in the batch. [SendDiscoverer][send-discoverer] consumes `TransactionStuck` and `TransactionDiscovered` | Applicable receive/open transactions trigger discovery of a missing send; follow this loop before treating unchecked work as terminally rejected. | | Bootstrap head → discovery | [LastDiscoverer.processVoteResponse][last-discoverer] calls `DiscoveryQueue.queue` after collected votes reach quorum | Expired or capacity-evicted collections are discarded without starting a live election. A later head advertisement can start another discovery attempt. | +| Scheduled head advertisement → peers | [BootstrapHeadBroadcaster.broadcastSample][head-broadcaster] samples stored transactions once per minute on a historical node with no unchecked backlog, then awaits connection-manager sends | Outbound advertising has its own mutex and lifecycle; it does not own inbound quorum collection. | | Shared completion → source-specific reaction | `AccountUpdated.source` distinguishes `BOOTSTRAP` and `ELECTION` | Most completion owners see both. `ElectionVoter` filters bootstrap updates from its committed-election final-vote path. | ## Trace an affected flow @@ -273,6 +281,7 @@ calls. Keep source links to files and named methods so routine line movement doe [connections]: src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt [transaction-types]: src/main/kotlin/cash/atto/node/transaction/Transaction.kt [transaction-controller]: src/main/kotlin/cash/atto/node/transaction/TransactionController.kt +[transaction-publication]: src/main/kotlin/cash/atto/node/transaction/TransactionPublicationController.kt [transaction-prioritizer]: src/main/kotlin/cash/atto/node/transaction/prioritization/TransactionPrioritizer.kt [transaction-queue]: src/main/kotlin/cash/atto/node/transaction/prioritization/TransactionQueue.kt [transaction-rebroadcaster]: src/main/kotlin/cash/atto/node/transaction/TransactionRebroadcaster.kt @@ -300,11 +309,17 @@ calls. Keep source links to files and named methods so routine line movement doe [vote-prioritizer]: src/main/kotlin/cash/atto/node/vote/prioritization/VotePrioritizer.kt [vote-rebroadcaster]: src/main/kotlin/cash/atto/node/vote/VoteRebroadcaster.kt [vote-cleaner]: src/main/kotlin/cash/atto/node/vote/VoteCleaner.kt +[vote-service]: src/main/kotlin/cash/atto/node/vote/VoteService.kt [vote-weighter]: src/main/kotlin/cash/atto/node/vote/weight/VoteWeighter.kt [dependency-discoverer]: src/main/kotlin/cash/atto/node/bootstrap/discovery/DependencyDiscoverer.kt [discovery-queue]: src/main/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryQueue.kt [discovery-persistence]: src/main/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryPersistenceWorker.kt [bootstrap-controller]: src/main/kotlin/cash/atto/node/bootstrap/BootstrapController.kt +[bootstrap-load]: src/main/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitor.kt +[unchecked-inserter]: src/main/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserter.kt [unchecked-processor]: src/main/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionProcessor.kt [last-discoverer]: src/main/kotlin/cash/atto/node/bootstrap/discovery/LastDiscoverer.kt +[head-broadcaster]: src/main/kotlin/cash/atto/node/bootstrap/discovery/BootstrapHeadBroadcaster.kt [send-discoverer]: src/main/kotlin/cash/atto/node/bootstrap/discovery/SendDiscoverer.kt + +[account-entry-controller]: src/main/kotlin/cash/atto/node/account/entry/AccountEntryController.kt diff --git a/build.gradle.kts b/build.gradle.kts index b1129349..1902ba45 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -139,6 +139,10 @@ tasks.withType { maxHeapSize = "4g" } +tasks.named("check") { + dependsOn("compileBenchmarkKotlin") +} + graalvmNative { binaries { named("main") { diff --git a/src/benchmark/kotlin/cash/atto/node/election/ElectionVoterBenchmark.kt b/src/benchmark/kotlin/cash/atto/node/election/ElectionVoterBenchmark.kt index 35238d2e..9ef2b957 100644 --- a/src/benchmark/kotlin/cash/atto/node/election/ElectionVoterBenchmark.kt +++ b/src/benchmark/kotlin/cash/atto/node/election/ElectionVoterBenchmark.kt @@ -18,11 +18,12 @@ import cash.atto.node.account.Account import cash.atto.node.account.AccountRepository import cash.atto.node.account.AccountUpdated import cash.atto.node.network.NetworkMessagePublisher +import cash.atto.node.network.guardian.ExpectedResponses import cash.atto.node.transaction.Transaction import cash.atto.node.transaction.TransactionSource -import cash.atto.node.vote.VoteRepository import cash.atto.node.vote.weight.VoteWeightProperties import cash.atto.node.vote.weight.VoteWeighter +import cash.atto.node.vote.weight.Weight import cash.atto.node.vote.weight.WeightRepository import cash.atto.node.vote.weight.WeightService import cash.atto.protocol.AttoNode @@ -35,15 +36,20 @@ import kotlinx.benchmark.TearDown import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.emptyFlow +import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.runBlocking import org.openjdk.jmh.annotations.Level import org.openjdk.jmh.annotations.Threads import org.springframework.context.ApplicationEvent import org.springframework.context.ApplicationEventPublisher +import org.springframework.transaction.ReactiveTransaction +import org.springframework.transaction.ReactiveTransactionManager +import org.springframework.transaction.TransactionDefinition +import reactor.core.publisher.Mono import java.lang.reflect.Proxy import java.net.URI +import java.time.Clock import java.time.Instant -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger import kotlin.time.Duration.Companion.milliseconds @@ -86,12 +92,13 @@ class ElectionVoterBenchmark { signer = signer, voteWeighter = voteWeighter, eventPublisher = EventPublisher(springPublisher), - messagePublisher = NetworkMessagePublisher(springPublisher), + messagePublisher = NetworkMessagePublisher(springPublisher, ExpectedResponses(Clock.systemUTC())), accountRepository = accountRepository, ) account = createAccount(node) } + // Measures event handling and scheduling; signing completes asynchronously. @Benchmark @Threads(100) fun processElectionFlow() = @@ -213,47 +220,36 @@ private fun createVoteWeighter(node: AttoNode): VoteWeighter { minimalRebroadcastWeight = "1" samplePeriodInDays = 1 } - val voteWeighter = - VoteWeighter( - thisNode = node, - properties = properties, - weightService = WeightService(interfaceProxy()), - voteRepository = interfaceProxy(), - genesisTransaction = createTransaction(node, 1U), - ) - - val weightMapField = VoteWeighter::class.java.getDeclaredField("weightMap") - weightMapField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val weightMap = weightMapField.get(voteWeighter) as ConcurrentHashMap - weightMap[node.publicKey] = AttoAmount.MAX - - return voteWeighter + val weight = Weight(node.publicKey, AttoAmount.MAX, Instant.now()) + return VoteWeighter( + thisNode = node, + properties = properties, + weightService = WeightService(weightRepository(weight), BenchmarkTransactionManager()), + genesisTransaction = createTransaction(node, 1U), + ).also { it.init() } } -private inline fun interfaceProxy(): T = +private fun weightRepository(weight: Weight): WeightRepository = Proxy .newProxyInstance( - T::class.java.classLoader, - arrayOf(T::class.java), + WeightRepository::class.java.classLoader, + arrayOf(WeightRepository::class.java), ) { proxy, method, args -> when (method.name) { - "toString" -> "${T::class.java.simpleName}Proxy" + "findCalculatedWeights" -> flowOf(weight) + "upsert", "deleteAllExcept" -> Unit + "toString" -> "BenchmarkWeightRepository" "hashCode" -> System.identityHashCode(proxy) "equals" -> proxy === args?.singleOrNull() - else -> defaultValue(method.returnType) + else -> error("Unexpected benchmark repository call: ${method.name}") } - } as T + } as WeightRepository -private fun defaultValue(returnType: Class<*>): Any? = - when (returnType) { - java.lang.Boolean.TYPE -> false - java.lang.Byte.TYPE -> 0.toByte() - java.lang.Short.TYPE -> 0.toShort() - java.lang.Integer.TYPE -> 0 - java.lang.Long.TYPE -> 0L - java.lang.Float.TYPE -> 0f - java.lang.Double.TYPE -> 0.0 - java.lang.Character.TYPE -> 0.toChar() - else -> null - } +private class BenchmarkTransactionManager : ReactiveTransactionManager { + override fun getReactiveTransaction(definition: TransactionDefinition?): Mono = + Mono.just(object : ReactiveTransaction {}) + + override fun commit(transaction: ReactiveTransaction): Mono = Mono.empty() + + override fun rollback(transaction: ReactiveTransaction): Mono = error("Unexpected benchmark initialization rollback") +} diff --git a/src/main/kotlin/cash/atto/node/account/entry/AccountEntryController.kt b/src/main/kotlin/cash/atto/node/account/entry/AccountEntryController.kt index 58b79347..4bb29ecf 100644 --- a/src/main/kotlin/cash/atto/node/account/entry/AccountEntryController.kt +++ b/src/main/kotlin/cash/atto/node/account/entry/AccountEntryController.kt @@ -19,12 +19,13 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse import io.swagger.v3.oas.annotations.tags.Tag import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.emitAll import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.merge import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onStart +import kotlinx.coroutines.flow.onSubscription import kotlinx.coroutines.flow.take import org.springframework.context.event.EventListener import org.springframework.http.HttpStatus @@ -94,24 +95,14 @@ class AccountEntryController( ) suspend fun stream( @PathVariable hash: AttoHash, - ): Flow { - val entryDatabaseFlow: Flow = - flow { - val transaction = repository.findById(hash) - if (transaction != null) { - emit(transaction.toAtto()) - } - } - - val entryFlow = - entryFlow - .filter { it.hash == hash } - - return merge(entryFlow, entryDatabaseFlow) + ): Flow = + entryFlow + .onSubscription { + repository.findById(hash)?.let { emit(it.toAtto()) } + }.filter { it.hash == hash } .onStart { logger.trace { "Started streaming $hash account entry" } } .onCompletion { logger.trace { "Stopped streaming $hash account entry" } } .take(1) - } @PostMapping("/accounts/entries/stream", produces = [MediaType.APPLICATION_NDJSON_VALUE]) @Operation( @@ -149,17 +140,14 @@ class AccountEntryController( if (it.value > Int.MAX_VALUE.toUInt()) Int.MAX_VALUE else it.value.toInt() } - val dbFlow = - repository - .findAsc(publicKey, range.fromHeight.value.toBigInteger(), range.toHeight.value.toBigInteger()) - .map { it.toAtto() } - - val liveFlow = - entryFlow - .filter { it.publicKey == publicKey } - .filter { it.height in fromHeight..toHeight } - - merge(liveFlow, dbFlow) + entryFlow + .onSubscription { + emitAll( + repository + .findAsc(publicKey, fromHeight.value.toBigInteger(), toHeight.value.toBigInteger()) + .map { it.toAtto() }, + ) + }.filter { it.publicKey == publicKey && it.height in fromHeight..toHeight } .sortByHeight(fromHeight) .take(expectedCount) } diff --git a/src/main/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitor.kt b/src/main/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitor.kt index 84c62718..2b261a6a 100644 --- a/src/main/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitor.kt +++ b/src/main/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitor.kt @@ -1,21 +1,16 @@ package cash.atto.node.bootstrap -import cash.atto.node.bootstrap.discovery.DiscoveryMetrics import cash.atto.node.bootstrap.discovery.DiscoveryProperties -import cash.atto.node.bootstrap.unchecked.UncheckedTransactionInserter import io.micrometer.core.instrument.DistributionSummary import io.micrometer.core.instrument.Gauge import io.micrometer.core.instrument.MeterRegistry -import io.micrometer.core.instrument.Timer import org.springframework.stereotype.Component import java.time.Duration -import java.util.concurrent.TimeUnit -import kotlin.math.roundToLong @Component class BootstrapLoadMonitor( private val properties: DiscoveryProperties, - private val meterRegistry: MeterRegistry, + meterRegistry: MeterRegistry, ) { private val targetRatioSummary = DistributionSummary @@ -23,16 +18,12 @@ class BootstrapLoadMonitor( .description("Unchecked persistence elapsed time divided by its proportional target") .register(meterRegistry) - private val insertTimer: Timer by lazy { - meterRegistry.get(UncheckedTransactionInserter.METRIC_NAME).timer() - } - @Volatile private var targetCapacity = properties.batchSize - private var observedInsertCount = 0L - private var observedInsertNanos = 0L - private var observedPersistedRows = 0L - private var observedFailureCount = 0L + private var pendingInsertCount = 0L + private var pendingInsertNanos = 0L + private var pendingPersistedRows = 0L + private var pendingFailureCount = 0L init { Gauge @@ -47,31 +38,31 @@ class BootstrapLoadMonitor( internal fun targetCapacity(): Int = targetCapacity @Synchronized - internal fun poll() { - val insertCount = insertTimer.count() - val insertNanos = insertTimer.totalTime(TimeUnit.NANOSECONDS).roundToLong() - val persistedRows = - meterRegistry - .find(DiscoveryMetrics.PERSISTED_METRIC_NAME) - .counters() - .sumOf { it.count() } - .roundToLong() - val failureCount = - meterRegistry - .get(DiscoveryMetrics.PERSISTENCE_FAILURE_METRIC_NAME) - .counter() - .count() - .roundToLong() + internal fun recordInsert(elapsedNanos: Long) { + pendingInsertCount++ + pendingInsertNanos += elapsedNanos + } - val completedInsertDelta = insertCount - observedInsertCount - val elapsedNanosDelta = maxOf(0L, insertNanos - observedInsertNanos) - val persistedRowsDelta = persistedRows - observedPersistedRows - val failureCountDelta = failureCount - observedFailureCount + @Synchronized + internal fun recordPersisted(rowCount: Int) { + pendingPersistedRows += rowCount + } - observedInsertCount = insertCount - observedInsertNanos = insertNanos - observedPersistedRows = persistedRows - observedFailureCount = failureCount + @Synchronized + internal fun recordFailure() { + pendingFailureCount++ + } + + @Synchronized + internal fun poll() { + val completedInsertDelta = pendingInsertCount + val elapsedNanosDelta = maxOf(0L, pendingInsertNanos) + val persistedRowsDelta = pendingPersistedRows + val failureCountDelta = pendingFailureCount + pendingInsertCount = 0 + pendingInsertNanos = 0 + pendingPersistedRows = 0 + pendingFailureCount = 0 if (failureCountDelta > 0) { targetCapacity = properties.batchSize @@ -81,7 +72,7 @@ class BootstrapLoadMonitor( return } check(completedInsertDelta > 0) { - "Persisted $persistedRowsDelta unchecked transactions without an insert timer measurement" + "Persisted $persistedRowsDelta unchecked transactions without an insert measurement" } val elapsed = Duration.ofNanos(elapsedNanosDelta) diff --git a/src/main/kotlin/cash/atto/node/bootstrap/discovery/BootstrapHeadBroadcaster.kt b/src/main/kotlin/cash/atto/node/bootstrap/discovery/BootstrapHeadBroadcaster.kt new file mode 100644 index 00000000..23390af0 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/bootstrap/discovery/BootstrapHeadBroadcaster.kt @@ -0,0 +1,44 @@ +package cash.atto.node.bootstrap.discovery + +import cash.atto.node.bootstrap.unchecked.UncheckedTransactionRepository +import cash.atto.node.network.BroadcastNetworkMessage +import cash.atto.node.network.BroadcastStrategy +import cash.atto.node.network.NodeConnectionManager +import cash.atto.node.transaction.TransactionRepository +import cash.atto.protocol.AttoBootstrapTransactionPush +import cash.atto.protocol.AttoNode +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component +import java.util.concurrent.TimeUnit + +@Component +class BootstrapHeadBroadcaster( + private val thisNode: AttoNode, + private val transactionRepository: TransactionRepository, + private val uncheckedTransactionRepository: UncheckedTransactionRepository, + private val nodeConnectionManager: NodeConnectionManager, +) { + private val mutex = Mutex() + + @Scheduled(fixedRate = 1, timeUnit = TimeUnit.MINUTES) + suspend fun broadcastSample() { + if (thisNode.isNotHistorical()) { + return + } + + mutex.withLock { + if (uncheckedTransactionRepository.count() > 0) { + return + } + + transactionRepository + .getLastSample(10) + .map { AttoBootstrapTransactionPush(it.toAttoTransaction()) } + .map { BroadcastNetworkMessage(BroadcastStrategy.EVERYONE, setOf(), it) } + .collect { nodeConnectionManager.send(it) } + } + } +} diff --git a/src/main/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryPersistenceWorker.kt b/src/main/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryPersistenceWorker.kt index b6fdca05..22eedbe7 100644 --- a/src/main/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryPersistenceWorker.kt +++ b/src/main/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryPersistenceWorker.kt @@ -1,6 +1,7 @@ package cash.atto.node.bootstrap.discovery import cash.atto.node.CacheSupport +import cash.atto.node.bootstrap.BootstrapLoadMonitor import cash.atto.node.bootstrap.unchecked.UncheckedTransactionService import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.CancellationException @@ -15,6 +16,7 @@ class DiscoveryPersistenceWorker( private val uncheckedTransactionService: UncheckedTransactionService, private val queue: DiscoveryQueue, private val metrics: DiscoveryMetrics, + private val loadMonitor: BootstrapLoadMonitor, ) : CacheSupport { private val logger = KotlinLogging.logger {} private val flushMutex = Mutex() @@ -48,6 +50,7 @@ class DiscoveryPersistenceWorker( } catch (e: Exception) { if (e !is CancellationException) { retryBatch = discoveries + loadMonitor.recordFailure() metrics.persistenceFailed() } throw e @@ -55,6 +58,7 @@ class DiscoveryPersistenceWorker( val elapsed = Duration.ofNanos(System.nanoTime() - startedAt) retryBatch = null + loadMonitor.recordPersisted(discoveries.size) metrics.persisted(discoveries) metrics.affectedRows(affectedRows) logger.info { "Saved ${discoveries.size} unchecked transactions in $elapsed" } diff --git a/src/main/kotlin/cash/atto/node/bootstrap/discovery/GapDiscoverer.kt b/src/main/kotlin/cash/atto/node/bootstrap/discovery/GapDiscoverer.kt index 4095bc45..b1d1e830 100644 --- a/src/main/kotlin/cash/atto/node/bootstrap/discovery/GapDiscoverer.kt +++ b/src/main/kotlin/cash/atto/node/bootstrap/discovery/GapDiscoverer.kt @@ -5,10 +5,10 @@ import cash.atto.commons.AttoPublicKey import cash.atto.node.CacheSupport import cash.atto.node.bootstrap.unchecked.GapView import cash.atto.node.bootstrap.unchecked.UncheckedTransactionRepository +import cash.atto.node.network.ConnectedPeers import cash.atto.node.network.DirectNetworkMessage import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.node.network.NodeConnected import cash.atto.node.network.NodeDisconnected import cash.atto.protocol.AttoTransactionStreamRequest import cash.atto.protocol.AttoTransactionStreamResponse @@ -22,7 +22,6 @@ import org.springframework.context.event.EventListener import org.springframework.stereotype.Component import java.net.URI import java.time.Clock -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit @Component @@ -33,9 +32,9 @@ class GapDiscoverer( private val discoveryProperties: DiscoveryProperties, meterRegistry: MeterRegistry, private val clock: Clock, + private val connectedPeers: ConnectedPeers, ) : CacheSupport { private val logger = KotlinLogging.logger {} - private val peers = ConcurrentHashMap.newKeySet() private val activeSessionCache = Caffeine .newBuilder() @@ -56,20 +55,12 @@ class GapDiscoverer( .description("Missing unchecked transaction ranges returned") .register(meterRegistry) - @EventListener - fun add(nodeEvent: NodeConnected) { - val node = nodeEvent.node - if (!node.isHistorical()) { - return - } - - peers += node.publicUri - } - @EventListener fun remove(nodeEvent: NodeDisconnected) { val publicUri = nodeEvent.node.publicUri - peers.remove(publicUri) + if (connectedPeers.isConnected(publicUri)) { + return + } activeSessions .values .filter { it.peer == publicUri } @@ -139,7 +130,13 @@ class GapDiscoverer( private fun availablePeers(): List { val activePeers = activeSessions.values.mapTo(mutableSetOf(), GapSession::peer) - return peers.filterNot(activePeers::contains) + return connectedPeers + .snapshot() + .asSequence() + .filter { it.node.isHistorical() } + .map { it.node.publicUri } + .filterNot(activePeers::contains) + .toList() } private fun remainingRequestBudget(): Int { @@ -164,7 +161,7 @@ class GapDiscoverer( discoveryQueue = discoveryQueue, ) activeSessions[view.publicKey] = session - if (!peers.contains(peer)) { + if (!connectedPeers.isConnected(peer)) { activeSessions.remove(view.publicKey, session) return 0 } diff --git a/src/main/kotlin/cash/atto/node/bootstrap/discovery/LastDiscoverer.kt b/src/main/kotlin/cash/atto/node/bootstrap/discovery/LastDiscoverer.kt index c18d9d48..b57d8357 100644 --- a/src/main/kotlin/cash/atto/node/bootstrap/discovery/LastDiscoverer.kt +++ b/src/main/kotlin/cash/atto/node/bootstrap/discovery/LastDiscoverer.kt @@ -5,43 +5,31 @@ import cash.atto.node.CacheSupport import cash.atto.node.account.AccountRepository import cash.atto.node.account.getByAlgorithmAndPublicKey import cash.atto.node.bootstrap.TransactionDiscovered -import cash.atto.node.bootstrap.unchecked.UncheckedTransactionRepository import cash.atto.node.election.ElectionVoter import cash.atto.node.election.TransactionElection -import cash.atto.node.network.BroadcastNetworkMessage -import cash.atto.node.network.BroadcastStrategy import cash.atto.node.network.DirectNetworkMessage import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.node.network.NodeConnectionManager import cash.atto.node.transaction.TransactionRepository import cash.atto.node.transaction.toTransaction import cash.atto.node.vote.convertion.VoteConverter import cash.atto.node.vote.weight.VoteWeighter import cash.atto.protocol.AttoBootstrapTransactionPush -import cash.atto.protocol.AttoNode import cash.atto.protocol.AttoVoteStreamCancel import cash.atto.protocol.AttoVoteStreamRequest import cash.atto.protocol.AttoVoteStreamResponse import com.github.benmanes.caffeine.cache.Caffeine import com.github.benmanes.caffeine.cache.Scheduler import io.github.oshai.kotlinlogging.KotlinLogging -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock import org.springframework.context.event.EventListener -import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit @Component class LastDiscoverer( - private val thisNode: AttoNode, private val accountRepository: AccountRepository, private val transactionRepository: TransactionRepository, - private val uncheckedTransactionRepository: UncheckedTransactionRepository, - private val nodeConnectionManager: NodeConnectionManager, private val networkMessagePublisher: NetworkMessagePublisher, private val discoveryQueue: DiscoveryQueue, private val voteConverter: VoteConverter, @@ -58,33 +46,12 @@ class LastDiscoverer( .build() .asMap() - private val mutex = Mutex() private val pushPermits = Semaphore(64) override fun clear() { transactionElectionMap.clear() } - @Scheduled(fixedRate = 1, timeUnit = TimeUnit.MINUTES) - suspend fun broadcastSample() { - if (thisNode.isNotHistorical()) { - return - } - - mutex.withLock { - if (uncheckedTransactionRepository.count() > 0) { - return - } - - val transactions = transactionRepository.getLastSample(10) - - transactions - .map { AttoBootstrapTransactionPush(it.toAttoTransaction()) } - .map { BroadcastNetworkMessage(BroadcastStrategy.EVERYONE, setOf(), it) } - .collect { nodeConnectionManager.send(it) } - } - } - @EventListener suspend fun processPush(message: InboundNetworkMessage) { if (!pushPermits.tryAcquire()) { diff --git a/src/main/kotlin/cash/atto/node/bootstrap/discovery/SendDiscoverer.kt b/src/main/kotlin/cash/atto/node/bootstrap/discovery/SendDiscoverer.kt index 63e5fca5..63849e40 100644 --- a/src/main/kotlin/cash/atto/node/bootstrap/discovery/SendDiscoverer.kt +++ b/src/main/kotlin/cash/atto/node/bootstrap/discovery/SendDiscoverer.kt @@ -1,7 +1,6 @@ package cash.atto.node.bootstrap.discovery import cash.atto.commons.AttoHash -import cash.atto.commons.AttoPublicKey import cash.atto.commons.ReceiveSupport import cash.atto.node.CacheSupport import cash.atto.node.DuplicateDetector @@ -10,11 +9,10 @@ import cash.atto.node.bootstrap.TransactionStuck import cash.atto.node.bootstrap.unchecked.UncheckedTransactionRepository import cash.atto.node.network.BroadcastNetworkMessage import cash.atto.node.network.BroadcastStrategy +import cash.atto.node.network.ConnectedPeers import cash.atto.node.network.DirectNetworkMessage import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.node.network.NodeConnected -import cash.atto.node.network.NodeDisconnected import cash.atto.node.transaction.Transaction import cash.atto.node.transaction.TransactionRejectionReason import cash.atto.node.transaction.toTransaction @@ -30,7 +28,6 @@ import org.springframework.context.event.EventListener import org.springframework.stereotype.Component import java.net.URI import java.time.Duration -import java.util.concurrent.ConcurrentHashMap import kotlin.time.Duration.Companion.minutes @Component @@ -39,6 +36,7 @@ class SendDiscoverer( private val discoveryQueue: DiscoveryQueue, private val uncheckedTransactionRepository: UncheckedTransactionRepository, meterRegistry: MeterRegistry, + private val connectedPeers: ConnectedPeers, ) : CacheSupport { private val logger = KotlinLogging.logger {} private val skippedForGap = @@ -47,8 +45,6 @@ class SendDiscoverer( .description("Send responses skipped because a higher unchecked transaction lets gap discovery recover them") .register(meterRegistry) - private val peers = ConcurrentHashMap() - private val unknownHashCache = Caffeine .newBuilder() @@ -60,18 +56,6 @@ class SendDiscoverer( private val duplicateDetector = DuplicateDetector(1.minutes) - @EventListener - fun add(nodeEvent: NodeConnected) { - val node = nodeEvent.node - peers[node.publicKey] = node.publicUri - } - - @EventListener - fun remove(nodeEvent: NodeDisconnected) { - val node = nodeEvent.node - peers.remove(node.publicKey) - } - @EventListener fun process(event: TransactionDiscovered) { process(event.reason, event.transaction, event.votes) @@ -107,7 +91,7 @@ class SendDiscoverer( val request = AttoTransactionRequest(block.sendHash) - val socketAddress = randomUri(votes) + val socketAddress = representativeUri(votes) val message = if (socketAddress != null) { DirectNetworkMessage(socketAddress, request) @@ -144,15 +128,14 @@ class SendDiscoverer( ) } - private fun randomUri(votes: Collection): URI? = - votes - .asSequence() - .map { v -> peers[v.publicKey] } - .filterNotNull() - .firstOrNull() + private fun representativeUri(votes: Collection): URI? { + val peers = connectedPeers.snapshot() + return votes.firstNotNullOfOrNull { vote -> + peers.firstOrNull { it.node.publicKey == vote.publicKey }?.node?.publicUri + } + } override fun clear() { - peers.clear() unknownHashCache.clear() } } diff --git a/src/main/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserter.kt b/src/main/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserter.kt index 1656b963..1a25fc42 100644 --- a/src/main/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserter.kt +++ b/src/main/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserter.kt @@ -2,6 +2,7 @@ package cash.atto.node.bootstrap.unchecked import cash.atto.commons.toBigInteger import cash.atto.commons.toJavaInstant +import cash.atto.node.bootstrap.BootstrapLoadMonitor import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.Timer import kotlinx.coroutines.reactive.awaitSingle @@ -17,6 +18,7 @@ import java.util.concurrent.ConcurrentHashMap class UncheckedTransactionInserter( private val databaseClient: DatabaseClient, private val meterRegistry: MeterRegistry, + private val loadMonitor: BootstrapLoadMonitor, ) { private val insertTimer = Timer @@ -75,7 +77,7 @@ class UncheckedTransactionInserter( .reduce(0L, Long::plus) }.awaitSingle() } finally { - sample.stop(insertTimer) + loadMonitor.recordInsert(sample.stop(insertTimer)) } } diff --git a/src/main/kotlin/cash/atto/node/election/Election.kt b/src/main/kotlin/cash/atto/node/election/Election.kt index 287d4cc4..bf042cc2 100644 --- a/src/main/kotlin/cash/atto/node/election/Election.kt +++ b/src/main/kotlin/cash/atto/node/election/Election.kt @@ -22,6 +22,7 @@ import io.github.oshai.kotlinlogging.KotlinLogging import io.micrometer.core.instrument.Gauge import io.micrometer.core.instrument.MeterRegistry import jakarta.annotation.PostConstruct +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.springframework.context.event.EventListener @@ -44,7 +45,8 @@ class Election( val ELECTION_STABILITY_MINIMAL_TIME: Duration = Duration.ofSeconds(5) } - private val mutex = Mutex() + // Suspend transitions keep cancellable acquisition; synchronous diagnostics and clear share the same guard. + private val stateLock = Mutex() private val publicKeyHeightElectionMap = HashMap() private val pendingConsensus = HashSet() @@ -64,9 +66,17 @@ class Election( .register(meterRegistry) } - fun getSize(): Int = publicKeyHeightElectionMap.size + fun getSize(): Int = + runBlocking { + stateLock.withLock { publicKeyHeightElectionMap.size } + } - fun getElections(): Map = publicKeyHeightElectionMap + fun getElections(): Map = + runBlocking { + stateLock.withLock { + publicKeyHeightElectionMap.mapValues { (_, election) -> election.snapshot() } + } + } @EventListener suspend fun start(event: TransactionValidated) { @@ -82,14 +92,14 @@ class Election( @EventListener suspend fun process(event: AccountUpdated) = - mutex.withLock { + stateLock.withLock { val transaction = event.transaction complete(transaction.publicKey, transaction.height, transaction.hash) } @EventListener suspend fun process(event: ElectionDiscarded) = - mutex.withLock { + stateLock.withLock { val account = event.account complete(account.publicKey, AttoHeight(account.height.toULong()), account.lastTransactionHash) } @@ -127,7 +137,7 @@ class Election( private suspend fun start( account: Account, transaction: Transaction, - ) = mutex.withLock { + ) = stateLock.withLock { val publicKeyHeight = transaction.toPublicKeyHeight() val confirmedHeight = confirmedHeights[transaction.publicKey] if (confirmedHeight != null && transaction.height <= confirmedHeight) { @@ -161,7 +171,7 @@ class Election( private suspend fun process( transaction: Transaction, vote: Vote, - ) = mutex.withLock { + ) = stateLock.withLock { val publicKeyHeight = transaction.toPublicKeyHeight() logger.trace { "Processing $vote" } @@ -219,7 +229,7 @@ class Election( @Scheduled(fixedRate = 5, timeUnit = TimeUnit.SECONDS) suspend fun notifyExpiring() = - mutex.withLock { + stateLock.withLock { val minimalTimestamp = Instant.now().minusSeconds(properties.expiringAfterTimeInSeconds!!) publicKeyHeightElectionMap @@ -235,7 +245,7 @@ class Election( @Scheduled(initialDelay = 1, fixedRate = 5, timeUnit = TimeUnit.SECONDS) suspend fun expireOld() = - mutex.withLock { + stateLock.withLock { val minimalTimestamp = Instant.now().minusSeconds(properties.expiredAfterTimeInSeconds!!) publicKeyHeightElectionMap @@ -254,12 +264,15 @@ class Election( } } - override fun clear() { - publicKeyHeightElectionMap.clear() - pendingConsensus.clear() - heightsByPublicKey.clear() - confirmedHeights.clear() - } + override fun clear() = + runBlocking { + stateLock.withLock { + publicKeyHeightElectionMap.clear() + pendingConsensus.clear() + heightsByPublicKey.clear() + confirmedHeights.clear() + } + } } private class VoterChange( @@ -276,6 +289,23 @@ class PublicKeyHeightElection( private val voterChangeMap = HashMap() private val latestVoteTimestampMap = HashMap() + internal fun snapshot(): PublicKeyHeightElection { + val minimalWeight = minimalConfirmationWeightProvider() + val weights = + transactionElectionMap.values + .flatMap { it.votes.values } + .associate { it.publicKey to voteWeightProvider(it) } + val snapshot = PublicKeyHeightElection(account, { minimalWeight }, { weights.getValue(it.publicKey) }) + transactionElectionMap.forEach { (hash, election) -> + val copy = TransactionElection(election.transaction, { minimalWeight }, { weights.getValue(it.publicKey) }) + copy.votes.putAll(election.votes) + snapshot.transactionElectionMap[hash] = copy + } + snapshot.voterChangeMap.putAll(voterChangeMap) + snapshot.latestVoteTimestampMap.putAll(latestVoteTimestampMap) + return snapshot + } + fun add(transaction: Transaction) { if (transactionElectionMap.containsKey(transaction.hash)) { return diff --git a/src/main/kotlin/cash/atto/node/election/ElectionInfoContributor.kt b/src/main/kotlin/cash/atto/node/election/ElectionInfoContributor.kt index 55ff1029..8a259f0e 100644 --- a/src/main/kotlin/cash/atto/node/election/ElectionInfoContributor.kt +++ b/src/main/kotlin/cash/atto/node/election/ElectionInfoContributor.kt @@ -1,23 +1,20 @@ package cash.atto.node.election -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.runBlocking import org.springframework.boot.actuate.info.Info import org.springframework.boot.actuate.info.InfoContributor import org.springframework.stereotype.Component -@ExperimentalCoroutinesApi @Component class ElectionInfoContributor( val election: Election, ) : InfoContributor { - // TODO: Improve contributor override fun contribute(builder: Info.Builder) { - val election = + val elections = election.getElections() + val details = mapOf( - "weights" to runBlocking { election.getElections() }, - "active" to runBlocking { election.getSize() }, + "weights" to elections, + "active" to elections.size, ) - builder.withDetail("elections", election) + builder.withDetail("elections", details) } } diff --git a/src/main/kotlin/cash/atto/node/network/ConnectedPeers.kt b/src/main/kotlin/cash/atto/node/network/ConnectedPeers.kt new file mode 100644 index 00000000..14438c6b --- /dev/null +++ b/src/main/kotlin/cash/atto/node/network/ConnectedPeers.kt @@ -0,0 +1,17 @@ +package cash.atto.node.network + +import cash.atto.protocol.AttoNode +import java.net.InetSocketAddress +import java.net.URI + +interface ConnectedPeers { + // Detached metadata; a peer may disconnect immediately after this query. + fun snapshot(): List + + fun isConnected(publicUri: URI): Boolean +} + +data class ConnectedPeer( + val node: AttoNode, + val socketAddress: InetSocketAddress, +) diff --git a/src/main/kotlin/cash/atto/node/network/NetworkInfoContributor.kt b/src/main/kotlin/cash/atto/node/network/NetworkInfoContributor.kt index 8c5b13f3..5ae74bca 100644 --- a/src/main/kotlin/cash/atto/node/network/NetworkInfoContributor.kt +++ b/src/main/kotlin/cash/atto/node/network/NetworkInfoContributor.kt @@ -1,46 +1,20 @@ package cash.atto.node.network -import cash.atto.protocol.AttoNode -import kotlinx.serialization.ExperimentalSerializationApi +import cash.atto.node.network.guardian.PeerBans import org.springframework.boot.actuate.info.Info import org.springframework.boot.actuate.info.InfoContributor -import org.springframework.context.event.EventListener import org.springframework.stereotype.Component -import java.net.InetAddress -import java.net.InetSocketAddress -import java.util.concurrent.ConcurrentHashMap -@OptIn(ExperimentalSerializationApi::class) @Component -class NetworkInfoContributor : InfoContributor { - private val paired = ConcurrentHashMap() - private val banned = ConcurrentHashMap.newKeySet() - - @EventListener - fun add(event: NodeConnected) { - paired[event.connectionSocketAddress] = event.node - } - - @EventListener - fun remove(event: NodeDisconnected) { - paired.remove(event.connectionSocketAddress) - } - - @EventListener - fun ban(event: NodeBanned) { - banned.add(event.address) - } - - @EventListener - fun unban(event: NodeUnbanned) { - banned.remove(event.address) - } - +class NetworkInfoContributor( + private val connectedPeers: ConnectedPeers, + private val peerBans: PeerBans, +) : InfoContributor { override fun contribute(builder: Info.Builder) { val network = mapOf( - "paired" to paired.values, - "banned" to banned.map { it.hostAddress }, + "paired" to connectedPeers.snapshot().map { it.node }, + "banned" to peerBans.snapshot().map { it.hostAddress }, ) builder.withDetail("network", network) } diff --git a/src/main/kotlin/cash/atto/node/network/NetworkMessage.kt b/src/main/kotlin/cash/atto/node/network/NetworkMessage.kt index dbcc0fec..aa2ecae9 100644 --- a/src/main/kotlin/cash/atto/node/network/NetworkMessage.kt +++ b/src/main/kotlin/cash/atto/node/network/NetworkMessage.kt @@ -1,5 +1,6 @@ package cash.atto.node.network +import cash.atto.node.network.guardian.ExpectedResponses import cash.atto.protocol.AttoMessage import cash.atto.protocol.AttoNode import io.github.oshai.kotlinlogging.KotlinLogging @@ -65,11 +66,16 @@ data class BroadcastNetworkMessage( @Component class NetworkMessagePublisher( private val publisher: ApplicationEventPublisher, + private val expectedResponses: ExpectedResponses, ) { private val logger = KotlinLogging.logger {} fun publish(message: NetworkMessage<*>) { logger.trace { "$message" } + // Reserve response credits before an asynchronous transport listener can send the request. + if (message is DirectNetworkMessage<*>) { + expectedResponses.record(message) + } publisher.publishEvent(message) } } diff --git a/src/main/kotlin/cash/atto/node/network/Node.kt b/src/main/kotlin/cash/atto/node/network/Node.kt index 91d18201..8036917e 100644 --- a/src/main/kotlin/cash/atto/node/network/Node.kt +++ b/src/main/kotlin/cash/atto/node/network/Node.kt @@ -11,12 +11,6 @@ data class NodeBanned( override val timestamp: Instant = Instant.now(), ) : Event -data class NodeConnected( - val connectionSocketAddress: InetSocketAddress, - val node: AttoNode, - override val timestamp: Instant = Instant.now(), -) : Event - data class NodeUnbanned( val address: InetAddress, override val timestamp: Instant = Instant.now(), diff --git a/src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt b/src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt index ce3a1e63..9f04a633 100644 --- a/src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt +++ b/src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt @@ -42,7 +42,8 @@ class NodeConnectionManager( private val thisNode: AttoNode, private val messagePublisher: NetworkMessagePublisher, private val eventPublisher: EventPublisher, -) : CacheSupport { +) : CacheSupport, + ConnectedPeers { private val logger = KotlinLogging.logger {} private val supervisorJob = SupervisorJob() @@ -92,7 +93,9 @@ class NodeConnectionManager( scope.cancel() } - fun isConnected(publicUri: URI): Boolean = connectionMap.containsKey(publicUri) + override fun snapshot(): List = connectionMap.values.map { ConnectedPeer(it.node, it.connectionInetSocketAddress) } + + override fun isConnected(publicUri: URI): Boolean = connectionMap.containsKey(publicUri) suspend fun manage( node: AttoNode, @@ -111,9 +114,7 @@ class NodeConnectionManager( try { connection .incomingFlow() - .onStart { - eventPublisher.publish(NodeConnected(connectionSocketAddress, node)) - }.onCompletion { + .onCompletion { val cause = it?.takeUnless { it is CancellationException } logger.trace(cause) { "Inbound message stream from ${node.publicUri} completed" } connectionMap.remove(publicUri, connection) @@ -127,7 +128,8 @@ class NodeConnectionManager( } if (message is AttoKeepAlive) { - connectionMap[publicUri] = connection + // Buffered frames from a replaced session must not restore its ownership. + connectionMap.replace(publicUri, connection, connection) } logger.trace { "Received from $publicUri $message ${it.toHex()}" } diff --git a/src/main/kotlin/cash/atto/node/network/guardian/ExpectedResponses.kt b/src/main/kotlin/cash/atto/node/network/guardian/ExpectedResponses.kt new file mode 100644 index 00000000..031e4280 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/network/guardian/ExpectedResponses.kt @@ -0,0 +1,67 @@ +package cash.atto.node.network.guardian + +import cash.atto.node.CacheSupport +import cash.atto.node.network.DirectNetworkMessage +import cash.atto.protocol.AttoTransactionStreamRequest +import org.springframework.stereotype.Component +import java.net.URI +import java.time.Clock +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap + +@Component +class ExpectedResponses( + private val clock: Clock, +) : CacheSupport { + private val responsesByPeer = ConcurrentHashMap() + + internal val peerCount: Int + get() = responsesByPeer.size + + fun record(message: DirectNetworkMessage<*>) { + if (message.expectedResponseCount == 0UL) { + return + } + val now = clock.instant() + val expiresAt = message.timestamp.plus(AttoTransactionStreamRequest.TIMEOUT) + if (!now.isBefore(expiresAt)) { + return + } + responsesByPeer.compute(message.publicUri) { _, credit -> + val remaining = credit?.takeIf { now.isBefore(it.expiresAt) } + Credit( + count = (remaining?.count ?: 0UL) + message.expectedResponseCount, + expiresAt = maxOf(remaining?.expiresAt ?: expiresAt, expiresAt), + ) + } + } + + fun consume(publicUri: URI): Boolean { + val now = clock.instant() + var expected = false + responsesByPeer.computeIfPresent(publicUri) { _, credit -> + if (!now.isBefore(credit.expiresAt)) { + return@computeIfPresent null + } + expected = true + if (credit.count == 1UL) null else credit.copy(count = credit.count - 1UL) + } + return expected + } + + fun removeExpired() { + val now = clock.instant() + responsesByPeer.keys.forEach { publicUri -> + responsesByPeer.computeIfPresent(publicUri) { _, credit -> credit.takeIf { now.isBefore(it.expiresAt) } } + } + } + + override fun clear() { + responsesByPeer.clear() + } + + private data class Credit( + val count: ULong, + val expiresAt: Instant, + ) +} diff --git a/src/main/kotlin/cash/atto/node/network/guardian/Guardian.kt b/src/main/kotlin/cash/atto/node/network/guardian/Guardian.kt index 61f95b45..f3ee71b5 100644 --- a/src/main/kotlin/cash/atto/node/network/guardian/Guardian.kt +++ b/src/main/kotlin/cash/atto/node/network/guardian/Guardian.kt @@ -2,16 +2,10 @@ package cash.atto.node.network.guardian import cash.atto.commons.AttoPublicKey import cash.atto.node.CacheSupport -import cash.atto.node.EventPublisher -import cash.atto.node.network.DirectNetworkMessage +import cash.atto.node.network.ConnectedPeers import cash.atto.node.network.InboundNetworkMessage -import cash.atto.node.network.NodeBanned -import cash.atto.node.network.NodeConnected -import cash.atto.node.network.NodeDisconnected -import cash.atto.node.network.NodeUnbanned import cash.atto.node.vote.weight.VoteWeighter import cash.atto.protocol.AttoNode -import cash.atto.protocol.AttoTransactionStreamRequest import com.github.benmanes.caffeine.cache.Caffeine import com.github.benmanes.caffeine.cache.Scheduler import io.github.oshai.kotlinlogging.KotlinLogging @@ -20,11 +14,7 @@ import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Service import java.net.InetAddress import java.net.InetSocketAddress -import java.net.URI -import java.time.Clock import java.time.Duration -import java.time.Instant -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger @@ -32,17 +22,16 @@ import java.util.concurrent.atomic.AtomicInteger class Guardian( private val thisNode: AttoNode, private val voteWeighter: VoteWeighter, - private val eventPublisher: EventPublisher, private val guardianProperties: GuardianProperties, - private val clock: Clock, + private val peerBans: PeerBans, + private val expectedResponses: ExpectedResponses, + private val connectedPeers: ConnectedPeers, ) : CacheSupport { private val logger = KotlinLogging.logger {} // Counting and window rotation share this lock so each hit belongs to exactly one window. private val statisticsLock = Any() private var statisticsMap = HashMap() - private val voterMap = ConcurrentHashMap() - private val banned = ConcurrentHashMap.newKeySet() private val connectionRequestCounters = Caffeine .newBuilder() @@ -52,14 +41,11 @@ class Guardian( .build() .asMap() - // Only outbound requests extend expiry; delayed delivery keeps the request's original deadline. - private val expectedResponseCountMap = ConcurrentHashMap() - @Volatile private var snapshot: Map = mapOf() fun requestInboundConnection(address: InetAddress): InboundConnectionDecision { - if (address in banned) { + if (peerBans.isBanned(address)) { return InboundConnectionDecision.Banned } @@ -78,27 +64,6 @@ class Guardian( return InboundConnectionDecision.Accepted } - @EventListener - fun count(message: DirectNetworkMessage<*>) { - if (message.expectedResponseCount == 0UL) { - return - } - - val now = clock.instant() - val expiresAt = message.timestamp.plus(AttoTransactionStreamRequest.TIMEOUT) - if (!now.isBefore(expiresAt)) { - return - } - - expectedResponseCountMap.compute(message.publicUri) { _, responses -> - val remaining = responses?.takeIf { now.isBefore(it.expiresAt) } - ExpectedResponses( - count = (remaining?.count ?: 0UL) + message.expectedResponseCount, - expiresAt = maxOf(remaining?.expiresAt ?: expiresAt, expiresAt), - ) - } - } - @EventListener fun count(message: InboundNetworkMessage<*>) { if (message.publicUri == thisNode.publicUri) { @@ -106,58 +71,16 @@ class Guardian( return } - val now = clock.instant() - var expected = false - expectedResponseCountMap.computeIfPresent(message.publicUri) { _, responses -> - if (!now.isBefore(responses.expiresAt)) { - return@computeIfPresent null - } - expected = true - if (responses.count == 1UL) { - null - } else { - responses.copy(count = responses.count - 1UL) - } - } - if (expected) { + if (expectedResponses.consume(message.publicUri)) { return } count(message.socketAddress) } - @EventListener - fun ban(event: NodeBanned) { - banned += event.address - } - - @EventListener - fun unban(event: NodeUnbanned) { - banned -= event.address - } - - @EventListener - fun add(nodeEvent: NodeConnected) { - if (nodeEvent.node.isNotVoter()) { - return - } - - voterMap[nodeEvent.connectionSocketAddress] = nodeEvent.node.publicKey - } - - @EventListener - fun remove(nodeEvent: NodeDisconnected) { - voterMap.remove(nodeEvent.connectionSocketAddress) - } - @Scheduled(fixedRate = 15, timeUnit = TimeUnit.SECONDS) @Synchronized fun guard() { - val now = clock.instant() - expectedResponseCountMap.keys.forEach { uri -> - expectedResponseCountMap.computeIfPresent(uri) { _, responses -> - responses.takeIf { now.isBefore(it.expiresAt) } - } - } + expectedResponses.removeExpired() val windowCounts = synchronized(statisticsLock) { val completed = statisticsMap @@ -167,7 +90,8 @@ class Guardian( snapshot = windowCounts val voterValues = - voterMap.entries + getVoters() + .entries .filter { voteWeighter.isAboveMinimalRebroadcastWeight(it.value) } .map { windowCounts[it.key] ?: 0UL } if (voterValues.isEmpty()) { @@ -192,7 +116,7 @@ class Guardian( .filter { it.value >= threshold } .forEach { (address, count) -> logger.info { "Banning $address. Received $count requests while median of voters is $median per second" } - eventPublisher.publish(NodeBanned(address)) + peerBans.ban(address) } } @@ -209,9 +133,11 @@ class Guardian( fun getSnapshot(): Map = snapshot.toMap() - fun getVoters(): Map = voterMap.toMap() - - internal fun expectedResponsePeerCount(): Int = expectedResponseCountMap.size + fun getVoters(): Map = + connectedPeers + .snapshot() + .filter { it.node.isVoter() } + .associate { it.socketAddress to it.node.publicKey } private fun countInboundConnection(address: InetAddress) { count(InetSocketAddress(address, 0)) @@ -228,16 +154,8 @@ class Guardian( statisticsMap.clear() } snapshot = mapOf() - voterMap.clear() - expectedResponseCountMap.clear() - banned.clear() connectionRequestCounters.clear() } - - private data class ExpectedResponses( - val count: ULong, - val expiresAt: Instant, - ) } sealed interface InboundConnectionDecision { diff --git a/src/main/kotlin/cash/atto/node/network/guardian/PeerBans.kt b/src/main/kotlin/cash/atto/node/network/guardian/PeerBans.kt new file mode 100644 index 00000000..7514d0d5 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/network/guardian/PeerBans.kt @@ -0,0 +1,53 @@ +package cash.atto.node.network.guardian + +import cash.atto.node.CacheSupport +import cash.atto.node.EventPublisher +import cash.atto.node.network.NodeBanned +import cash.atto.node.network.NodeUnbanned +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component +import java.net.InetAddress +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.concurrent.TimeUnit + +@Component +class PeerBans( + private val eventPublisher: EventPublisher, + private val clock: Clock, +) : CacheSupport { + private val logger = KotlinLogging.logger {} + private val bannedAt = HashMap() + + @Synchronized + fun ban(address: InetAddress) { + val now = clock.instant() + bannedAt[address] = maxOf(bannedAt[address] ?: now, now) + eventPublisher.publish(NodeBanned(address, now)) + } + + @Synchronized + fun isBanned(address: InetAddress): Boolean = bannedAt.containsKey(address) + + @Synchronized + fun snapshot(): Set = bannedAt.keys.toSet() + + @Scheduled(fixedRate = 1, timeUnit = TimeUnit.MINUTES) + @Synchronized + fun review() { + val now = clock.instant() + val expired = bannedAt.filterValues { Duration.between(it, now) >= Duration.ofHours(1) }.keys + expired.forEach { address -> + bannedAt.remove(address) + logger.info { "Unbanning $address after 1 hour" } + eventPublisher.publish(NodeUnbanned(address, now)) + } + } + + @Synchronized + override fun clear() { + bannedAt.clear() + } +} diff --git a/src/main/kotlin/cash/atto/node/network/guardian/Unbanner.kt b/src/main/kotlin/cash/atto/node/network/guardian/Unbanner.kt deleted file mode 100644 index 284cf8db..00000000 --- a/src/main/kotlin/cash/atto/node/network/guardian/Unbanner.kt +++ /dev/null @@ -1,42 +0,0 @@ -package cash.atto.node.network.guardian - -import cash.atto.node.EventPublisher -import cash.atto.node.network.NodeBanned -import cash.atto.node.network.NodeUnbanned -import io.github.oshai.kotlinlogging.KotlinLogging -import org.springframework.context.event.EventListener -import org.springframework.scheduling.annotation.Scheduled -import org.springframework.stereotype.Component -import java.net.InetAddress -import java.time.Duration -import java.time.Instant -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.TimeUnit - -@Component -class Unbanner( - private val eventPublisher: EventPublisher, -) { - private val logger = KotlinLogging.logger {} - - private val bannedAt = ConcurrentHashMap() - - @EventListener - fun track(event: NodeBanned) { - bannedAt[event.address] = event.timestamp - } - - @Scheduled(fixedRate = 1, timeUnit = TimeUnit.MINUTES) - fun review() { - val now = Instant.now() - val iterator = bannedAt.entries.iterator() - while (iterator.hasNext()) { - val (address, timestamp) = iterator.next() - if (Duration.between(timestamp, now) >= Duration.ofHours(1)) { - iterator.remove() - logger.info { "Unbanning $address after 1 hour" } - eventPublisher.publish(NodeUnbanned(address)) - } - } - } -} diff --git a/src/main/kotlin/cash/atto/node/transaction/TransactionController.kt b/src/main/kotlin/cash/atto/node/transaction/TransactionController.kt index 77223b04..93a58107 100644 --- a/src/main/kotlin/cash/atto/node/transaction/TransactionController.kt +++ b/src/main/kotlin/cash/atto/node/transaction/TransactionController.kt @@ -10,39 +10,27 @@ import cash.atto.commons.node.AccountHeightSearch import cash.atto.commons.node.HeightSearch import cash.atto.commons.spring.sortByHeight import cash.atto.commons.toBigInteger -import cash.atto.node.ApplicationProperties import cash.atto.node.account.AccountUpdated -import cash.atto.node.election.ElectionExpired -import cash.atto.node.election.ElectionRejected -import cash.atto.node.network.InboundNetworkMessage -import cash.atto.node.network.MessageSource -import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.protocol.AttoNode -import cash.atto.protocol.AttoTransactionPush import io.github.oshai.kotlinlogging.KotlinLogging import io.swagger.v3.oas.annotations.Operation -import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.media.Content import io.swagger.v3.oas.annotations.media.Schema import io.swagger.v3.oas.annotations.responses.ApiResponse import io.swagger.v3.oas.annotations.tags.Tag -import kotlinx.coroutines.CancellationException import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.emitAll import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.merge import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onStart +import kotlinx.coroutines.flow.onSubscription import kotlinx.coroutines.flow.take -import kotlinx.coroutines.withTimeout import org.springframework.context.event.EventListener import org.springframework.http.HttpStatus import org.springframework.http.MediaType import org.springframework.http.ResponseEntity -import org.springframework.http.server.reactive.ServerHttpRequest import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping @@ -51,10 +39,6 @@ import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController import org.springframework.web.server.ResponseStatusException -import java.net.InetSocketAddress -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger -import kotlin.time.Duration.Companion.seconds @RestController @RequestMapping @@ -66,114 +50,15 @@ import kotlin.time.Duration.Companion.seconds "See [payment failure recovery](https://atto.cash/docs/whitepaper/technical#payment-failure-recovery).", ) class TransactionController( - val applicationProperties: ApplicationProperties, - val thisNode: AttoNode, - val messagePublisher: NetworkMessagePublisher, val repository: TransactionRepository, ) { private val logger = KotlinLogging.logger {} - private val useXForwardedForKey = "X-FORWARDED-FOR" - private val transactionFlow = MutableSharedFlow() - private val pendingTransactionStreams = ConcurrentHashMap() @EventListener suspend fun process(accountUpdated: AccountUpdated) { - val transaction = accountUpdated.transaction.toAttoTransaction() - emitPending(transaction.hash, PendingTransactionResult.Confirmed(transaction)) - transactionFlow.emit(transaction) - } - - @EventListener - suspend fun process(rejection: TransactionRejected) { - val reason = rejection.reason.toString() - - if (rejection.reason == TransactionRejectionReason.ALREADY_CONFIRMED) { - emitPending( - rejection.transaction.hash, - PendingTransactionResult.Confirmed(rejection.transaction.toAttoTransaction()), - ) - } else { - emitPending( - rejection.transaction.hash, - PendingTransactionResult.Failed( - ResponseStatusException( - HttpStatus.BAD_REQUEST, - "Transaction rejected due to $reason: ${rejection.message}", - ), - ), - ) - } - } - - @EventListener - suspend fun process(rejection: ElectionRejected) { - val hash = rejection.transaction.hash - val pending = pendingTransactionStreams[hash] ?: return - val result = - try { - val confirmed = repository.findById(hash) - if (confirmed != null) { - PendingTransactionResult.Confirmed(confirmed.toAttoTransaction()) - } else { - PendingTransactionResult.Failed( - ResponseStatusException(HttpStatus.CONFLICT, "Transaction rejected by local election"), - ) - } - } catch (exception: CancellationException) { - throw exception - } catch (exception: Exception) { - logger.error(exception) { "Failed to resolve rejected transaction $hash" } - PendingTransactionResult.Failed( - ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Transaction confirmation lookup failed"), - ) - } - - // Confirmation can complete this attempt while the repository read is suspended. - if (pendingTransactionStreams.remove(hash, pending)) { - pending.flow.emit(result) - } - } - - @EventListener - suspend fun process(failure: TransactionValidationFailed) { - emitPending( - failure.transaction.hash, - PendingTransactionResult.Failed( - ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Transaction validation failed"), - ), - ) - } - - @EventListener - suspend fun process(expired: ElectionExpired) { - val reason = "ELECTION_EXPIRED" - val message = "Election took too long" - emitPending( - expired.transaction.hash, - PendingTransactionResult.Failed( - ResponseStatusException( - HttpStatus.SERVICE_UNAVAILABLE, - "Transaction rejected due to $reason: $message", - ), - ), - ) - } - - @EventListener - suspend fun process(dropped: TransactionDropped) { - val reason = "TRANSACTION_DROPPED" - val message = "Transaction queue dropped the transaction" - emitPending( - dropped.transaction.hash, - PendingTransactionResult.Failed( - ResponseStatusException( - HttpStatus.TOO_MANY_REQUESTS, - "Transaction rejected due to $reason: $message", - ), - ), - ) + transactionFlow.emit(accountUpdated.transaction.toAttoTransaction()) } @Operation( @@ -235,24 +120,14 @@ class TransactionController( ) suspend fun stream( @PathVariable hash: AttoHash, - ): Flow { - val transactionDatabaseFlow: Flow = - flow { - val transaction = repository.findById(hash) - if (transaction != null) { - emit(transaction.toAttoTransaction()) - } - } - - val transactionFlow = - transactionFlow - .filter { it.hash == hash } - - return merge(transactionFlow, transactionDatabaseFlow) + ): Flow = + transactionFlow + .onSubscription { + repository.findById(hash)?.let { emit(it.toAttoTransaction()) } + }.filter { it.hash == hash } .onStart { logger.trace { "Started streaming $hash transaction" } } .onCompletion { logger.trace { "Stopped streaming $hash transaction" } } .take(1) - } @PostMapping("/accounts/transactions/stream", produces = [MediaType.APPLICATION_NDJSON_VALUE]) @Operation( @@ -290,17 +165,14 @@ class TransactionController( if (it.value > Int.MAX_VALUE.toUInt()) Int.MAX_VALUE else it.value.toInt() } - val dbFlow = - repository - .findAsc(publicKey, range.fromHeight.value.toBigInteger(), range.toHeight.value.toBigInteger()) - .map { it.toAttoTransaction() } - - val liveFlow = - transactionFlow - .filter { it.block.publicKey == publicKey } - .filter { it.height in fromHeight..toHeight } - - merge(liveFlow, dbFlow) + transactionFlow + .onSubscription { + emitAll( + repository + .findAsc(publicKey, fromHeight.value.toBigInteger(), toHeight.value.toBigInteger()) + .map { it.toAttoTransaction() }, + ) + }.filter { it.block.publicKey == publicKey && it.height in fromHeight..toHeight } .sortByHeight(fromHeight) .take(expectedCount) } @@ -341,257 +213,4 @@ class TransactionController( val search = HeightSearch(listOf(transactionSearch)) return streamMultiple(search) } - - @PostMapping("/transactions", consumes = [MediaType.APPLICATION_JSON_VALUE]) - @Operation( - summary = "Publish a transaction", - description = - "Validates and publishes the signed transaction, then waits for local confirmation and returns the confirmed transaction. " + - "The confirmation wait has a 40-second timeout. After a timeout or lost response, inspect the original hash and " + - "account history before creating another payment; confirmation may already have happened. " + - "A confirmed Send creates a receivable; a separate Open or Receive adds it to the recipient's spendable balance.", - responses = [ - ApiResponse( - responseCode = "200", - description = "The confirmed transaction, including an already confirmed result when recognized by the node.", - content = [ - Content( - schema = Schema(implementation = AttoTransaction::class), - ), - ], - ), - ApiResponse( - responseCode = "400", - description = "Invalid transaction, wrong network, invalid proxy-header configuration, or transaction rejection.", - content = [Content()], - ), - ApiResponse( - responseCode = "409", - description = "The local election rejected this publication attempt and the hash was not found in local storage.", - content = [Content()], - ), - ApiResponse( - responseCode = "429", - description = "The local transaction queue dropped the transaction. Check its hash before retrying the payment.", - content = [Content()], - ), - ApiResponse( - responseCode = "503", - description = "The local election expired. This does not prove that the transaction failed on every node.", - content = [Content()], - ), - ], - ) - suspend fun publish( - @RequestBody transaction: AttoTransaction, - request: ServerHttpRequest, - @Parameter( - description = - "When true, reuse a stored confirmation or an existing pending publication for the same transaction hash. " + - "This does not deduplicate payment attempts with different transaction hashes.", - ) - @RequestParam(defaultValue = "false", required = false) deduplicate: Boolean = false, - ): AttoTransaction = publishAndConfirm(transaction, request, deduplicate) - - private suspend fun publishAndConfirm( - transaction: AttoTransaction, - request: ServerHttpRequest, - deduplicate: Boolean, - ): AttoTransaction { - logger.debug { "Received $transaction" } - validate(transaction) - - if (deduplicate) { - repository.findById(transaction.hash)?.let { - return it.toAttoTransaction() - } - } - - val socketAddress = getSocketAddress(transaction, request) - val pending = registerPending(transaction.hash) - - try { - if (!deduplicate || pending.created) { - publish(transaction, socketAddress) - } - - return awaitPending(pending.stream) - } finally { - removePending(transaction.hash, pending.stream) - } - } - - private suspend fun validate(transaction: AttoTransaction) { - if (!transaction.isValid()) { - logger.debug { "Invalid! $transaction" } - throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid transaction") - } - - if (transaction.block.network != thisNode.network) { - logger.debug { "Invalid network! $transaction" } - throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid transaction network") - } - } - - private fun getSocketAddress( - transaction: AttoTransaction, - request: ServerHttpRequest, - ): InetSocketAddress { - val ips = request.headers[useXForwardedForKey] ?: listOf() - val remoteAddress = request.remoteAddress!! - - return if (!applicationProperties.useXForwardedFor) { - remoteAddress - } else if (ips.isNotEmpty()) { - InetSocketAddress.createUnresolved(ips[0], remoteAddress.port) - } else { - logger.debug { "X-Forwarded-For header is empty. Are you sure you are behind a load balancer? $transaction" } - throw ResponseStatusException( - HttpStatus.BAD_REQUEST, - "X-Forwarded-For header is empty. Are you sure you are behind a load balancer?", - ) - } - } - - private fun publish( - transaction: AttoTransaction, - socketAddress: InetSocketAddress, - ) { - messagePublisher.publish( - InboundNetworkMessage( - MessageSource.REST, - thisNode.publicUri, - socketAddress, - AttoTransactionPush(transaction), - ), - ) - } - - @PostMapping( - "/transactions/stream", - consumes = [MediaType.APPLICATION_JSON_VALUE], - produces = [MediaType.APPLICATION_NDJSON_VALUE], - ) - @Operation( - summary = "Publish a transaction and stream its confirmation", - description = - "Uses the same publication and local-confirmation operation as POST /transactions, " + - "emitting one confirmed transaction as NDJSON. " + - "The confirmation wait has a 40-second timeout. Inspect the emitted transaction, not only the initial HTTP status. " + - "Errors after response commitment can terminate the stream; do not assume every timeout returns HTTP 504. " + - "After an interrupted stream, inspect the original hash and account history before creating another payment.", - responses = [ - ApiResponse( - responseCode = "200", - description = "One confirmed transaction emitted as NDJSON.", - content = [ - Content( - schema = Schema(implementation = AttoTransaction::class), - ), - ], - ), - ApiResponse( - responseCode = "400", - description = "Invalid or rejected transaction, or invalid proxy-header configuration, before response commitment.", - content = [Content()], - ), - ApiResponse( - responseCode = "409", - description = "The local election rejected this publication attempt before response commitment.", - content = [Content()], - ), - ApiResponse( - responseCode = "429", - description = "The local transaction queue dropped the transaction before response commitment.", - content = [Content()], - ), - ApiResponse( - responseCode = "503", - description = "The local election expired before response commitment.", - content = [Content()], - ), - ], - ) - suspend fun publishAndStream( - @RequestBody transaction: AttoTransaction, - request: ServerHttpRequest, - @Parameter( - description = - "When true, reuse a stored confirmation or an existing pending publication for the same transaction hash. " + - "This does not deduplicate payment attempts with different transaction hashes.", - ) - @RequestParam(defaultValue = "false", required = false) deduplicate: Boolean = false, - ): Flow = - flow { - emit(publishAndConfirm(transaction, request, deduplicate)) - } - - private suspend fun awaitPending(stream: PendingTransactionStream): AttoTransaction = - withTimeout(40.seconds) { - stream - .flow - .map { - when (it) { - is PendingTransactionResult.Confirmed -> it.transaction - is PendingTransactionResult.Failed -> throw it.failure - } - }.first() - } - - private fun registerPending(hash: AttoHash): PendingTransactionRegistration { - var created = false - var stream: PendingTransactionStream? = null - - pendingTransactionStreams.compute(hash) { _, existing -> - val value = - existing ?: PendingTransactionStream().also { - created = true - } - value.waiters.incrementAndGet() - stream = value - value - } - - return PendingTransactionRegistration(stream!!, created) - } - - private fun removePending( - hash: AttoHash, - stream: PendingTransactionStream, - ) { - if (stream.waiters.decrementAndGet() != 0) { - return - } - - pendingTransactionStreams.computeIfPresent(hash) { _, existing -> - existing.takeUnless { it === stream } - } - } - - private suspend fun emitPending( - hash: AttoHash, - result: PendingTransactionResult, - ) { - pendingTransactionStreams.remove(hash)?.flow?.emit(result) - } - - private class PendingTransactionStream { - val flow = MutableSharedFlow(replay = 1) - val waiters = AtomicInteger() - } - - private class PendingTransactionRegistration( - val stream: PendingTransactionStream, - val created: Boolean, - ) - - private sealed interface PendingTransactionResult { - data class Confirmed( - val transaction: AttoTransaction, - ) : PendingTransactionResult - - data class Failed( - val failure: Throwable, - ) : PendingTransactionResult - } } diff --git a/src/main/kotlin/cash/atto/node/transaction/TransactionNetworkProvider.kt b/src/main/kotlin/cash/atto/node/transaction/TransactionNetworkProvider.kt index 088f7616..4f63c0d6 100644 --- a/src/main/kotlin/cash/atto/node/transaction/TransactionNetworkProvider.kt +++ b/src/main/kotlin/cash/atto/node/transaction/TransactionNetworkProvider.kt @@ -1,10 +1,9 @@ package cash.atto.node.transaction +import cash.atto.node.network.ConnectedPeers import cash.atto.node.network.DirectNetworkMessage import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.node.network.NodeConnected -import cash.atto.node.network.NodeDisconnected import cash.atto.protocol.AttoNode import cash.atto.protocol.AttoTransactionRequest import cash.atto.protocol.AttoTransactionResponse @@ -14,9 +13,7 @@ import kotlinx.coroutines.flow.toList import kotlinx.coroutines.sync.Mutex import org.springframework.context.event.EventListener import org.springframework.stereotype.Component -import java.net.URI import java.time.Clock -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Semaphore @Component @@ -25,34 +22,22 @@ class TransactionNetworkProvider( private val transactionRepository: TransactionRepository, private val networkMessagePublisher: NetworkMessagePublisher, private val clock: Clock, + private val connectedPeers: ConnectedPeers, ) { companion object { private const val MAX_CONCURRENT_FINDS = 64 } - private val peers = ConcurrentHashMap.newKeySet() private val findPermits = Semaphore(MAX_CONCURRENT_FINDS) private val streamMutex = Mutex() - @EventListener - fun add(nodeEvent: NodeConnected) { - val node = nodeEvent.node - peers.add(node.publicUri) - } - - @EventListener - fun remove(nodeEvent: NodeDisconnected) { - val node = nodeEvent.node - peers.remove(node.publicUri) - } - @EventListener suspend fun find(message: InboundNetworkMessage) { if (thisNode.isNotHistorical()) { return } - if (!peers.contains(message.publicUri)) { + if (!connectedPeers.isConnected(message.publicUri)) { return } @@ -83,7 +68,7 @@ class TransactionNetworkProvider( } try { - if (!peers.contains(message.publicUri) || message.isExpired()) { + if (!connectedPeers.isConnected(message.publicUri) || message.isExpired()) { return } @@ -101,7 +86,7 @@ class TransactionNetworkProvider( } for (transaction in transactions) { - if (!peers.contains(message.publicUri)) { + if (!connectedPeers.isConnected(message.publicUri)) { break } val response = AttoTransactionStreamResponse(transaction.toAttoTransaction()) diff --git a/src/main/kotlin/cash/atto/node/transaction/TransactionPublicationController.kt b/src/main/kotlin/cash/atto/node/transaction/TransactionPublicationController.kt new file mode 100644 index 00000000..efd515b6 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/transaction/TransactionPublicationController.kt @@ -0,0 +1,415 @@ +package cash.atto.node.transaction + +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoTransaction +import cash.atto.node.ApplicationProperties +import cash.atto.node.account.AccountUpdated +import cash.atto.node.election.ElectionExpired +import cash.atto.node.election.ElectionRejected +import cash.atto.node.network.InboundNetworkMessage +import cash.atto.node.network.MessageSource +import cash.atto.node.network.NetworkMessagePublisher +import cash.atto.protocol.AttoNode +import cash.atto.protocol.AttoTransactionPush +import io.github.oshai.kotlinlogging.KotlinLogging +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.media.Content +import io.swagger.v3.oas.annotations.media.Schema +import io.swagger.v3.oas.annotations.responses.ApiResponse +import io.swagger.v3.oas.annotations.tags.Tag +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.withTimeout +import org.springframework.context.event.EventListener +import org.springframework.http.HttpStatus +import org.springframework.http.MediaType +import org.springframework.http.server.reactive.ServerHttpRequest +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController +import org.springframework.web.server.ResponseStatusException +import java.net.InetSocketAddress +import java.util.concurrent.ConcurrentHashMap +import kotlin.time.Duration.Companion.seconds + +@RestController +@RequestMapping +@Tag( + name = "Transactions", + description = + "Submit or query transaction blocks. After an interrupted publication request, inspect the original transaction hash " + + "and account history before creating another payment. A timeout does not prove that the transaction failed. " + + "See [payment failure recovery](https://atto.cash/docs/whitepaper/technical#payment-failure-recovery).", +) +class TransactionPublicationController( + val applicationProperties: ApplicationProperties, + val thisNode: AttoNode, + val messagePublisher: NetworkMessagePublisher, + val repository: TransactionRepository, +) { + private val logger = KotlinLogging.logger {} + + private val useXForwardedForKey = "X-FORWARDED-FOR" + + private val pendingTransactionStreams = ConcurrentHashMap() + + @EventListener + suspend fun process(accountUpdated: AccountUpdated) { + val transaction = accountUpdated.transaction.toAttoTransaction() + emitPending(transaction.hash, PendingTransactionResult.Confirmed(transaction)) + } + + @EventListener + suspend fun process(rejection: TransactionRejected) { + val reason = rejection.reason.toString() + + if (rejection.reason == TransactionRejectionReason.ALREADY_CONFIRMED) { + emitPending( + rejection.transaction.hash, + PendingTransactionResult.Confirmed(rejection.transaction.toAttoTransaction()), + ) + } else { + emitPending( + rejection.transaction.hash, + PendingTransactionResult.Failed( + ResponseStatusException( + HttpStatus.BAD_REQUEST, + "Transaction rejected due to $reason: ${rejection.message}", + ), + ), + ) + } + } + + @EventListener + suspend fun process(rejection: ElectionRejected) { + val hash = rejection.transaction.hash + val pending = pendingTransactionStreams[hash] ?: return + val result = + try { + val confirmed = repository.findById(hash) + if (confirmed != null) { + PendingTransactionResult.Confirmed(confirmed.toAttoTransaction()) + } else { + PendingTransactionResult.Failed( + ResponseStatusException(HttpStatus.CONFLICT, "Transaction rejected by local election"), + ) + } + } catch (exception: CancellationException) { + throw exception + } catch (exception: Exception) { + logger.error(exception) { "Failed to resolve rejected transaction $hash" } + PendingTransactionResult.Failed( + ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Transaction confirmation lookup failed"), + ) + } + + // Confirmation can complete this attempt while the repository read is suspended. + if (pendingTransactionStreams.remove(hash, pending)) { + pending.flow.emit(result) + } + } + + @EventListener + suspend fun process(failure: TransactionValidationFailed) { + emitPending( + failure.transaction.hash, + PendingTransactionResult.Failed( + ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Transaction validation failed"), + ), + ) + } + + @EventListener + suspend fun process(expired: ElectionExpired) { + val reason = "ELECTION_EXPIRED" + val message = "Election took too long" + emitPending( + expired.transaction.hash, + PendingTransactionResult.Failed( + ResponseStatusException( + HttpStatus.SERVICE_UNAVAILABLE, + "Transaction rejected due to $reason: $message", + ), + ), + ) + } + + @EventListener + suspend fun process(dropped: TransactionDropped) { + val reason = "TRANSACTION_DROPPED" + val message = "Transaction queue dropped the transaction" + emitPending( + dropped.transaction.hash, + PendingTransactionResult.Failed( + ResponseStatusException( + HttpStatus.TOO_MANY_REQUESTS, + "Transaction rejected due to $reason: $message", + ), + ), + ) + } + + @PostMapping("/transactions", consumes = [MediaType.APPLICATION_JSON_VALUE]) + @Operation( + summary = "Publish a transaction", + description = + "Validates and publishes the signed transaction, then waits for local confirmation and returns the confirmed transaction. " + + "The confirmation wait has a 40-second timeout. After a timeout or lost response, inspect the original hash and " + + "account history before creating another payment; confirmation may already have happened. " + + "A confirmed Send creates a receivable; a separate Open or Receive adds it to the recipient's spendable balance.", + responses = [ + ApiResponse( + responseCode = "200", + description = "The confirmed transaction, including an already confirmed result when recognized by the node.", + content = [ + Content( + schema = Schema(implementation = AttoTransaction::class), + ), + ], + ), + ApiResponse( + responseCode = "400", + description = "Invalid transaction, wrong network, invalid proxy-header configuration, or transaction rejection.", + content = [Content()], + ), + ApiResponse( + responseCode = "409", + description = "The local election rejected this publication attempt and the hash was not found in local storage.", + content = [Content()], + ), + ApiResponse( + responseCode = "429", + description = "The local transaction queue dropped the transaction. Check its hash before retrying the payment.", + content = [Content()], + ), + ApiResponse( + responseCode = "503", + description = "The local election expired. This does not prove that the transaction failed on every node.", + content = [Content()], + ), + ], + ) + suspend fun publish( + @RequestBody transaction: AttoTransaction, + request: ServerHttpRequest, + @Parameter( + description = + "When true, reuse a stored confirmation or an existing pending publication for the same transaction hash. " + + "This does not deduplicate payment attempts with different transaction hashes.", + ) + @RequestParam(defaultValue = "false", required = false) deduplicate: Boolean = false, + ): AttoTransaction = publishAndConfirm(transaction, request, deduplicate) + + private suspend fun publishAndConfirm( + transaction: AttoTransaction, + request: ServerHttpRequest, + deduplicate: Boolean, + ): AttoTransaction { + logger.debug { "Received $transaction" } + validate(transaction) + + if (deduplicate) { + repository.findById(transaction.hash)?.let { + return it.toAttoTransaction() + } + } + + val socketAddress = getSocketAddress(transaction, request) + val pending = registerPending(transaction.hash) + + try { + if (!deduplicate || pending.created) { + publish(transaction, socketAddress) + } + + return awaitPending(pending.stream) + } finally { + removePending(transaction.hash, pending.stream) + } + } + + private suspend fun validate(transaction: AttoTransaction) { + if (!transaction.isValid()) { + logger.debug { "Invalid! $transaction" } + throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid transaction") + } + + if (transaction.block.network != thisNode.network) { + logger.debug { "Invalid network! $transaction" } + throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid transaction network") + } + } + + private fun getSocketAddress( + transaction: AttoTransaction, + request: ServerHttpRequest, + ): InetSocketAddress { + val ips = request.headers[useXForwardedForKey] ?: listOf() + val remoteAddress = request.remoteAddress!! + + return if (!applicationProperties.useXForwardedFor) { + remoteAddress + } else if (ips.isNotEmpty()) { + InetSocketAddress.createUnresolved(ips[0], remoteAddress.port) + } else { + logger.debug { "X-Forwarded-For header is empty. Are you sure you are behind a load balancer? $transaction" } + throw ResponseStatusException( + HttpStatus.BAD_REQUEST, + "X-Forwarded-For header is empty. Are you sure you are behind a load balancer?", + ) + } + } + + private fun publish( + transaction: AttoTransaction, + socketAddress: InetSocketAddress, + ) { + messagePublisher.publish( + InboundNetworkMessage( + MessageSource.REST, + thisNode.publicUri, + socketAddress, + AttoTransactionPush(transaction), + ), + ) + } + + @PostMapping( + "/transactions/stream", + consumes = [MediaType.APPLICATION_JSON_VALUE], + produces = [MediaType.APPLICATION_NDJSON_VALUE], + ) + @Operation( + summary = "Publish a transaction and stream its confirmation", + description = + "Uses the same publication and local-confirmation operation as POST /transactions, " + + "emitting one confirmed transaction as NDJSON. " + + "The confirmation wait has a 40-second timeout. Inspect the emitted transaction, not only the initial HTTP status. " + + "Errors after response commitment can terminate the stream; do not assume every timeout returns HTTP 504. " + + "After an interrupted stream, inspect the original hash and account history before creating another payment.", + responses = [ + ApiResponse( + responseCode = "200", + description = "One confirmed transaction emitted as NDJSON.", + content = [ + Content( + schema = Schema(implementation = AttoTransaction::class), + ), + ], + ), + ApiResponse( + responseCode = "400", + description = "Invalid or rejected transaction, or invalid proxy-header configuration, before response commitment.", + content = [Content()], + ), + ApiResponse( + responseCode = "409", + description = "The local election rejected this publication attempt before response commitment.", + content = [Content()], + ), + ApiResponse( + responseCode = "429", + description = "The local transaction queue dropped the transaction before response commitment.", + content = [Content()], + ), + ApiResponse( + responseCode = "503", + description = "The local election expired before response commitment.", + content = [Content()], + ), + ], + ) + suspend fun publishAndStream( + @RequestBody transaction: AttoTransaction, + request: ServerHttpRequest, + @Parameter( + description = + "When true, reuse a stored confirmation or an existing pending publication for the same transaction hash. " + + "This does not deduplicate payment attempts with different transaction hashes.", + ) + @RequestParam(defaultValue = "false", required = false) deduplicate: Boolean = false, + ): Flow = + flow { + emit(publishAndConfirm(transaction, request, deduplicate)) + } + + private suspend fun awaitPending(stream: PendingTransactionStream): AttoTransaction = + withTimeout(40.seconds) { + stream + .flow + .map { + when (it) { + is PendingTransactionResult.Confirmed -> it.transaction + is PendingTransactionResult.Failed -> throw it.failure + } + }.first() + } + + private fun registerPending(hash: AttoHash): PendingTransactionRegistration { + var created = false + var stream: PendingTransactionStream? = null + + pendingTransactionStreams.compute(hash) { _, existing -> + val value = + existing ?: PendingTransactionStream().also { + created = true + } + value.waiters++ + stream = value + value + } + + return PendingTransactionRegistration(stream!!, created) + } + + private fun removePending( + hash: AttoHash, + stream: PendingTransactionStream, + ) { + pendingTransactionStreams.computeIfPresent(hash) { _, existing -> + if (existing === stream) { + existing.waiters-- + existing.takeIf { it.waiters > 0 } + } else { + existing + } + } + } + + private suspend fun emitPending( + hash: AttoHash, + result: PendingTransactionResult, + ) { + pendingTransactionStreams.remove(hash)?.flow?.emit(result) + } + + private class PendingTransactionStream { + val flow = MutableSharedFlow(replay = 1) + + // Registration and removal update this count under the per-hash map computation. + var waiters = 0 + } + + private class PendingTransactionRegistration( + val stream: PendingTransactionStream, + val created: Boolean, + ) + + private sealed interface PendingTransactionResult { + data class Confirmed( + val transaction: AttoTransaction, + ) : PendingTransactionResult + + data class Failed( + val failure: Throwable, + ) : PendingTransactionResult + } +} diff --git a/src/main/kotlin/cash/atto/node/vote/VoteService.kt b/src/main/kotlin/cash/atto/node/vote/VoteService.kt index c8f78ee0..68f7d1fb 100644 --- a/src/main/kotlin/cash/atto/node/vote/VoteService.kt +++ b/src/main/kotlin/cash/atto/node/vote/VoteService.kt @@ -4,8 +4,6 @@ import cash.atto.node.CacheSupport import cash.atto.node.DemandDrivenWorker import jakarta.annotation.PreDestroy import org.springframework.stereotype.Service -import java.util.concurrent.ConcurrentLinkedDeque -import java.util.concurrent.atomic.AtomicInteger @Service class VoteService( @@ -15,13 +13,15 @@ class VoteService( private const val BATCH_SIZE = 1_000 } - private val buffer = ConcurrentLinkedDeque() - private val bufferDepth = AtomicInteger() + private val bufferLock = Any() + private val buffer = ArrayDeque() + private var clearGeneration = 0L private val worker = DemandDrivenWorker("vote-service", drain = ::drain) fun enqueue(vote: Vote) { - buffer.addLast(vote) - bufferDepth.incrementAndGet() + synchronized(bufferLock) { + buffer.addLast(vote) + } worker.request() } @@ -30,14 +30,17 @@ class VoteService( return } - votes.forEach { buffer.addLast(it) } - bufferDepth.addAndGet(votes.size) + synchronized(bufferLock) { + buffer.addAll(votes) + } worker.request() } override fun clear() { - buffer.clear() - bufferDepth.set(0) + synchronized(bufferLock) { + buffer.clear() + clearGeneration++ + } } @PreDestroy @@ -45,48 +48,35 @@ class VoteService( worker.cancel() } - fun getBufferSize(): Int = bufferDepth.get() + fun getBufferSize(): Int = synchronized(bufferLock) { buffer.size } private suspend fun drain() { - while (bufferDepth.get() > 0) { - flushBatch(BATCH_SIZE) - } - } - - private suspend fun saveAll(votes: Collection): List { - val distinctVotes = votes.distinctBy { it.signature } - - if (distinctVotes.isEmpty()) { - return distinctVotes - } - - voteRepository.insertIgnoreAll(distinctVotes) - - return distinctVotes - } - - private suspend fun flushBatch(size: Int): Int { - val votes = drainBatch(size) - if (votes.isEmpty()) { - return 0 + while (true) { + val batch = drainBatch() ?: return + try { + voteRepository.insertIgnoreAll(batch.votes.distinctBy { it.signature }) + } catch (exception: Exception) { + synchronized(bufferLock) { + if (batch.clearGeneration == clearGeneration) { + batch.votes.asReversed().forEach(buffer::addFirst) + } + } + throw exception + } } - - saveAll(votes) - return votes.size } - private fun drainBatch(size: Int): List { - val votes = mutableListOf() - - for (i in 1..size) { - val vote = buffer.pollFirst() ?: break - votes += vote - } - - if (votes.isNotEmpty()) { - bufferDepth.addAndGet(-votes.size) + private fun drainBatch(): PendingBatch? = + synchronized(bufferLock) { + if (buffer.isEmpty()) { + return@synchronized null + } + val votes = List(minOf(BATCH_SIZE, buffer.size)) { buffer.removeFirst() } + PendingBatch(votes, clearGeneration) } - return votes - } + private data class PendingBatch( + val votes: List, + val clearGeneration: Long, + ) } diff --git a/src/main/kotlin/cash/atto/node/vote/keeping/VoteKeeper.kt b/src/main/kotlin/cash/atto/node/vote/keeping/VoteKeeper.kt index 60c2c30d..0b2e6638 100644 --- a/src/main/kotlin/cash/atto/node/vote/keeping/VoteKeeper.kt +++ b/src/main/kotlin/cash/atto/node/vote/keeping/VoteKeeper.kt @@ -2,18 +2,16 @@ package cash.atto.node.vote.keeping import cash.atto.commons.AttoAlgorithm import cash.atto.commons.AttoHash -import cash.atto.commons.AttoPublicKey import cash.atto.commons.AttoSignedVote import cash.atto.commons.AttoSigner import cash.atto.commons.AttoVote import cash.atto.commons.toAttoVersion import cash.atto.commons.toBigInteger import cash.atto.node.CacheSupport +import cash.atto.node.network.ConnectedPeers import cash.atto.node.network.DirectNetworkMessage import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.node.network.NodeConnected -import cash.atto.node.network.NodeDisconnected import cash.atto.node.vote.MissingVote import cash.atto.node.vote.Vote import cash.atto.node.vote.VoteRepository @@ -27,9 +25,7 @@ import com.github.benmanes.caffeine.cache.Scheduler import org.springframework.context.event.EventListener import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component -import java.net.URI import java.time.Duration -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit @Component @@ -40,13 +36,12 @@ class VoteKeeper( private val voteService: VoteService, private val networkMessagePublisher: NetworkMessagePublisher, private val signer: AttoSigner, + private val connectedPeers: ConnectedPeers, ) : CacheSupport { companion object { internal val cacheExpiration = Duration.ofSeconds(10) } - private val peers = ConcurrentHashMap() - private val missingVoteMap = Caffeine .newBuilder() @@ -55,21 +50,6 @@ class VoteKeeper( .build() .asMap() - @EventListener - fun add(nodeEvent: NodeConnected) { - val node = nodeEvent.node - if (!node.isHistorical()) { - return - } - peers[node.publicKey] = node.publicUri - } - - @EventListener - fun remove(nodeEvent: NodeDisconnected) { - val node = nodeEvent.node - peers.remove(node.publicKey) - } - @EventListener suspend fun process(message: InboundNetworkMessage) { val response = message.payload @@ -104,7 +84,13 @@ class VoteKeeper( return } - val representative = peers[missingVote.representativePublicKey] ?: return + val representative = + connectedPeers + .snapshot() + .firstOrNull { + it.node.isHistorical() && it.node.publicKey == missingVote.representativePublicKey + }?.node + ?.publicUri ?: return missingVoteMap.computeIfAbsent(missingVote) { val voteRequest = AttoVoteRequest(AttoAlgorithm.V1, missingVote.lastTransactionHash) val message = DirectNetworkMessage(representative, voteRequest) @@ -133,7 +119,6 @@ class VoteKeeper( } override fun clear() { - peers.clear() missingVoteMap.clear() } } diff --git a/src/test/kotlin/cash/atto/node/OpenAPITest.kt b/src/test/kotlin/cash/atto/node/OpenAPITest.kt index f993c0ee..1ab381a6 100644 --- a/src/test/kotlin/cash/atto/node/OpenAPITest.kt +++ b/src/test/kotlin/cash/atto/node/OpenAPITest.kt @@ -20,13 +20,26 @@ class OpenAPITest { var managementPort: Int = 0 @Test - fun `GET OpenAPI docs are reachable`() { - webTestClient - .get() - .uri("/v3/api-docs") - .exchange() + fun `OpenAPI exposes transaction query and publication endpoints`() { + // Given + val request = webTestClient.get().uri("/v3/api-docs") + + // When + val response = request.exchange() + + // Then + response .expectStatus() .isOk + .expectBody() + .jsonPath("$.paths['/transactions'].post.responses['200']") + .exists() + .jsonPath("$.paths['/transactions/stream'].post.responses['200']") + .exists() + .jsonPath("$.paths['/transactions/stream'].get.responses['200']") + .exists() + .jsonPath("$.paths['/transactions/{hash}'].get.responses['200']") + .exists() } @Test diff --git a/src/test/kotlin/cash/atto/node/account/entry/AccountEntryControllerQueryTest.kt b/src/test/kotlin/cash/atto/node/account/entry/AccountEntryControllerQueryTest.kt new file mode 100644 index 00000000..b2f025ca --- /dev/null +++ b/src/test/kotlin/cash/atto/node/account/entry/AccountEntryControllerQueryTest.kt @@ -0,0 +1,159 @@ +package cash.atto.node.account.entry + +import cash.atto.commons.AttoAccountEntry +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoAmount +import cash.atto.commons.AttoBlockType +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoPublicKey +import cash.atto.commons.toAttoHeight +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +@OptIn(ExperimentalCoroutinesApi::class) +class AccountEntryControllerQueryTest { + private val publicKey = AttoPublicKey(ByteArray(32) { 1 }) + private val repository = mockk() + private val controller = AccountEntryController(repository) + + @Test + fun `hash stream subscribes before lookup and retains a concurrent committed result`() = + runTest { + // Given + val value = sample(2) + val finishRead = CompletableDeferred() + lateinit var publication: Deferred + coEvery { repository.findById(value.hash) } coAnswers { + publication = backgroundScope.async(start = CoroutineStart.UNDISPATCHED) { publish(value) } + finishRead.await() + null + } + val result = backgroundScope.async { controller.stream(value.hash).first() } + + // When + runCurrent() + + // Then + assertFalse(publication.isCompleted) + assertFalse(result.isCompleted) + + // When + finishRead.complete(Unit) + runCurrent() + + // Then + assertEquals(value.toAtto(), result.await()) + publication.await() + } + + @Test + fun `height stream orders snapshot and overlapping live updates and completes its range`() = + runTest { + // Given + val first = sample(2) + val second = sample(3) + val third = sample(4) + val finishRead = CompletableDeferred() + lateinit var overlap: Deferred + coEvery { repository.findAsc(any(), any(), any()) } coAnswers { + overlap = backgroundScope.async(start = CoroutineStart.UNDISPATCHED) { publish(second) } + flow { + emit(first) + emit(second) + finishRead.await() + } + } + val values = mutableListOf() + val collection = + backgroundScope.launch { + controller.stream(publicKey, 2UL.toAttoHeight(), 4UL.toAttoHeight()).toList(values) + } + + // When + runCurrent() + + // Then + assertFalse(overlap.isCompleted) + assertEquals(listOf(first, second).map { it.toAtto() }, values) + + // When + finishRead.complete(Unit) + runCurrent() + overlap.await() + publish(third) + runCurrent() + + // Then + collection.join() + assertTrue(collection.isCompleted) + assertEquals(listOf(first, second, third).map { it.toAtto() }, values) + } + + @Test + fun `cancelling a blocked snapshot cancels the query and releases the live publisher`() = + runTest { + // Given + val value = sample(2) + val cancelled = CompletableDeferred() + coEvery { repository.findAsc(any(), any(), any()) } returns + flow { + try { + awaitCancellation() + } finally { + cancelled.complete(Unit) + } + } + val collection = + backgroundScope.launch { + controller.stream(publicKey, 2UL.toAttoHeight(), 3UL.toAttoHeight()).toList() + } + runCurrent() + val publication = backgroundScope.async { publish(value) } + runCurrent() + assertFalse(publication.isCompleted) + + // When + collection.cancelAndJoin() + runCurrent() + + // Then + publication.await() + assertTrue(cancelled.isCompleted) + } + + private suspend fun publish(value: AccountEntry) { + controller.process(AccountEntrySaved(value)) + } + + private fun sample(height: Int): AccountEntry = + AccountEntry( + hash = AttoHash(ByteArray(32) { height.toByte() }), + algorithm = AttoAlgorithm.V1, + publicKey = publicKey, + height = height.toULong().toAttoHeight(), + blockType = AttoBlockType.RECEIVE, + subjectAlgorithm = AttoAlgorithm.V1, + subjectPublicKey = publicKey, + previousBalance = AttoAmount((height - 1).toULong()), + balance = AttoAmount(height.toULong()), + timestamp = Instant.EPOCH.plusSeconds(height.toLong()), + ) +} diff --git a/src/test/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitorTest.kt b/src/test/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitorTest.kt index cd9885c0..76e32c27 100644 --- a/src/test/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitorTest.kt +++ b/src/test/kotlin/cash/atto/node/bootstrap/BootstrapLoadMonitorTest.kt @@ -1,10 +1,6 @@ package cash.atto.node.bootstrap -import cash.atto.node.bootstrap.discovery.DiscoveryMetrics import cash.atto.node.bootstrap.discovery.DiscoveryProperties -import cash.atto.node.bootstrap.unchecked.UncheckedTransactionInserter -import io.micrometer.core.instrument.Counter -import io.micrometer.core.instrument.Timer import io.micrometer.core.instrument.simple.SimpleMeterRegistry import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertThrows @@ -147,6 +143,83 @@ class BootstrapLoadMonitorTest { assertEquals(2L, fixture.targetRatio.count()) } + @Test + fun `observations are consumed once and do not require persistence meters`() { + // Given + val fixture = fixture() + fixture.completeInsertWithoutPolling(500, Duration.ofMillis(250)) + + // When + fixture.monitor.poll() + fixture.monitor.poll() + + // Then + assertEquals(1_500, fixture.monitor.targetCapacity()) + assertEquals(1L, fixture.targetRatio.count()) + assertEquals(0.5, fixture.targetRatio.totalAmount()) + assertEquals( + 0, + fixture.registry + .find("transactions.unchecked.insert") + .timers() + .size, + ) + assertEquals( + 0, + fixture.registry + .find("transactions.discovery.persisted") + .counters() + .size, + ) + } + + @Test + fun `failure consumes accumulated success and elapsed before the next poll`() { + // Given + val fixture = fixture() + fixture.completeInsert(1_000, Duration.ofMillis(500)) + fixture.completeInsertWithoutPolling(1_000, Duration.ofMillis(500)) + fixture.failInsert(Duration.ofSeconds(10)) + + // When + fixture.monitor.poll() + + // Then + assertEquals(1_000, fixture.monitor.targetCapacity()) + assertEquals(1L, fixture.targetRatio.count()) + + // When + fixture.completeInsert(500, Duration.ofMillis(250)) + + // Then + assertEquals(1_500, fixture.monitor.targetCapacity()) + assertEquals(2L, fixture.targetRatio.count()) + assertEquals(1.0, fixture.targetRatio.totalAmount()) + } + + @Test + fun `uncommitted insert elapsed is aggregated only until the next poll`() { + // Given + val fixture = fixture() + fixture.monitor.recordInsert(Duration.ofSeconds(10).toNanos()) + + // When + fixture.monitor.poll() + + // Then + assertEquals(1_000, fixture.monitor.targetCapacity()) + assertEquals(0L, fixture.targetRatio.count()) + + // When + fixture.monitor.recordInsert(Duration.ofMillis(250).toNanos()) + fixture.completeInsertWithoutPolling(500, Duration.ofMillis(250)) + fixture.monitor.poll() + + // Then + assertEquals(1_500, fixture.monitor.targetCapacity()) + assertEquals(1.0, fixture.targetRatio.totalAmount()) + } + @Test fun `persistence throughput target must be positive`() { // Given @@ -168,25 +241,9 @@ class BootstrapLoadMonitorTest { ): Fixture { val properties = properties(capacity, batchSize).also(DiscoveryProperties::validate) val registry = SimpleMeterRegistry() - val timer = - Timer - .builder(UncheckedTransactionInserter.METRIC_NAME) - .register(registry) - val persistedCounter = - Counter - .builder(DiscoveryMetrics.PERSISTED_METRIC_NAME) - .tag("source", "test") - .register(registry) - val failureCounter = - Counter - .builder(DiscoveryMetrics.PERSISTENCE_FAILURE_METRIC_NAME) - .register(registry) return Fixture( monitor = BootstrapLoadMonitor(properties, registry), registry = registry, - timer = timer, - persistedCounter = persistedCounter, - failureCounter = failureCounter, ) } @@ -203,9 +260,6 @@ class BootstrapLoadMonitorTest { private data class Fixture( val monitor: BootstrapLoadMonitor, val registry: SimpleMeterRegistry, - val timer: Timer, - val persistedCounter: Counter, - val failureCounter: Counter, ) { val targetRatio = registry.get(BootstrapLoadMonitor.TARGET_RATIO_METRIC_NAME).summary() @@ -221,13 +275,13 @@ class BootstrapLoadMonitorTest { attemptedRows: Int, elapsed: Duration, ) { - timer.record(elapsed) - persistedCounter.increment(attemptedRows.toDouble()) + monitor.recordInsert(elapsed.toNanos()) + monitor.recordPersisted(attemptedRows) } fun failInsert(elapsed: Duration) { - timer.record(elapsed) - failureCounter.increment() + monitor.recordInsert(elapsed.toNanos()) + monitor.recordFailure() } } } diff --git a/src/test/kotlin/cash/atto/node/bootstrap/discovery/BootstrapHeadBroadcasterTest.kt b/src/test/kotlin/cash/atto/node/bootstrap/discovery/BootstrapHeadBroadcasterTest.kt new file mode 100644 index 00000000..c2363c60 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/bootstrap/discovery/BootstrapHeadBroadcasterTest.kt @@ -0,0 +1,140 @@ +package cash.atto.node.bootstrap.discovery + +import cash.atto.commons.AttoTransaction +import cash.atto.node.bootstrap.unchecked.UncheckedTransactionRepository +import cash.atto.node.network.BroadcastNetworkMessage +import cash.atto.node.network.BroadcastStrategy +import cash.atto.node.network.NodeConnectionManager +import cash.atto.node.transaction.Transaction +import cash.atto.node.transaction.TransactionRepository +import cash.atto.protocol.AttoBootstrapTransactionPush +import cash.atto.protocol.AttoNode +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +@OptIn(ExperimentalCoroutinesApi::class) +class BootstrapHeadBroadcasterTest { + @Test + fun `non historical node does not query or broadcast heads`() = + runTest { + // Given + val fixture = fixture(historical = false) + + // When + fixture.broadcaster.broadcastSample() + + // Then + coVerify(exactly = 0) { fixture.uncheckedRepository.count() } + coVerify(exactly = 0) { fixture.transactionRepository.getLastSample(any()) } + coVerify(exactly = 0) { fixture.connections.send(any>()) } + } + + @Test + fun `unchecked backlog prevents a head sample query`() = + runTest { + // Given + val fixture = fixture(uncheckedCount = 1) + + // When + fixture.broadcaster.broadcastSample() + + // Then + coVerify(exactly = 1) { fixture.uncheckedRepository.count() } + coVerify(exactly = 0) { fixture.transactionRepository.getLastSample(any()) } + coVerify(exactly = 0) { fixture.connections.send(any>()) } + } + + @Test + fun `historical node broadcasts each sampled head to everyone`() = + runTest { + // Given + val fixture = fixture() + val transactions = listOf(mockk(), mockk()) + val sample = transactions.map { transaction -> mockk { every { toAttoTransaction() } returns transaction } } + coEvery { fixture.transactionRepository.getLastSample(10) } returns flowOf(*sample.toTypedArray()) + val messages = mutableListOf>() + coEvery { fixture.connections.send(any>()) } coAnswers { + messages += firstArg>() + } + + // When + fixture.broadcaster.broadcastSample() + + // Then + assertEquals(transactions, messages.map { (it.payload as AttoBootstrapTransactionPush).transaction }) + assertEquals(listOf(BroadcastStrategy.EVERYONE, BroadcastStrategy.EVERYONE), messages.map { it.strategy }) + assertTrue(messages.all { it.exceptions.isEmpty() }) + coVerify(exactly = 1) { fixture.transactionRepository.getLastSample(10) } + } + + @Test + fun `broadcast awaits sending and serializes overlapping sample requests`() = + runTest { + // Given + val fixture = fixture() + val transaction = mockk { every { toAttoTransaction() } returns mockk() } + coEvery { fixture.transactionRepository.getLastSample(10) } returns flowOf(transaction) + val sendStarted = CompletableDeferred() + val releaseSend = CompletableDeferred() + coEvery { fixture.connections.send(any>()) } coAnswers { + sendStarted.complete(Unit) + releaseSend.await() + } + + // When + val first = async { fixture.broadcaster.broadcastSample() } + sendStarted.await() + val second = async { fixture.broadcaster.broadcastSample() } + runCurrent() + + // Then + assertFalse(first.isCompleted) + assertFalse(second.isCompleted) + coVerify(exactly = 1) { fixture.transactionRepository.getLastSample(10) } + + // When + releaseSend.complete(Unit) + first.await() + second.await() + + // Then + coVerify(exactly = 2) { fixture.transactionRepository.getLastSample(10) } + coVerify(exactly = 2) { fixture.connections.send(any>()) } + } + + private fun fixture( + historical: Boolean = true, + uncheckedCount: Long = 0, + ): Fixture { + val node = mockk { every { isNotHistorical() } returns !historical } + val transactionRepository = mockk() + val uncheckedRepository = mockk() + coEvery { uncheckedRepository.count() } returns uncheckedCount + val connections = mockk() + return Fixture( + BootstrapHeadBroadcaster(node, transactionRepository, uncheckedRepository, connections), + transactionRepository, + uncheckedRepository, + connections, + ) + } + + private data class Fixture( + val broadcaster: BootstrapHeadBroadcaster, + val transactionRepository: TransactionRepository, + val uncheckedRepository: UncheckedTransactionRepository, + val connections: NodeConnectionManager, + ) +} diff --git a/src/test/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryQueueTest.kt b/src/test/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryQueueTest.kt index 81dc796b..86a21d07 100644 --- a/src/test/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryQueueTest.kt +++ b/src/test/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryQueueTest.kt @@ -253,6 +253,9 @@ class DiscoveryQueueTest { ) assertEquals(0.0, fixture.gauge("transactions.discovery.backlog.depth")) assertEquals(1.0, fixture.counter("transactions.discovery.persistence.failures")) + verify(exactly = 1) { fixture.loadMonitor.recordFailure() } + verify(exactly = 1) { fixture.loadMonitor.recordPersisted(2) } + verify(exactly = 1) { fixture.loadMonitor.recordPersisted(1) } } @Test @@ -274,6 +277,8 @@ class DiscoveryQueueTest { // Then assertEquals(0.0, fixture.counter("transactions.discovery.persistence.failures")) + verify(exactly = 0) { fixture.loadMonitor.recordFailure() } + verify(exactly = 0) { fixture.loadMonitor.recordPersisted(any()) } } @Test @@ -449,6 +454,7 @@ class DiscoveryQueueTest { assertFalse(fixture.queue.isAtCapacity()) assertEquals(2, fixture.queue.remainingTargetCapacity()) assertEquals(0.0, fixture.gauge("transactions.discovery.backlog.depth")) + verify(exactly = 0) { fixture.loadMonitor.recordPersisted(any()) } // When releaseSave.complete(Unit) @@ -458,6 +464,26 @@ class DiscoveryQueueTest { assertFalse(fixture.queue.isAtCapacity()) assertEquals(2, fixture.queue.remainingTargetCapacity()) assertEquals(0.0, fixture.gauge("transactions.discovery.backlog.depth")) + verify(exactly = 1) { fixture.loadMonitor.recordPersisted(2) } + } + + @Test + fun `committed duplicate rows credit attempted discoveries instead of affected rows`() = + runTest { + // Given + val service = mockk() + coEvery { service.save(any()) } returns 0L + val fixture = fixture(service) + fixture.queue.queue(event(45), DiscoverySource.HEAD) + + // When + val persisted = fixture.worker.persist() + + // Then + assertEquals(1, persisted) + verify(exactly = 1) { fixture.loadMonitor.recordPersisted(1) } + assertEquals(1.0, fixture.counter("transactions.discovery.persisted", "source", "head")) + assertEquals(0.0, fixture.counter("transactions.discovery.affected")) } @Test @@ -512,6 +538,7 @@ class DiscoveryQueueTest { uncheckedTransactionService = service, queue = queue, metrics = metrics, + loadMonitor = loadMonitor, ) return Fixture( queue, @@ -523,7 +550,7 @@ class DiscoveryQueueTest { } private fun loadMonitor(discoveryCapacity: AtomicInteger): BootstrapLoadMonitor = - mockk { + mockk(relaxed = true) { every { targetCapacity() } answers { diff --git a/src/test/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryTestController.kt b/src/test/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryTestController.kt index db2e3d6a..6112cbc2 100644 --- a/src/test/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryTestController.kt +++ b/src/test/kotlin/cash/atto/node/bootstrap/discovery/DiscoveryTestController.kt @@ -11,7 +11,7 @@ import org.springframework.web.bind.annotation.RestController class DiscoveryTestController( val discoveryPersistenceWorker: DiscoveryPersistenceWorker, val gapDiscoverer: GapDiscoverer, - val lastDiscoverer: LastDiscoverer, + val bootstrapHeadBroadcaster: BootstrapHeadBroadcaster, val uncheckedTransactionProcessor: UncheckedTransactionProcessor, ) { @PostMapping("/gap") @@ -23,7 +23,7 @@ class DiscoveryTestController( @PostMapping("/last") @Operation(description = "Start broadcast of last transactions") suspend fun last() { - lastDiscoverer.broadcastSample() + bootstrapHeadBroadcaster.broadcastSample() } @PostMapping("/flush") diff --git a/src/test/kotlin/cash/atto/node/bootstrap/discovery/GapDiscovererTest.kt b/src/test/kotlin/cash/atto/node/bootstrap/discovery/GapDiscovererTest.kt index 8d34c019..41bb816d 100644 --- a/src/test/kotlin/cash/atto/node/bootstrap/discovery/GapDiscovererTest.kt +++ b/src/test/kotlin/cash/atto/node/bootstrap/discovery/GapDiscovererTest.kt @@ -17,8 +17,8 @@ import cash.atto.node.bootstrap.unchecked.UncheckedTransactionRepository import cash.atto.node.network.DirectNetworkMessage import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.MessageSource +import cash.atto.node.network.MutableConnectedPeers import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.node.network.NodeConnected import cash.atto.node.network.NodeDisconnected import cash.atto.protocol.AttoNode import cash.atto.protocol.AttoTransactionStreamRequest @@ -252,6 +252,41 @@ class GapDiscovererTest { assertEquals(1, fixture.discoverer.activeSessionCount()) } + @Test + fun `delayed old disconnect preserves session after same URI reconnects`() = + runTest { + // Given + val peer = node(100) + val fixture = fixture(gaps = listOf(gap(publicKey(1), 1U, 2U, hash(1))), nodes = listOf(peer)) + fixture.discover() + val oldDisconnect = NodeDisconnected(InetSocketAddress(peer.publicUri.host, peer.publicUri.port), peer) + fixture.connectedPeers.disconnect(peer.publicUri) + fixture.connectedPeers.connect(peer, InetSocketAddress("127.0.0.1", 10_001)) + + // When + fixture.discoverer.remove(oldDisconnect) + fixture.discover() + + // Then + assertEquals(1, fixture.discoverer.activeSessionCount()) + assertEquals(1, fixture.requests.size) + } + + @Test + fun `connected peer without historical capability prevents gap query`() = + runTest { + // Given + val peer = node(100).copy(features = setOf(NodeFeature.VOTING)) + val fixture = fixture(gaps = listOf(gap(publicKey(1), 1U, 1U, hash(1))), nodes = listOf(peer)) + + // When + fixture.discover() + + // Then + coVerify(exactly = 0) { fixture.repository.findGaps(any()) } + assertTrue(fixture.requests.isEmpty()) + } + @Test fun `caffeine expires sessions after one minute without progress`() = runTest { @@ -431,6 +466,7 @@ class GapDiscovererTest { requests += firstArg>() } + val connectedPeers = MutableConnectedPeers() val discoverer = GapDiscoverer( uncheckedTransactionRepository = repository, @@ -439,6 +475,7 @@ class GapDiscovererTest { discoveryProperties = properties, meterRegistry = SimpleMeterRegistry(), clock = clock, + connectedPeers = connectedPeers, ) return Fixture( discoverer = discoverer, @@ -446,6 +483,7 @@ class GapDiscovererTest { repository = repository, requests = requests, nodes = nodes, + connectedPeers = connectedPeers, ).also { if (connect) { it.connect() @@ -459,16 +497,18 @@ class GapDiscovererTest { val repository: UncheckedTransactionRepository, val requests: List>, val nodes: List, + val connectedPeers: MutableConnectedPeers, ) { suspend fun discover(): Int = discoverer.discover() fun connect() { nodes.forEach { node -> - discoverer.add(NodeConnected(socketAddress(node), node)) + connectedPeers.connect(node, socketAddress(node)) } } fun disconnect(node: AttoNode = nodes.first()) { + connectedPeers.disconnect(node.publicUri) discoverer.remove(NodeDisconnected(socketAddress(node), node)) } diff --git a/src/test/kotlin/cash/atto/node/bootstrap/discovery/LastDiscovererTest.kt b/src/test/kotlin/cash/atto/node/bootstrap/discovery/LastDiscovererTest.kt index 6162dabf..64d3297c 100644 --- a/src/test/kotlin/cash/atto/node/bootstrap/discovery/LastDiscovererTest.kt +++ b/src/test/kotlin/cash/atto/node/bootstrap/discovery/LastDiscovererTest.kt @@ -15,19 +15,16 @@ import cash.atto.commons.AttoWork import cash.atto.commons.toAttoHeight import cash.atto.commons.toAttoVersion import cash.atto.node.account.AccountRepository -import cash.atto.node.bootstrap.unchecked.UncheckedTransactionRepository import cash.atto.node.election.ElectionVoter import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.MessageSource import cash.atto.node.network.NetworkMessage import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.node.network.NodeConnectionManager import cash.atto.node.transaction.TransactionRepository import cash.atto.node.vote.Vote import cash.atto.node.vote.convertion.VoteConverter import cash.atto.node.vote.weight.VoteWeighter import cash.atto.protocol.AttoBootstrapTransactionPush -import cash.atto.protocol.AttoNode import cash.atto.protocol.AttoVoteStreamCancel import cash.atto.protocol.AttoVoteStreamRequest import cash.atto.protocol.AttoVoteStreamResponse @@ -189,11 +186,8 @@ class LastDiscovererTest { val discoverer = LastDiscoverer( - thisNode = mockk(relaxed = true), accountRepository = accountRepository, transactionRepository = mockk(), - uncheckedTransactionRepository = mockk(), - nodeConnectionManager = mockk(), networkMessagePublisher = networkMessagePublisher, discoveryQueue = discoveryQueue, voteConverter = voteConverter, diff --git a/src/test/kotlin/cash/atto/node/bootstrap/discovery/SendDiscovererTest.kt b/src/test/kotlin/cash/atto/node/bootstrap/discovery/SendDiscovererTest.kt index 0a88002b..f7910af7 100644 --- a/src/test/kotlin/cash/atto/node/bootstrap/discovery/SendDiscovererTest.kt +++ b/src/test/kotlin/cash/atto/node/bootstrap/discovery/SendDiscovererTest.kt @@ -13,14 +13,23 @@ import cash.atto.commons.AttoTransaction import cash.atto.commons.AttoWork import cash.atto.commons.toAttoHeight import cash.atto.commons.toAttoVersion +import cash.atto.node.bootstrap.TransactionDiscovered import cash.atto.node.bootstrap.TransactionStuck import cash.atto.node.bootstrap.unchecked.UncheckedTransactionRepository +import cash.atto.node.network.BroadcastNetworkMessage +import cash.atto.node.network.BroadcastStrategy +import cash.atto.node.network.DirectNetworkMessage import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.MessageSource +import cash.atto.node.network.MutableConnectedPeers import cash.atto.node.network.NetworkMessagePublisher import cash.atto.node.transaction.Transaction import cash.atto.node.transaction.TransactionRejectionReason +import cash.atto.node.vote.Vote +import cash.atto.protocol.AttoNode +import cash.atto.protocol.AttoTransactionRequest import cash.atto.protocol.AttoTransactionResponse +import cash.atto.protocol.NodeFeature import io.micrometer.core.instrument.simple.SimpleMeterRegistry import io.mockk.coEvery import io.mockk.coVerify @@ -47,6 +56,7 @@ class SendDiscovererTest { discoveryQueue, uncheckedTransactionRepository, SimpleMeterRegistry(), + MutableConnectedPeers(), ) val missingSend = sendTransaction(1) val receive = transaction(2, missingSend.hash) @@ -99,6 +109,7 @@ class SendDiscovererTest { discoveryQueue, uncheckedTransactionRepository, SimpleMeterRegistry(), + MutableConnectedPeers(), ) val missingSend = sendTransaction(1) val receive = transaction(2, missingSend.hash) @@ -135,6 +146,7 @@ class SendDiscovererTest { discoveryQueue, uncheckedTransactionRepository, meterRegistry, + MutableConnectedPeers(), ) val missingSend = sendTransaction(1) val firstReceive = transaction(2, missingSend.hash) @@ -190,6 +202,81 @@ class SendDiscovererTest { ) } + @Test + fun `missing send request follows first connected representative in vote order`() { + // Given + val publisher = mockk(relaxed = true) + val queue = mockk { every { isAtCapacity() } returns false } + val peers = MutableConnectedPeers() + val disconnected = representative(10) + val first = representative(11) + val second = representative(12) + peers.connect(second) + peers.connect(first) + val discoverer = SendDiscoverer(publisher, queue, mockk(), SimpleMeterRegistry(), peers) + val missingSend = sendTransaction(1) + val votes = + listOf(disconnected, first, second).map { node -> + mockk { every { publicKey } returns node.publicKey } + } + + // When + discoverer.process( + TransactionDiscovered(TransactionRejectionReason.RECEIVABLE_NOT_FOUND, transaction(2, missingSend.hash), votes), + ) + + // Then + verify(exactly = 1) { + publisher.publish( + match { + it is DirectNetworkMessage<*> && + it.publicUri == first.publicUri && + (it.payload as AttoTransactionRequest).hash == missingSend.hash + }, + ) + } + } + + @Test + fun `departed representative falls back to broadcast without a disconnect listener`() { + // Given + val publisher = mockk(relaxed = true) + val queue = mockk { every { isAtCapacity() } returns false } + val peers = MutableConnectedPeers() + val representative = representative(10) + peers.connect(representative) + val discoverer = SendDiscoverer(publisher, queue, mockk(), SimpleMeterRegistry(), peers) + val missingSend = sendTransaction(1) + val vote = mockk { every { publicKey } returns representative.publicKey } + peers.disconnect(representative.publicUri) + + // When + discoverer.process( + TransactionDiscovered(TransactionRejectionReason.RECEIVABLE_NOT_FOUND, transaction(2, missingSend.hash), listOf(vote)), + ) + + // Then + verify(exactly = 1) { + publisher.publish( + match { + it is BroadcastNetworkMessage<*> && + it.strategy == BroadcastStrategy.EVERYONE && + (it.payload as AttoTransactionRequest).hash == missingSend.hash + }, + ) + } + } + + private fun representative(marker: Byte): AttoNode = + AttoNode( + network = AttoNetwork.LOCAL, + protocolVersion = AttoNode.CURRENT_PROTOCOL_VERSION, + algorithm = AttoAlgorithm.V1, + publicKey = AttoPublicKey(ByteArray(32) { marker }), + publicUri = URI("ws://127.0.0.1:${8_000 + marker}"), + features = setOf(NodeFeature.VOTING), + ) + private fun sendTransaction(marker: Byte): AttoTransaction { val publicKey = AttoPublicKey(ByteArray(32) { marker }) val block = diff --git a/src/test/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserterTest.kt b/src/test/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserterTest.kt index 7a7e6dae..ccddb5a0 100644 --- a/src/test/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserterTest.kt +++ b/src/test/kotlin/cash/atto/node/bootstrap/unchecked/UncheckedTransactionInserterTest.kt @@ -12,6 +12,7 @@ import cash.atto.commons.AttoSignature import cash.atto.commons.AttoWork import cash.atto.commons.toAttoHeight import cash.atto.commons.toAttoVersion +import cash.atto.node.bootstrap.BootstrapLoadMonitor import cash.atto.node.transaction.Transaction import io.micrometer.core.instrument.simple.SimpleMeterRegistry import io.mockk.every @@ -23,9 +24,11 @@ import io.r2dbc.spi.ConnectionFactory import io.r2dbc.spi.ConnectionFactoryMetadata import io.r2dbc.spi.Result import io.r2dbc.spi.Statement +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assertions.fail import org.junit.jupiter.api.Test import org.springframework.r2dbc.core.DatabaseClient import reactor.core.publisher.Flux @@ -55,7 +58,8 @@ class UncheckedTransactionInserterTest { every { result.rowsUpdated } returns Mono.just(2L) val meterRegistry = SimpleMeterRegistry() - val inserter = UncheckedTransactionInserter(DatabaseClient.create(connectionFactory), meterRegistry) + val loadMonitor = mockk(relaxed = true) + val inserter = UncheckedTransactionInserter(DatabaseClient.create(connectionFactory), meterRegistry, loadMonitor) val transactions = listOf(openTransaction(), receiveTransaction()) // when @@ -72,6 +76,7 @@ class UncheckedTransactionInserterTest { val timer = meterRegistry.get(UncheckedTransactionInserter.METRIC_NAME).timer() assertEquals(1L, timer.count()) assertTrue(timer.totalTime(TimeUnit.NANOSECONDS) > 0) + verify(exactly = 1) { loadMonitor.recordInsert(timer.totalTime(TimeUnit.NANOSECONDS).toLong()) } verify(exactly = 1) { connection.createStatement(any()) } verify(exactly = 13) { statement.bind(any(), any()) } verify(exactly = 1) { statement.bindNull(3, ByteArray::class.java) } @@ -88,7 +93,8 @@ class UncheckedTransactionInserterTest { every { connectionFactory.metadata } returns connectionFactoryMetadata every { connectionFactoryMetadata.name } returns "MySQL" val meterRegistry = SimpleMeterRegistry() - val inserter = UncheckedTransactionInserter(DatabaseClient.create(connectionFactory), meterRegistry) + val loadMonitor = mockk(relaxed = true) + val inserter = UncheckedTransactionInserter(DatabaseClient.create(connectionFactory), meterRegistry, loadMonitor) // when val inserted = inserter.insert(emptyList()) @@ -97,8 +103,45 @@ class UncheckedTransactionInserterTest { assertEquals(0L, inserted) assertEquals(0L, meterRegistry.get(UncheckedTransactionInserter.METRIC_NAME).timer().count()) verify(exactly = 0) { connectionFactory.create() } + verify(exactly = 0) { loadMonitor.recordInsert(any()) } } + @Test + fun `failed insert reports the same elapsed time as its timer`() = + runTest { + verifyFailedInsert(IllegalStateException("simulated insert failure")) + } + + @Test + fun `cancelled insert reports elapsed time before propagating cancellation`() = + runTest { + verifyFailedInsert(CancellationException("simulated insert cancellation")) + } + + private suspend fun verifyFailedInsert(failure: Exception) { + // Given + val connectionFactory = mockk() + val connectionFactoryMetadata = mockk() + every { connectionFactory.metadata } returns connectionFactoryMetadata + every { connectionFactoryMetadata.name } returns "MySQL" + every { connectionFactory.create() } returns Mono.error(failure) + val meterRegistry = SimpleMeterRegistry() + val loadMonitor = mockk(relaxed = true) + val inserter = UncheckedTransactionInserter(DatabaseClient.create(connectionFactory), meterRegistry, loadMonitor) + + // When + try { + inserter.insert(listOf(openTransaction())) + fail("Expected insert failure") + } catch (exception: Exception) { + // Then + assertTrue(generateSequence(exception) { it.cause }.any { it === failure }) + } + val timer = meterRegistry.get(UncheckedTransactionInserter.METRIC_NAME).timer() + assertEquals(1L, timer.count()) + verify(exactly = 1) { loadMonitor.recordInsert(timer.totalTime(TimeUnit.NANOSECONDS).toLong()) } + } + private fun openTransaction(): UncheckedTransaction { val publicKey = AttoPublicKey(ByteArray(32) { 1 }) return Transaction( diff --git a/src/test/kotlin/cash/atto/node/election/ElectionTest.kt b/src/test/kotlin/cash/atto/node/election/ElectionTest.kt index d7407cb4..baf14b18 100644 --- a/src/test/kotlin/cash/atto/node/election/ElectionTest.kt +++ b/src/test/kotlin/cash/atto/node/election/ElectionTest.kt @@ -30,14 +30,27 @@ import io.mockk.impl.annotations.MockK import io.mockk.impl.annotations.RelaxedMockK import io.mockk.junit5.MockKExtension import io.mockk.verify +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource +import org.springframework.boot.actuate.info.Info import java.time.Instant +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit import kotlin.random.Random @ExtendWith(MockKExtension::class) @@ -105,6 +118,138 @@ class ElectionTest { assertEquals(1, election.getSize()) } + @Test + fun `diagnostics retain detached candidates votes and voting weights after later transitions`() = + runBlocking { + // Given + val transaction = receiveTransaction() + val previousAccount = accountBefore(transaction) + val competitor = transaction.competing() + val vote = Vote.sample(blockHash = transaction.hash, weight = AttoAmount(100UL)) + every { voteWeighter.get(vote.publicKey) } returns vote.weight + election.start(TransactionValidated(previousAccount, transaction)) + election.process(VoteValidated(transaction, vote)) + val builder = Info.Builder() + ElectionInfoContributor(election).contribute(builder) + val details = builder.build().details["elections"] as Map<*, *> + val snapshot = details["weights"] as Map<*, *> + val captured = snapshot[transaction.toPublicKeyHeight()] as PublicKeyHeightElection + + // When + val newerVote = vote.copy(blockHash = competitor.hash, timestamp = vote.timestamp.plusSeconds(1)) + every { voteWeighter.get(vote.publicKey) } returns AttoAmount(200UL) + election.start(TransactionValidated(previousAccount, competitor)) + election.process(VoteValidated(competitor, newerVote)) + election.clear() + every { voteWeighter.getMinimalConfirmationWeight() } returns AttoAmount(50UL) + + // Then + assertEquals(1, details["active"]) + assertEquals(setOf(transaction.toPublicKeyHeight()), snapshot.keys) + assertEquals(listOf(transaction), captured.getTransactions()) + assertEquals(mapOf(vote.publicKey to vote), captured.getProvisionalLeader().votes) + assertEquals(vote.weight, captured.getProvisionalLeader().totalWeight) + assertNull(captured.getConsensus()) + assertEquals(0, election.getSize()) + } + + @Test + fun `diagnostic consumers cannot mutate live election candidates or votes`() = + runBlocking { + // Given + val transaction = receiveTransaction() + val previousAccount = accountBefore(transaction) + val vote = Vote.sample(blockHash = transaction.hash, weight = AttoAmount(100UL)) + every { voteWeighter.get(vote.publicKey) } returns vote.weight + election.start(TransactionValidated(previousAccount, transaction)) + election.process(VoteValidated(transaction, vote)) + val captured = election.getElections().getValue(transaction.toPublicKeyHeight()) + + // When + captured.add(transaction.competing()) + captured.getProvisionalLeader().votes.clear() + + // Then + val current = election.getElections().getValue(transaction.toPublicKeyHeight()) + assertEquals(listOf(transaction), current.getTransactions()) + assertEquals(mapOf(vote.publicKey to vote), current.getProvisionalLeader().votes) + assertEquals(vote.weight, current.getProvisionalLeader().totalWeight) + } + + @Test + fun `diagnostics remain coherent while elections start and clear concurrently`() = + runBlocking { + // Given + val transaction = receiveTransaction() + val previousAccount = accountBefore(transaction) + val start = CompletableDeferred() + val contributor = ElectionInfoContributor(election) + + // When + val transitions = + async(Dispatchers.Default) { + start.await() + repeat(500) { + election.start(TransactionValidated(previousAccount, transaction)) + election.clear() + } + } + val observations = + async(Dispatchers.Default) { + start.await() + repeat(500) { + val builder = Info.Builder() + contributor.contribute(builder) + val details = builder.build().details["elections"] as Map<*, *> + val elections = details["weights"] as Map<*, *> + + // Then + assertEquals(elections.size, details["active"]) + elections.values.forEach { value -> + assertEquals(listOf(transaction), (value as PublicKeyHeightElection).getTransactions()) + } + } + } + + start.complete(Unit) + awaitAll(transitions, observations) + } + + @Test + fun `scheduled expiry can be cancelled while a diagnostic snapshot owns election state`() = + runBlocking { + // Given + val transaction = receiveTransaction().copy(receivedAt = Instant.now().minusSeconds(1_000)) + election.start(TransactionValidated(accountBefore(transaction), transaction)) + val snapshotStarted = CountDownLatch(1) + val releaseSnapshot = CountDownLatch(1) + every { voteWeighter.getMinimalConfirmationWeight() } answers { + snapshotStarted.countDown() + check(releaseSnapshot.await(5, TimeUnit.SECONDS)) { "Snapshot was not released" } + minimalWeight + } + val snapshot = async(Dispatchers.Default) { election.getElections() } + + try { + // When + assertTrue(snapshotStarted.await(5, TimeUnit.SECONDS)) + val expiry = launch(start = CoroutineStart.UNDISPATCHED) { election.expireOld() } + assertFalse(expiry.isCompleted) + expiry.cancelAndJoin() + + // Then + assertTrue(expiry.isCancelled) + assertEquals(1L, releaseSnapshot.count) + verify(exactly = 0) { eventPublisher.publish(any()) } + releaseSnapshot.countDown() + assertEquals(setOf(transaction.toPublicKeyHeight()), snapshot.await().keys) + assertEquals(1, election.getSize()) + } finally { + releaseSnapshot.countDown() + snapshot.cancelAndJoin() + } + } + @Test fun `should remove election when AccountUpdated event is received`() { // given diff --git a/src/test/kotlin/cash/atto/node/network/MutableConnectedPeers.kt b/src/test/kotlin/cash/atto/node/network/MutableConnectedPeers.kt new file mode 100644 index 00000000..c907c818 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/MutableConnectedPeers.kt @@ -0,0 +1,32 @@ +package cash.atto.node.network + +import cash.atto.protocol.AttoNode +import java.net.InetSocketAddress +import java.net.URI + +internal class MutableConnectedPeers : ConnectedPeers { + private val peers = LinkedHashMap() + + override fun snapshot(): List = peers.values.toList() + + override fun isConnected(publicUri: URI): Boolean = publicUri in peers + + fun connect( + node: AttoNode, + socketAddress: InetSocketAddress = InetSocketAddress(node.publicUri.host, node.publicUri.port), + ) { + connect(ConnectedPeer(node, socketAddress)) + } + + fun connect(peer: ConnectedPeer) { + peers[peer.node.publicUri] = peer + } + + fun disconnect(publicUri: URI) { + peers.remove(publicUri) + } + + fun clear() { + peers.clear() + } +} diff --git a/src/test/kotlin/cash/atto/node/network/NetworkInfoContributorTest.kt b/src/test/kotlin/cash/atto/node/network/NetworkInfoContributorTest.kt new file mode 100644 index 00000000..721f17b9 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/NetworkInfoContributorTest.kt @@ -0,0 +1,72 @@ +package cash.atto.node.network + +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoNetwork +import cash.atto.commons.AttoPublicKey +import cash.atto.node.EventPublisher +import cash.atto.node.network.guardian.PeerBans +import cash.atto.protocol.AttoNode +import cash.atto.protocol.NodeFeature +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.springframework.boot.actuate.info.Info +import java.net.InetAddress +import java.net.URI +import java.time.Clock + +class NetworkInfoContributorTest { + @Test + fun `diagnostics query current connections and authoritative bans`() { + // Given + val peers = MutableConnectedPeers() + val bans = PeerBans(mockk(relaxed = true), Clock.systemUTC()) + val contributor = NetworkInfoContributor(peers, bans) + val old = + AttoNode( + network = AttoNetwork.LOCAL, + protocolVersion = AttoNode.CURRENT_PROTOCOL_VERSION, + algorithm = AttoAlgorithm.V1, + publicKey = AttoPublicKey(ByteArray(32) { 1 }), + publicUri = URI("ws://127.0.0.1:8080"), + features = setOf(NodeFeature.HISTORICAL), + ) + peers.connect(old) + val banned = InetAddress.getByName("8.8.8.8") + bans.ban(banned) + + // When + val current = old.copy(publicKey = AttoPublicKey(ByteArray(32) { 2 })) + peers.connect(current) + val builder = Info.Builder() + contributor.contribute(builder) + + // Then + assertEquals( + mapOf("paired" to listOf(current), "banned" to listOf(banned.hostAddress)), + builder.build().details["network"], + ) + + // When + peers.disconnect(current.publicUri) + val disconnected = Info.Builder() + contributor.contribute(disconnected) + + // Then + assertEquals( + mapOf("paired" to emptyList(), "banned" to listOf(banned.hostAddress)), + disconnected.build().details["network"], + ) + + // When + bans.clear() + val cleared = Info.Builder() + contributor.contribute(cleared) + + // Then + assertEquals( + mapOf("paired" to emptyList(), "banned" to emptyList()), + cleared.build().details["network"], + ) + } +} diff --git a/src/test/kotlin/cash/atto/node/network/NetworkMessagePublisherTest.kt b/src/test/kotlin/cash/atto/node/network/NetworkMessagePublisherTest.kt new file mode 100644 index 00000000..93bf9502 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/NetworkMessagePublisherTest.kt @@ -0,0 +1,40 @@ +package cash.atto.node.network + +import cash.atto.commons.AttoPublicKey +import cash.atto.commons.toAttoHeight +import cash.atto.node.network.guardian.ExpectedResponses +import cash.atto.protocol.AttoTransactionStreamRequest +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.context.ApplicationEventPublisher +import java.net.URI +import java.time.Clock +import java.time.Instant +import java.time.ZoneOffset + +class NetworkMessagePublisherTest { + @Test + fun `response credits exist before dispatch can send and immediately receive a response`() { + // Given + val now = Instant.parse("2026-11-07T00:00:00Z") + val expectedResponses = ExpectedResponses(Clock.fixed(now, ZoneOffset.UTC)) + val publicUri = URI("ws://localhost:8080") + val dispatcher = mockk() + every { dispatcher.publishEvent(any()) } answers { + // A transport listener can send immediately, before other asynchronous listeners run. + assertTrue(expectedResponses.consume(publicUri)) + } + val publisher = NetworkMessagePublisher(dispatcher, expectedResponses) + val payload = AttoTransactionStreamRequest(AttoPublicKey(ByteArray(32) { 1 }), 1UL.toAttoHeight(), 1UL.toAttoHeight()) + val request = DirectNetworkMessage(publicUri, payload, expectedResponseCount = 1UL, timestamp = now) + + // When + publisher.publish(request) + + // Then + assertFalse(expectedResponses.consume(publicUri)) + } +} diff --git a/src/test/kotlin/cash/atto/node/network/NodeConnectionManagerTest.kt b/src/test/kotlin/cash/atto/node/network/NodeConnectionManagerTest.kt new file mode 100644 index 00000000..6d80e4ca --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/NodeConnectionManagerTest.kt @@ -0,0 +1,171 @@ +package cash.atto.node.network + +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoNetwork +import cash.atto.commons.AttoPublicKey +import cash.atto.node.EventPublisher +import cash.atto.protocol.AttoKeepAlive +import cash.atto.protocol.AttoNode +import cash.atto.protocol.NodeFeature +import io.ktor.websocket.Frame +import io.ktor.websocket.WebSocketExtension +import io.ktor.websocket.WebSocketSession +import io.mockk.mockk +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetSocketAddress +import java.net.URI +import kotlin.coroutines.CoroutineContext + +@OptIn(ExperimentalCoroutinesApi::class) +class NodeConnectionManagerTest { + @Test + fun `connection metadata immediately follows registration replacement and completion`() = + runTest { + // Given + val manager = manager() + val original = node(1) + val replacement = original.copy(publicKey = publicKey(2)) + val firstSocket = InetSocketAddress("127.0.0.1", 10_001) + val nextSocket = InetSocketAddress("127.0.0.1", 10_002) + val firstSession = ChannelSession() + val nextSession = ChannelSession() + try { + // When + val first = launch(start = CoroutineStart.UNDISPATCHED) { manager.manage(original, firstSocket, firstSession) } + val initialSnapshot = manager.snapshot() + + // Then + assertEquals(listOf(ConnectedPeer(original, firstSocket)), initialSnapshot) + assertTrue(manager.isConnected(original.publicUri)) + + // When + val next = launch(start = CoroutineStart.UNDISPATCHED) { manager.manage(replacement, nextSocket, nextSession) } + firstSession.incoming.close() + first.join() + + // Then + assertEquals(listOf(ConnectedPeer(replacement, nextSocket)), manager.snapshot()) + assertEquals(listOf(ConnectedPeer(original, firstSocket)), initialSnapshot) + + // When + nextSession.incoming.close() + next.join() + + // Then + assertTrue(manager.snapshot().isEmpty()) + assertFalse(manager.isConnected(original.publicUri)) + } finally { + firstSession.cancel() + nextSession.cancel() + manager.stop() + } + } + + @Test + fun `buffered keepalive from replaced session cannot restore old connection`() = + runTest { + // Given + val manager = manager() + val original = node(1) + val replacement = original.copy(publicKey = publicKey(2)) + val firstSocket = InetSocketAddress("127.0.0.1", 10_001) + val nextSocket = InetSocketAddress("127.0.0.1", 10_002) + val firstSession = ChannelSession() + val nextSession = ChannelSession() + try { + val first = launch(start = CoroutineStart.UNDISPATCHED) { manager.manage(original, firstSocket, firstSession) } + firstSession.incoming.send(Frame.Binary(true, NetworkSerializer.serialize(AttoKeepAlive()))) + + // When: replace before the scheduled old frame collector resumes. + val next = launch(start = CoroutineStart.UNDISPATCHED) { manager.manage(replacement, nextSocket, nextSession) } + runCurrent() + + // Then + assertEquals(listOf(ConnectedPeer(replacement, nextSocket)), manager.snapshot()) + + // When + firstSession.incoming.close() + nextSession.incoming.close() + first.join() + next.join() + } finally { + firstSession.cancel() + nextSession.cancel() + manager.stop() + } + } + + @Test + fun `cancelling managed work removes peer and initiates transport close`() = + runTest { + // Given + val manager = manager() + val peer = node(1) + val session = ChannelSession() + try { + val managed = + launch(start = CoroutineStart.UNDISPATCHED) { + manager.manage(peer, InetSocketAddress("127.0.0.1", 10_001), session) + } + + // When + managed.cancelAndJoin() + + // Then + assertFalse(manager.isConnected(peer.publicUri)) + assertTrue(manager.snapshot().isEmpty()) + assertTrue(session.outgoing.tryReceive().getOrNull() is Frame.Close) + } finally { + session.cancel() + manager.stop() + } + } + + private fun manager(): NodeConnectionManager = + NodeConnectionManager( + thisNode = node(100), + messagePublisher = mockk(relaxed = true), + eventPublisher = mockk(relaxed = true), + ) + + // Sending Close starts graceful shutdown; frames already received can still be consumed until the peer closes. + private class ChannelSession : WebSocketSession { + override val coroutineContext: CoroutineContext = SupervisorJob() + override var masking: Boolean = false + override var maxFrameSize: Long = Long.MAX_VALUE + override val incoming = Channel(Channel.UNLIMITED) + override val outgoing = Channel(Channel.UNLIMITED) + override val extensions: List> = emptyList() + + override suspend fun flush() = Unit + + @Deprecated("Use cancel() instead.", level = DeprecationLevel.ERROR) + override fun terminate() { + cancel() + } + } + + private fun node(marker: Byte): AttoNode = + AttoNode( + network = AttoNetwork.LOCAL, + protocolVersion = AttoNode.CURRENT_PROTOCOL_VERSION, + algorithm = AttoAlgorithm.V1, + publicKey = publicKey(marker), + publicUri = URI("ws://127.0.0.1:${8_000 + marker}"), + features = setOf(NodeFeature.HISTORICAL), + ) + + private fun publicKey(marker: Byte): AttoPublicKey = AttoPublicKey(ByteArray(32) { marker }) +} diff --git a/src/test/kotlin/cash/atto/node/network/guardian/GuardianTest.kt b/src/test/kotlin/cash/atto/node/network/guardian/GuardianTest.kt index 5225633c..b6b7111b 100644 --- a/src/test/kotlin/cash/atto/node/network/guardian/GuardianTest.kt +++ b/src/test/kotlin/cash/atto/node/network/guardian/GuardianTest.kt @@ -7,19 +7,19 @@ import cash.atto.commons.AttoNetwork import cash.atto.commons.AttoPublicKey import cash.atto.commons.toAttoHeight import cash.atto.node.EventPublisher +import cash.atto.node.network.ConnectedPeer +import cash.atto.node.network.ConnectedPeers import cash.atto.node.network.DirectNetworkMessage import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.MessageSource +import cash.atto.node.network.MutableConnectedPeers import cash.atto.node.network.NodeBanned -import cash.atto.node.network.NodeConnected -import cash.atto.node.network.NodeDisconnected import cash.atto.node.vote.weight.VoteWeighter import cash.atto.protocol.AttoNode import cash.atto.protocol.AttoTransactionStreamRequest import cash.atto.protocol.AttoVoteStreamRequest import cash.atto.protocol.NodeFeature import io.mockk.every -import io.mockk.impl.annotations.InjectMockKs import io.mockk.impl.annotations.RelaxedMockK import io.mockk.junit5.MockKExtension import io.mockk.verify @@ -57,10 +57,16 @@ class GuardianTest { @RelaxedMockK lateinit var clock: Clock + @RelaxedMockK + lateinit var connectedPeers: ConnectedPeers + + private val peers = MutableConnectedPeers() + private var now = Instant.parse("2026-11-07T00:00:00Z") - @InjectMockKs - lateinit var guardian: Guardian + private lateinit var expectedResponses: ExpectedResponses + private lateinit var peerBans: PeerBans + private lateinit var guardian: Guardian @BeforeEach fun beforeEach() { @@ -68,36 +74,40 @@ class GuardianTest { every { guardianProperties.toleranceMultiplier } returns toleranceMultiplier every { guardianProperties.connectionRequestLimitPerMinute } returns 60 every { clock.instant() } answers { now } - guardian.clear() + peers.clear() + every { connectedPeers.snapshot() } answers { peers.snapshot() } + expectedResponses = ExpectedResponses(clock) + peerBans = PeerBans(eventPublisher, clock) + guardian = Guardian(thisNode, voteWeighter, guardianProperties, peerBans, expectedResponses, connectedPeers) } @Test fun `should take snapshot`() { val node = createNode(AttoAmount.MAX) - val event = NodeConnected(randomSocketAddress(), node) - guardian.add(NodeConnected(randomSocketAddress(), node)) - guardian.count(inboundMessage(node.publicUri, event.connectionSocketAddress)) + val event = ConnectedPeer(node, randomSocketAddress()) + peers.connect(event) + guardian.count(inboundMessage(node.publicUri, event.socketAddress)) // when guardian.guard() // then - assertEquals(mapOf(event.connectionSocketAddress to 1UL), guardian.getSnapshot()) + assertEquals(mapOf(event.socketAddress to 1UL), guardian.getSnapshot()) verify(exactly = 0) { eventPublisher.publish(any()) } } @Test fun `should ban when node deviate from single voter`() { val voteNode = createNode(AttoAmount.MAX) - val voterEvent = NodeConnected(randomSocketAddress(), voteNode) - guardian.add(voterEvent) - guardian.count(inboundMessage(voterEvent.node.publicUri, voterEvent.connectionSocketAddress)) + val voterEvent = ConnectedPeer(voteNode, randomSocketAddress()) + peers.connect(voterEvent) + guardian.count(inboundMessage(voterEvent.node.publicUri, voterEvent.socketAddress)) val normalPeer = createNode(AttoAmount.MAX) - val normalEvent = NodeConnected(randomSocketAddress(), normalPeer) - guardian.add(normalEvent) + val normalEvent = ConnectedPeer(normalPeer, randomSocketAddress()) + peers.connect(normalEvent) for (i in 0UL..toleranceMultiplier) { - guardian.count(inboundMessage(normalEvent.node.publicUri, normalEvent.connectionSocketAddress)) + guardian.count(inboundMessage(normalEvent.node.publicUri, normalEvent.socketAddress)) } every { voteWeighter.isAboveMinimalRebroadcastWeight(voterEvent.node.publicKey) } returns true @@ -110,29 +120,53 @@ class GuardianTest { eventPublisher.publish( match { event -> event as NodeBanned - event.address == normalEvent.connectionSocketAddress.address + event.address == normalEvent.socketAddress.address }, ) } } + @Test + fun `guard rejects banned addresses before dispatching their notification`() { + // Given + val voter = createNode(AttoAmount.MAX) + val voterSocket = InetSocketAddress(InetAddress.getByName("8.8.8.8"), 10_000) + peers.connect(ConnectedPeer(voter, voterSocket)) + every { voteWeighter.isAboveMinimalRebroadcastWeight(voter.publicKey) } returns true + guardian.count(inboundMessage(voter.publicUri, voterSocket)) + val peer = createNode(AttoAmount.MAX) + val peerSocket = InetSocketAddress(InetAddress.getByName("1.1.1.1"), 10_001) + repeat(11) { guardian.count(inboundMessage(peer.publicUri, peerSocket)) } + every { eventPublisher.publish(any()) } answers { + val event = firstArg() + assertEquals(InboundConnectionDecision.Banned, guardian.requestInboundConnection(event.address)) + } + + // When + guardian.guard() + + // Then + assertEquals(setOf(peerSocket.address), peerBans.snapshot()) + verify(exactly = 1) { eventPublisher.publish(match { it is NodeBanned && it.address == peerSocket.address }) } + } + @Test fun `should ban when node deviate from voters median`() { val votePeer1 = createNode(AttoAmount(ULong.MAX_VALUE / 2U)) - val voterEvent1 = NodeConnected(randomSocketAddress(), votePeer1) - guardian.add(voterEvent1) - guardian.count(inboundMessage(voterEvent1.node.publicUri, voterEvent1.connectionSocketAddress)) + val voterEvent1 = ConnectedPeer(votePeer1, randomSocketAddress()) + peers.connect(voterEvent1) + guardian.count(inboundMessage(voterEvent1.node.publicUri, voterEvent1.socketAddress)) val votePeer2 = createNode(AttoAmount(ULong.MAX_VALUE / 2U)) - val voterEvent2 = NodeConnected(randomSocketAddress(), votePeer2) - guardian.add(voterEvent2) - guardian.count(inboundMessage(voterEvent2.node.publicUri, voterEvent2.connectionSocketAddress)) + val voterEvent2 = ConnectedPeer(votePeer2, randomSocketAddress()) + peers.connect(voterEvent2) + guardian.count(inboundMessage(voterEvent2.node.publicUri, voterEvent2.socketAddress)) val normalPeer = createNode(AttoAmount.MAX) - val normalEvent = NodeConnected(randomSocketAddress(), normalPeer) - guardian.add(normalEvent) + val normalEvent = ConnectedPeer(normalPeer, randomSocketAddress()) + peers.connect(normalEvent) for (i in 0UL..toleranceMultiplier) { - guardian.count(inboundMessage(normalEvent.node.publicUri, normalEvent.connectionSocketAddress)) + guardian.count(inboundMessage(normalEvent.node.publicUri, normalEvent.socketAddress)) } every { voteWeighter.isAboveMinimalRebroadcastWeight(voterEvent1.node.publicKey) } returns true @@ -146,7 +180,7 @@ class GuardianTest { eventPublisher.publish( match { event -> event as NodeBanned - event.address == normalEvent.connectionSocketAddress.address + event.address == normalEvent.socketAddress.address }, ) } @@ -156,16 +190,32 @@ class GuardianTest { fun `should remove voter when disconnected`() { // given val node = createNode(AttoAmount.MAX) - val event = NodeConnected(randomSocketAddress(), node) - guardian.add(event) + val event = ConnectedPeer(node, randomSocketAddress()) + peers.connect(event) // when - guardian.remove(NodeDisconnected(event.connectionSocketAddress, node)) + peers.disconnect(node.publicUri) // then assertThat(guardian.getVoters()).isEmpty() } + @Test + fun `voter query follows current connections and excludes non voters`() { + // Given + val old = createNode(AttoAmount.MAX) + peers.connect(old, randomSocketAddress()) + val voter = createNode(AttoAmount.MAX) + val voterSocket = randomSocketAddress() + peers.connect(voter, voterSocket) + + // When + peers.connect(old.copy(features = setOf(NodeFeature.HISTORICAL)), randomSocketAddress()) + + // Then + assertEquals(mapOf(voterSocket to voter.publicKey), guardian.getVoters()) + } + @Test fun `should accept inbound connection under limit and record statistics`() { val address = InetAddress.getByName("8.8.8.8") @@ -194,14 +244,25 @@ class GuardianTest { @Test fun `should reject banned inbound connection without recording statistics`() { + // Given val address = InetAddress.getByName("1.1.1.1") - guardian.ban(NodeBanned(address)) + peerBans.ban(address) + // When val decision = guardian.requestInboundConnection(address) guardian.guard() + // Then assertEquals(InboundConnectionDecision.Banned, decision) assertThat(guardian.getSnapshot()).isEmpty() + + // When + now = now.plusSeconds(3_600) + peerBans.review() + val afterExpiry = guardian.requestInboundConnection(address) + + // Then + assertEquals(InboundConnectionDecision.Accepted, afterExpiry) } @Test @@ -224,11 +285,11 @@ class GuardianTest { @Test fun `should include idle voters in successive window medians`() { // Given - val voters = (1..3).map { NodeConnected(randomSocketAddress(), createNode(AttoAmount.MAX)) } + val voters = (1..3).map { ConnectedPeer(createNode(AttoAmount.MAX), randomSocketAddress()) } voters.forEach { - guardian.add(it) + peers.connect(it) every { voteWeighter.isAboveMinimalRebroadcastWeight(it.node.publicKey) } returns true - guardian.count(inboundMessage(it.node.publicUri, it.connectionSocketAddress)) + guardian.count(inboundMessage(it.node.publicUri, it.socketAddress)) } guardian.guard() val active = voters.first() @@ -236,13 +297,13 @@ class GuardianTest { // When repeat(2) { repeat(100) { - guardian.count(inboundMessage(active.node.publicUri, active.connectionSocketAddress)) + guardian.count(inboundMessage(active.node.publicUri, active.socketAddress)) } guardian.guard() } // Then - assertEquals(mapOf(active.connectionSocketAddress to 100UL), guardian.getSnapshot()) + assertEquals(mapOf(active.socketAddress to 100UL), guardian.getSnapshot()) verify(exactly = 0) { eventPublisher.publish(match { it is NodeBanned }) } } @@ -255,9 +316,9 @@ class GuardianTest { // When repeat(20) { index -> val socket = InetSocketAddress(address, 10_000 + index) - guardian.add(NodeConnected(socket, node)) + peers.connect(ConnectedPeer(node, socket)) guardian.count(inboundMessage(node.publicUri, socket)) - guardian.remove(NodeDisconnected(socket, node)) + peers.disconnect(node.publicUri) guardian.guard() assertEquals(mapOf(socket to 1UL), guardian.getSnapshot()) } @@ -311,7 +372,7 @@ class GuardianTest { // Given val node = createNode(AttoAmount.MAX) val message = inboundMessage(node.publicUri, randomSocketAddress()) - guardian.count(request(node, 2UL)) + expectedResponses.record(request(node, 2UL)) // When guardian.count(message) @@ -319,7 +380,7 @@ class GuardianTest { guardian.guard() // Then - assertEquals(0, guardian.expectedResponsePeerCount()) + assertEquals(0, expectedResponses.peerCount) assertThat(guardian.getSnapshot()).isEmpty() guardian.count(message) guardian.guard() @@ -327,29 +388,28 @@ class GuardianTest { } @Test - fun `should preserve reconnect credits across delayed old disconnect and expire unanswered credit`() { + fun `should preserve credits across reconnect and expire unanswered credit`() { // Given val node = createNode(AttoAmount.MAX) val oldSocket = InetSocketAddress(InetAddress.getLoopbackAddress(), 10_000) val newSocket = InetSocketAddress(InetAddress.getLoopbackAddress(), 10_001) - guardian.add(NodeConnected(oldSocket, node)) - val delayedDisconnect = NodeDisconnected(oldSocket, node, timestamp = now) + peers.connect(ConnectedPeer(node, oldSocket)) + peers.disconnect(node.publicUri) now = now.plusSeconds(1) - guardian.add(NodeConnected(newSocket, node)) - guardian.count(request(node, 2UL)) + peers.connect(ConnectedPeer(node, newSocket)) + expectedResponses.record(request(node, 2UL)) // When - guardian.remove(delayedDisconnect) guardian.count(inboundMessage(node.publicUri, newSocket)) guardian.guard() // Then assertEquals(mapOf(newSocket to node.publicKey), guardian.getVoters()) - assertEquals(1, guardian.expectedResponsePeerCount()) + assertEquals(1, expectedResponses.peerCount) assertThat(guardian.getSnapshot()).isEmpty() now = now.plus(AttoTransactionStreamRequest.TIMEOUT) guardian.guard() - assertEquals(0, guardian.expectedResponsePeerCount()) + assertEquals(0, expectedResponses.peerCount) guardian.count(inboundMessage(node.publicUri, newSocket)) guardian.guard() assertEquals(mapOf(newSocket to 1UL), guardian.getSnapshot()) @@ -360,17 +420,17 @@ class GuardianTest { // Given val node = createNode(AttoAmount.MAX) val request = request(node, 2UL) - guardian.remove(NodeDisconnected(randomSocketAddress(), node)) + peers.disconnect(node.publicUri) now = now.plusSeconds(30) // When - guardian.count(request) - assertEquals(1, guardian.expectedResponsePeerCount()) + expectedResponses.record(request) + assertEquals(1, expectedResponses.peerCount) now = now.plusSeconds(30) guardian.guard() // Then - assertEquals(0, guardian.expectedResponsePeerCount()) + assertEquals(0, expectedResponses.peerCount) } @Test @@ -381,10 +441,10 @@ class GuardianTest { now = now.plus(AttoTransactionStreamRequest.TIMEOUT) // When - guardian.count(request) + expectedResponses.record(request) // Then - assertEquals(0, guardian.expectedResponsePeerCount()) + assertEquals(0, expectedResponses.peerCount) } @Test @@ -392,7 +452,7 @@ class GuardianTest { // Given val node = createNode(AttoAmount.MAX) val message = inboundMessage(node.publicUri, randomSocketAddress()) - guardian.count(request(node, 2UL)) + expectedResponses.record(request(node, 2UL)) now = now.plusSeconds(59) guardian.count(message) @@ -402,7 +462,7 @@ class GuardianTest { guardian.guard() // Then - assertEquals(0, guardian.expectedResponsePeerCount()) + assertEquals(0, expectedResponses.peerCount) assertEquals(mapOf(message.socketAddress to 1UL), guardian.getSnapshot()) } @@ -412,8 +472,8 @@ class GuardianTest { val node = createNode(AttoAmount.MAX) val older = request(node, 2UL) now = now.plusSeconds(30) - guardian.count(request(node, 2UL)) - guardian.count(older) + expectedResponses.record(request(node, 2UL)) + expectedResponses.record(older) val message = inboundMessage(node.publicUri, randomSocketAddress()) // When @@ -422,11 +482,11 @@ class GuardianTest { guardian.guard() // Then - assertEquals(1, guardian.expectedResponsePeerCount()) + assertEquals(1, expectedResponses.peerCount) assertThat(guardian.getSnapshot()).isEmpty() now = now.plusSeconds(30) guardian.guard() - assertEquals(0, guardian.expectedResponsePeerCount()) + assertEquals(0, expectedResponses.peerCount) } @Test @@ -434,7 +494,7 @@ class GuardianTest { // Given val node = createNode(AttoAmount.MAX) val message = inboundMessage(node.publicUri, randomSocketAddress()) - guardian.count(request(node, 200UL)) + expectedResponses.record(request(node, 200UL)) val start = CountDownLatch(1) // When @@ -452,7 +512,7 @@ class GuardianTest { guardian.guard() // Then - assertEquals(0, guardian.expectedResponsePeerCount()) + assertEquals(0, expectedResponses.peerCount) assertEquals(mapOf(message.socketAddress to 200UL), guardian.getSnapshot()) } diff --git a/src/test/kotlin/cash/atto/node/network/guardian/PeerBansTest.kt b/src/test/kotlin/cash/atto/node/network/guardian/PeerBansTest.kt new file mode 100644 index 00000000..78e91555 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/guardian/PeerBansTest.kt @@ -0,0 +1,87 @@ +package cash.atto.node.network.guardian + +import cash.atto.node.EventPublisher +import cash.atto.node.network.NodeBanned +import cash.atto.node.network.NodeUnbanned +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress +import java.time.Clock +import java.time.Instant + +class PeerBansTest { + private var now = Instant.parse("2026-11-07T00:00:00Z") + private val clock = mockk { every { instant() } answers { now } } + private val publisher = mockk(relaxed = true) + private val bans = PeerBans(publisher, clock) + private val address = InetAddress.getLoopbackAddress() + + @Test + fun `ban and expiry change authoritative state before notification`() { + // Given + every { publisher.publish(any()) } answers { + when (val event = firstArg()) { + is NodeBanned -> assertTrue(bans.isBanned(event.address)) + is NodeUnbanned -> assertFalse(bans.isBanned(event.address)) + else -> error("Unexpected event $event") + } + } + + // When + bans.ban(address) + + // Then + assertTrue(bans.isBanned(address)) + assertEquals(setOf(address), bans.snapshot()) + verify(exactly = 1) { publisher.publish(NodeBanned(address, now)) } + + // When + now = now.plusSeconds(3_599) + bans.review() + + // Then + assertTrue(bans.isBanned(address)) + verify(exactly = 0) { publisher.publish(ofType()) } + + // When + now = now.plusSeconds(1) + bans.review() + bans.review() + + // Then + assertFalse(bans.isBanned(address)) + assertTrue(bans.snapshot().isEmpty()) + verify(exactly = 1) { publisher.publish(NodeUnbanned(address, now)) } + } + + @Test + fun `renewal owns a full hour and snapshots remain detached`() { + // Given + bans.ban(address) + val originalSnapshot = bans.snapshot() + now = now.plusSeconds(1_800) + bans.ban(address) + + // When + now = now.plusSeconds(1_800) + bans.review() + + // Then + assertTrue(bans.isBanned(address)) + verify(exactly = 0) { publisher.publish(ofType()) } + + // When + now = now.plusSeconds(1_800) + bans.review() + + // Then + assertFalse(bans.isBanned(address)) + assertEquals(setOf(address), originalSnapshot) + verify(exactly = 1) { publisher.publish(NodeUnbanned(address, now)) } + } +} diff --git a/src/test/kotlin/cash/atto/node/transaction/TransactionControllerElectionRejectionTest.kt b/src/test/kotlin/cash/atto/node/transaction/TransactionControllerElectionRejectionTest.kt index 50568e73..5939af2c 100644 --- a/src/test/kotlin/cash/atto/node/transaction/TransactionControllerElectionRejectionTest.kt +++ b/src/test/kotlin/cash/atto/node/transaction/TransactionControllerElectionRejectionTest.kt @@ -23,6 +23,7 @@ import cash.atto.node.election.Election import cash.atto.node.election.ElectionProperties import cash.atto.node.election.ElectionRejected import cash.atto.node.network.NetworkMessagePublisher +import cash.atto.node.network.guardian.ExpectedResponses import cash.atto.node.transaction.prioritization.TransactionPrioritizationProperties import cash.atto.node.transaction.prioritization.TransactionPrioritizer import cash.atto.node.transaction.validation.TransactionValidationManager @@ -59,6 +60,7 @@ import org.springframework.mock.http.server.reactive.MockServerHttpRequest import org.springframework.web.server.ResponseStatusException import java.net.InetSocketAddress import java.net.URI +import java.time.Clock import java.time.Instant import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.TimeUnit @@ -145,11 +147,16 @@ internal class TransactionControllerElectionRejectionTest { publisher, ) val controller = - TransactionController(ApplicationProperties(), node, NetworkMessagePublisher(context), repository) + TransactionPublicationController( + ApplicationProperties(), + node, + NetworkMessagePublisher(context, ExpectedResponses(Clock.systemUTC())), + repository, + ) val multicaster = ApplicationConfiguration().applicationEventMulticaster() as SimpleApplicationEventMulticaster multicaster.setErrorHandler { errors += it } context.beanFactory.registerSingleton("applicationEventMulticaster", multicaster) - context.beanFactory.registerSingleton("transactionController", controller) + context.beanFactory.registerSingleton("transactionPublicationController", controller) context.beanFactory.registerSingleton("transactionPrioritizer", prioritizer) context.beanFactory.registerSingleton("transactionValidationManager", validation) context.beanFactory.registerSingleton("election", election) diff --git a/src/test/kotlin/cash/atto/node/transaction/TransactionControllerQueryTest.kt b/src/test/kotlin/cash/atto/node/transaction/TransactionControllerQueryTest.kt new file mode 100644 index 00000000..aa2e65db --- /dev/null +++ b/src/test/kotlin/cash/atto/node/transaction/TransactionControllerQueryTest.kt @@ -0,0 +1,172 @@ +package cash.atto.node.transaction + +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoAmount +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoNetwork +import cash.atto.commons.AttoPublicKey +import cash.atto.commons.AttoReceiveBlock +import cash.atto.commons.AttoSignature +import cash.atto.commons.AttoTransaction +import cash.atto.commons.AttoWork +import cash.atto.commons.toAtto +import cash.atto.commons.toAttoHeight +import cash.atto.commons.toAttoVersion +import cash.atto.node.account.AccountUpdated +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +@OptIn(ExperimentalCoroutinesApi::class) +class TransactionControllerQueryTest { + private val publicKey = AttoPublicKey(ByteArray(32) { 1 }) + private val repository = mockk() + private val controller = TransactionController(repository) + + @Test + fun `hash stream subscribes before lookup and retains a concurrent committed result`() = + runTest { + // Given + val value = sample(2) + val finishRead = CompletableDeferred() + lateinit var publication: Deferred + coEvery { repository.findById(value.hash) } coAnswers { + publication = backgroundScope.async(start = CoroutineStart.UNDISPATCHED) { publish(value) } + finishRead.await() + null + } + val result = backgroundScope.async { controller.stream(value.hash).first() } + + // When + runCurrent() + + // Then + assertFalse(publication.isCompleted) + assertFalse(result.isCompleted) + + // When + finishRead.complete(Unit) + runCurrent() + + // Then + assertEquals(value.toAttoTransaction(), result.await()) + publication.await() + } + + @Test + fun `height stream orders snapshot and overlapping live updates and completes its range`() = + runTest { + // Given + val first = sample(2) + val second = sample(3, first.hash) + val third = sample(4, second.hash) + val finishRead = CompletableDeferred() + lateinit var overlap: Deferred + coEvery { repository.findAsc(any(), any(), any()) } coAnswers { + overlap = backgroundScope.async(start = CoroutineStart.UNDISPATCHED) { publish(second) } + flow { + emit(first) + emit(second) + finishRead.await() + } + } + val values = mutableListOf() + val collection = + backgroundScope.launch { + controller.stream(publicKey, 2UL.toAttoHeight(), 4UL.toAttoHeight()).toList(values) + } + + // When + runCurrent() + + // Then + assertFalse(overlap.isCompleted) + assertEquals(listOf(first, second).map { it.toAttoTransaction() }, values) + + // When + finishRead.complete(Unit) + runCurrent() + overlap.await() + publish(third) + runCurrent() + + // Then + collection.join() + assertTrue(collection.isCompleted) + assertEquals(listOf(first, second, third).map { it.toAttoTransaction() }, values) + } + + @Test + fun `cancelling a blocked snapshot cancels the query and releases the live publisher`() = + runTest { + // Given + val value = sample(2) + val cancelled = CompletableDeferred() + coEvery { repository.findAsc(any(), any(), any()) } returns + flow { + try { + awaitCancellation() + } finally { + cancelled.complete(Unit) + } + } + val collection = + backgroundScope.launch { + controller.stream(publicKey, 2UL.toAttoHeight(), 3UL.toAttoHeight()).toList() + } + runCurrent() + val publication = backgroundScope.async { publish(value) } + runCurrent() + assertFalse(publication.isCompleted) + + // When + collection.cancelAndJoin() + runCurrent() + + // Then + publication.await() + assertTrue(cancelled.isCompleted) + } + + private suspend fun publish(value: Transaction) { + controller.process(AccountUpdated(TransactionSource.BOOTSTRAP, mockk(), mockk(), value)) + } + + private fun sample( + height: Int, + previous: AttoHash = AttoHash(ByteArray(32)), + ): Transaction = + Transaction( + AttoReceiveBlock( + version = 0U.toAttoVersion(), + network = AttoNetwork.LOCAL, + algorithm = AttoAlgorithm.V1, + publicKey = publicKey, + height = height.toULong().toAttoHeight(), + balance = AttoAmount(height.toULong()), + timestamp = Instant.EPOCH.plusSeconds(height.toLong()).toAtto(), + previous = previous, + sendHashAlgorithm = AttoAlgorithm.V1, + sendHash = AttoHash(ByteArray(32) { height.toByte() }), + ), + AttoSignature(ByteArray(64)), + AttoWork(ByteArray(8)), + ) +} diff --git a/src/test/kotlin/cash/atto/node/transaction/TransactionNetworkProviderTest.kt b/src/test/kotlin/cash/atto/node/transaction/TransactionNetworkProviderTest.kt index e7786c96..ec926daf 100644 --- a/src/test/kotlin/cash/atto/node/transaction/TransactionNetworkProviderTest.kt +++ b/src/test/kotlin/cash/atto/node/transaction/TransactionNetworkProviderTest.kt @@ -13,8 +13,8 @@ import cash.atto.commons.toAttoHeight import cash.atto.commons.toAttoVersion import cash.atto.node.network.InboundNetworkMessage import cash.atto.node.network.MessageSource +import cash.atto.node.network.MutableConnectedPeers import cash.atto.node.network.NetworkMessagePublisher -import cash.atto.node.network.NodeConnected import cash.atto.protocol.AttoNode import cash.atto.protocol.AttoTransactionStreamRequest import cash.atto.protocol.NodeFeature @@ -160,6 +160,50 @@ class TransactionNetworkProviderTest { verify(exactly = 0) { publisher.publish(any()) } } + @Test + fun `disconnected requester cannot start repository work`() = + runTest { + // Given + val repository = mockk() + val fixture = fixture(repository) + fixture.connectedPeers.disconnect(fixture.peer.publicUri) + + // When + fixture.provider.stream(fixture.request(publicKey(1))) + + // Then + coVerify(exactly = 0) { repository.findDesc(any(), any(), any()) } + } + + @Test + fun `disconnect while publishing stops remaining materialized responses`() = + runTest { + // Given + val repository = mockk() + val first = transaction(publicKey(1)) + val second = transaction(publicKey(1), height = 2U, previous = first.hash) + coEvery { repository.findDesc(any(), any(), any()) } returns + flow { + emit(first) + emit(second) + } + val publisher = mockk() + val fixture = fixture(repository, publisher) + every { publisher.publish(any()) } answers { + fixture.connectedPeers.disconnect(fixture.peer.publicUri) + } + val request = + fixture.request(first.publicKey).copy( + payload = AttoTransactionStreamRequest(first.publicKey, 1U.toAttoHeight(), 2U.toAttoHeight()), + ) + + // When + fixture.provider.stream(request) + + // Then + verify(exactly = 1) { publisher.publish(any()) } + } + private fun fixture( repository: TransactionRepository, publisher: NetworkMessagePublisher = mockk(relaxed = true), @@ -167,14 +211,15 @@ class TransactionNetworkProviderTest { ): Fixture { val thisNode = node(100) val peer = node(101) - val provider = TransactionNetworkProvider(thisNode, repository, publisher, clock) - provider.add(NodeConnected(socketAddress(peer), peer)) - return Fixture(provider, peer) + val connectedPeers = MutableConnectedPeers().apply { connect(peer) } + val provider = TransactionNetworkProvider(thisNode, repository, publisher, clock, connectedPeers) + return Fixture(provider, peer, connectedPeers) } private data class Fixture( val provider: TransactionNetworkProvider, val peer: AttoNode, + val connectedPeers: MutableConnectedPeers, ) { fun request( publicKey: AttoPublicKey, @@ -222,17 +267,21 @@ class TransactionNetworkProviderTest { fun socketAddress(node: AttoNode): InetSocketAddress = InetSocketAddress(node.publicUri.host, node.publicUri.port) - fun transaction(publicKey: AttoPublicKey): Transaction { + fun transaction( + publicKey: AttoPublicKey, + height: UInt = 1U, + previous: AttoHash = AttoHash(ByteArray(32)), + ): Transaction { val block = AttoSendBlock( version = 0U.toAttoVersion(), network = AttoNetwork.LOCAL, algorithm = AttoAlgorithm.V1, publicKey = publicKey, - height = 1U.toAttoHeight(), + height = height.toAttoHeight(), balance = AttoAmount.MAX, timestamp = AttoInstant.now(), - previous = AttoHash(ByteArray(32)), + previous = previous, receiverAlgorithm = AttoAlgorithm.V1, receiverPublicKey = publicKey(102), amount = AttoAmount.MAX, diff --git a/src/test/kotlin/cash/atto/node/transaction/TransactionControllerTest.kt b/src/test/kotlin/cash/atto/node/transaction/TransactionPublicationControllerTest.kt similarity index 86% rename from src/test/kotlin/cash/atto/node/transaction/TransactionControllerTest.kt rename to src/test/kotlin/cash/atto/node/transaction/TransactionPublicationControllerTest.kt index b4525d4e..71bb978d 100644 --- a/src/test/kotlin/cash/atto/node/transaction/TransactionControllerTest.kt +++ b/src/test/kotlin/cash/atto/node/transaction/TransactionPublicationControllerTest.kt @@ -27,9 +27,13 @@ import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import kotlinx.coroutines.withTimeout @@ -48,7 +52,7 @@ import java.net.URI import kotlin.random.Random import kotlin.time.Duration.Companion.seconds -internal class TransactionControllerTest { +internal class TransactionPublicationControllerTest { private val privateKey = AttoPrivateKey.generate() private val block = AttoReceiveBlock( @@ -100,7 +104,7 @@ internal class TransactionControllerTest { val messagePublisher = mockk(relaxed = true) val repository = mockk() coEvery { repository.findById(transaction.hash) } returns transaction.takeIf { confirmed } - val controller = TransactionController(ApplicationProperties(), node, messagePublisher, repository) + val controller = TransactionPublicationController(ApplicationProperties(), node, messagePublisher, repository) val request = MockServerHttpRequest .post("/transactions") @@ -139,7 +143,7 @@ internal class TransactionControllerTest { val messagePublisher = mockk(relaxed = true) val repository = mockk() coEvery { repository.findById(transaction.hash) } throws IllegalStateException("Private database details") - val controller = TransactionController(ApplicationProperties(), node, messagePublisher, repository) + val controller = TransactionPublicationController(ApplicationProperties(), node, messagePublisher, repository) val request = MockServerHttpRequest .post("/transactions") @@ -179,7 +183,7 @@ internal class TransactionControllerTest { releaseLookup.await() null } - val controller = TransactionController(ApplicationProperties(), node, mockk(relaxed = true), repository) + val controller = TransactionPublicationController(ApplicationProperties(), node, mockk(relaxed = true), repository) val request = MockServerHttpRequest .post("/transactions") @@ -217,7 +221,7 @@ internal class TransactionControllerTest { runTest { // Given val repository = mockk() - val controller = TransactionController(ApplicationProperties(), node, mockk(), repository) + val controller = TransactionPublicationController(ApplicationProperties(), node, mockk(), repository) // When controller.process(ElectionRejected(account, transaction)) @@ -241,7 +245,7 @@ internal class TransactionControllerTest { publishSignals.trySend(Unit) Unit } - val controller = TransactionController(applicationProperties, node, messagePublisher, repository) + val controller = TransactionPublicationController(applicationProperties, node, messagePublisher, repository) val request = MockServerHttpRequest .post("/transactions/stream") @@ -292,7 +296,7 @@ internal class TransactionControllerTest { } val controller = - TransactionController( + TransactionPublicationController( applicationProperties, node, messagePublisher, @@ -339,7 +343,7 @@ internal class TransactionControllerTest { } val controller = - TransactionController( + TransactionPublicationController( applicationProperties, node, messagePublisher, @@ -380,7 +384,7 @@ internal class TransactionControllerTest { coEvery { repository.findById(transaction.hash) } returns transaction val controller = - TransactionController( + TransactionPublicationController( applicationProperties, node, messagePublisher, @@ -398,6 +402,44 @@ internal class TransactionControllerTest { verify(exactly = 0) { messagePublisher.publish(any()) } } + @Test + fun `cancelling the last waiter while another joins preserves the replacement confirmation`() = + runBlocking { + // Given + val messagePublisher = mockk(relaxed = true) + val repository = mockk() + coEvery { repository.findById(transaction.hash) } returns null + val controller = TransactionPublicationController(ApplicationProperties(), node, messagePublisher, repository) + val request = + MockServerHttpRequest + .post("/transactions") + .remoteAddress(InetSocketAddress("127.0.0.1", 12345)) + .build() + + withTimeout(10.seconds) { + repeat(100) { + coroutineScope { + val retiring = + async(start = CoroutineStart.UNDISPATCHED) { + controller.publish(transaction.toAttoTransaction(), request, deduplicate = true) + } + + // When + val cleanup = launch(Dispatchers.Default) { retiring.cancelAndJoin() } + val replacement = + async(start = CoroutineStart.UNDISPATCHED) { + controller.publishAndStream(transaction.toAttoTransaction(), request, deduplicate = true).first() + } + cleanup.join() + controller.process(AccountUpdated(TransactionSource.ELECTION, account, account, transaction)) + + // Then + assertEquals(transaction.toAttoTransaction(), replacement.await()) + } + } + } + } + @Test fun `publish waits for confirmation response`() = runTest { @@ -414,7 +456,7 @@ internal class TransactionControllerTest { } val controller = - TransactionController( + TransactionPublicationController( applicationProperties, node, messagePublisher, diff --git a/src/test/kotlin/cash/atto/node/vote/VoteServiceTest.kt b/src/test/kotlin/cash/atto/node/vote/VoteServiceTest.kt index d7e9234a..ff241bf3 100644 --- a/src/test/kotlin/cash/atto/node/vote/VoteServiceTest.kt +++ b/src/test/kotlin/cash/atto/node/vote/VoteServiceTest.kt @@ -9,6 +9,8 @@ import cash.atto.commons.toAttoVersion import io.mockk.coEvery import io.mockk.coVerify import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.awaitCancellation import kotlinx.coroutines.test.runTest import org.awaitility.Awaitility.await import org.junit.jupiter.api.Assertions.assertEquals @@ -38,7 +40,7 @@ internal class VoteServiceTest { service.enqueueAll(votes) // Then - awaitCondition { service.getBufferSize() == 0 } + awaitCondition { savedVotes.sumOf { it.size } == votes.size && service.getBufferSize() == 0 } assertEquals(listOf(votes.take(1_000), votes.drop(1_000)), savedVotes) } finally { service.stop() @@ -46,18 +48,19 @@ internal class VoteServiceTest { } @Test - fun `worker remains available after a failed insert`() = + fun `worker retries failed votes without another enqueue`() = runTest { // Given val repository = mockk() val service = VoteService(repository) val failedVote = Vote.sample() - val savedVote = Vote.sample() val attempts = AtomicInteger() + val savedVotes = CopyOnWriteArrayList() coEvery { repository.insertIgnoreAll(any()) } coAnswers { if (attempts.incrementAndGet() == 1) { throw IllegalStateException("db down") } + savedVotes.addAll(firstArg>()) 1 } @@ -66,15 +69,133 @@ internal class VoteServiceTest { service.enqueue(failedVote) // Then - awaitCondition { attempts.get() == 1 && service.getBufferSize() == 0 } + awaitCondition { savedVotes.size == 1 && service.getBufferSize() == 0 } + assertEquals(listOf(failedVote), savedVotes) + coVerify(exactly = 2) { repository.insertIgnoreAll(listOf(failedVote)) } + } finally { + service.stop() + } + } + + @Test + fun `failed batch stays ahead of votes arriving during persistence and keeps signature deduplication`() = + runTest { + // Given + val repository = mockk() + val service = VoteService(repository) + val firstVote = Vote.sample() + val firstBatch = listOf(firstVote, firstVote) + List(998) { Vote.sample() } + val laterVote = Vote.sample() + val firstAttemptStarted = CompletableDeferred() + val failFirstAttempt = CompletableDeferred() + val attemptedBatches = CopyOnWriteArrayList>() + val savedBatches = CopyOnWriteArrayList>() + coEvery { repository.insertIgnoreAll(any()) } coAnswers { + val votes = firstArg>().toList() + attemptedBatches += votes + if (attemptedBatches.size == 1) { + firstAttemptStarted.complete(Unit) + failFirstAttempt.await() + throw IllegalStateException("db down") + } + savedBatches += votes + votes.size.toLong() + } + + try { + // When + service.enqueueAll(firstBatch) + awaitCondition { firstAttemptStarted.isCompleted } + service.enqueue(laterVote) + failFirstAttempt.complete(Unit) + + // Then + awaitCondition { savedBatches.size == 2 && service.getBufferSize() == 0 } + val distinctBatch = firstBatch.distinctBy { it.signature } + assertEquals(listOf(distinctBatch, distinctBatch, listOf(laterVote)), attemptedBatches) + assertEquals(listOf(distinctBatch, listOf(laterVote)), savedBatches) + } finally { + failFirstAttempt.complete(Unit) + service.stop() + } + } + + @Test + fun `clear prevents an in flight failure from restoring cleared votes`() = + runTest { + // Given + val repository = mockk() + val service = VoteService(repository) + val clearedVote = Vote.sample() + val queuedVote = Vote.sample() + val newerVote = Vote.sample() + val firstAttemptStarted = CompletableDeferred() + val failFirstAttempt = CompletableDeferred() + val attempts = AtomicInteger() + val savedVotes = CopyOnWriteArrayList() + coEvery { repository.insertIgnoreAll(any()) } coAnswers { + if (attempts.incrementAndGet() == 1) { + firstAttemptStarted.complete(Unit) + failFirstAttempt.await() + throw IllegalStateException("db down") + } + val votes = firstArg>() + savedVotes.addAll(votes) + votes.size.toLong() + } + + try { + // When + service.enqueue(clearedVote) + awaitCondition { firstAttemptStarted.isCompleted } + service.enqueue(queuedVote) + service.clear() + service.enqueue(newerVote) + failFirstAttempt.complete(Unit) + + // Then + awaitCondition { savedVotes.size == 1 && service.getBufferSize() == 0 } + assertEquals(listOf(newerVote), savedVotes) + coVerify(exactly = 1) { repository.insertIgnoreAll(listOf(clearedVote)) } + coVerify(exactly = 0) { repository.insertIgnoreAll(match { queuedVote in it }) } + } finally { + failFirstAttempt.complete(Unit) + service.stop() + } + } + + @Test + fun `stopping the worker cancels persistence without retrying or dropping queued votes`() = + runTest { + // Given + val repository = mockk() + val service = VoteService(repository) + val vote = Vote.sample() + val attempts = AtomicInteger() + val saveStarted = CompletableDeferred() + val saveCancelled = CompletableDeferred() + coEvery { repository.insertIgnoreAll(any()) } coAnswers { + attempts.incrementAndGet() + saveStarted.complete(Unit) + try { + awaitCancellation() + } finally { + saveCancelled.complete(Unit) + } + } + try { // When - service.enqueue(savedVote) + service.enqueue(vote) + awaitCondition { saveStarted.isCompleted } + service.stop() // Then - awaitCondition { attempts.get() == 2 && service.getBufferSize() == 0 } - coVerify(exactly = 1) { repository.insertIgnoreAll(listOf(failedVote)) } - coVerify(exactly = 1) { repository.insertIgnoreAll(listOf(savedVote)) } + awaitCondition { saveCancelled.isCompleted && service.getBufferSize() == 1 } + await() + .during(50, TimeUnit.MILLISECONDS) + .atMost(5, TimeUnit.SECONDS) + .until { attempts.get() == 1 } } finally { service.stop() } diff --git a/src/test/kotlin/cash/atto/node/vote/keeping/VoteKeeperTest.kt b/src/test/kotlin/cash/atto/node/vote/keeping/VoteKeeperTest.kt new file mode 100644 index 00000000..eb378b38 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/vote/keeping/VoteKeeperTest.kt @@ -0,0 +1,204 @@ +package cash.atto.node.vote.keeping + +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoAmount +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoNetwork +import cash.atto.commons.AttoPublicKey +import cash.atto.commons.AttoSignature +import cash.atto.commons.AttoSignedVote +import cash.atto.commons.AttoSigner +import cash.atto.commons.AttoVote +import cash.atto.commons.toAttoVersion +import cash.atto.node.network.DirectNetworkMessage +import cash.atto.node.network.InboundNetworkMessage +import cash.atto.node.network.MessageSource +import cash.atto.node.network.MutableConnectedPeers +import cash.atto.node.network.NetworkMessagePublisher +import cash.atto.node.vote.MissingVote +import cash.atto.node.vote.VoteRepository +import cash.atto.node.vote.VoteService +import cash.atto.node.vote.weight.VoteWeighter +import cash.atto.protocol.AttoNode +import cash.atto.protocol.AttoVoteRequest +import cash.atto.protocol.AttoVoteResponse +import cash.atto.protocol.NodeFeature +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import java.net.InetSocketAddress +import java.net.URI + +class VoteKeeperTest { + @Test + fun `missing vote requires historical representative and uses current URI after reconnect`() = + runTest { + // Given + val representative = node(1).copy(features = setOf(NodeFeature.VOTING)) + val missing = MissingVote(hash(1), representative.publicKey) + val fixture = fixture(missing) + fixture.peers.connect(representative) + + // When + fixture.keeper.keep() + + // Then + verify(exactly = 0) { fixture.publisher.publish(any()) } + + // When + fixture.peers.disconnect(representative.publicUri) + val reconnected = + representative.copy( + publicUri = URI("ws://127.0.0.1:9001"), + features = setOf(NodeFeature.HISTORICAL), + ) + fixture.peers.connect(reconnected) + fixture.keeper.keep() + fixture.keeper.keep() + + // Then + verify(exactly = 1) { + fixture.publisher.publish( + match { + it is DirectNetworkMessage<*> && + it.publicUri == reconnected.publicUri && + (it.payload as AttoVoteRequest).blockHash == missing.lastTransactionHash + }, + ) + } + } + + @Test + fun `departed historical representative cannot receive a new missing vote request`() = + runTest { + // Given + val representative = node(1) + val fixture = fixture(MissingVote(hash(1), representative.publicKey)) + fixture.peers.connect(representative) + fixture.peers.disconnect(representative.publicUri) + + // When + fixture.keeper.keep() + + // Then + verify(exactly = 0) { fixture.publisher.publish(any()) } + } + + @Test + fun `local representative signs and enqueues missing final vote without a peer connection`() = + runTest { + // Given + val thisNode = node(100) + val missing = MissingVote(hash(1), thisNode.publicKey) + val fixture = fixture(missing, thisNode) + + // When + fixture.keeper.keep() + + // Then + coVerify(exactly = 1) { + fixture.signer.sign( + match { + it.publicKey == thisNode.publicKey && + it.blockHash == missing.lastTransactionHash && + it.timestamp == AttoVote.finalTimestamp + }, + ) + } + verify(exactly = 1) { + fixture.service.enqueue( + match { + it.publicKey == thisNode.publicKey && it.blockHash == missing.lastTransactionHash && it.isFinal() + }, + ) + } + verify(exactly = 0) { fixture.publisher.publish(any()) } + } + + @Test + fun `pending response remains consumable after representative disconnects`() = + runTest { + // Given + val representative = node(1) + val missing = MissingVote(hash(1), representative.publicKey) + val fixture = fixture(missing) + fixture.peers.connect(representative) + fixture.keeper.keep() + val signedVote = + AttoSignedVote( + AttoVote( + version = 0U.toAttoVersion(), + algorithm = AttoAlgorithm.V1, + publicKey = representative.publicKey, + blockAlgorithm = AttoAlgorithm.V1, + blockHash = missing.lastTransactionHash, + timestamp = AttoVote.finalTimestamp, + ), + AttoSignature(ByteArray(64) { 1 }), + ) + val response = + InboundNetworkMessage( + MessageSource.WEBSOCKET, + representative.publicUri, + InetSocketAddress("127.0.0.1", 10_001), + AttoVoteResponse(signedVote), + ) + fixture.peers.disconnect(representative.publicUri) + + // When + fixture.keeper.process(response) + fixture.keeper.process(response) + + // Then + verify(exactly = 1) { + fixture.service.enqueue( + match { + it.publicKey == representative.publicKey && it.blockHash == missing.lastTransactionHash + }, + ) + } + } + + private fun fixture( + missing: MissingVote, + thisNode: AttoNode = node(100), + ): Fixture { + val weight = AttoAmount(1_000UL) + val voteWeighter = mockk() + every { voteWeighter.getMinimalConfirmationWeight() } returns weight + every { voteWeighter.get(any()) } returns weight + val repository = mockk() + coEvery { repository.findMissingVote(any()) } returns listOf(missing) + val service = mockk(relaxed = true) + val publisher = mockk(relaxed = true) + val signer = mockk() + coEvery { signer.sign(any()) } returns AttoSignature(ByteArray(64) { 1 }) + val peers = MutableConnectedPeers() + val keeper = VoteKeeper(thisNode, voteWeighter, repository, service, publisher, signer, peers) + return Fixture(keeper, service, publisher, signer, peers) + } + + private data class Fixture( + val keeper: VoteKeeper, + val service: VoteService, + val publisher: NetworkMessagePublisher, + val signer: AttoSigner, + val peers: MutableConnectedPeers, + ) + + private fun node(marker: Byte): AttoNode = + AttoNode( + network = AttoNetwork.LOCAL, + protocolVersion = AttoNode.CURRENT_PROTOCOL_VERSION, + algorithm = AttoAlgorithm.V1, + publicKey = AttoPublicKey(ByteArray(32) { marker }), + publicUri = URI("ws://127.0.0.1:${8_000 + marker}"), + features = setOf(NodeFeature.HISTORICAL, NodeFeature.VOTING), + ) + + private fun hash(marker: Byte): AttoHash = AttoHash(ByteArray(32) { marker }) +}