diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 3fd940fda5e..be9dff5951c 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -42,6 +42,12 @@ public static class Gauge { public static final String DB_SST_LEVEL = "tron:db_sst_level"; public static final String MANAGER_QUEUE = "tron:manager_queue_size"; public static final String TX_CACHE = "tron:tx_cache"; + // per local consensus witness miner; value 1, label is the Base58Check witness address + public static final String WITNESS_ADDRESS = "tron:witness_address"; + // numeric backup-election role of this node: 0 = init, 1 = master, 2 = slaver + public static final String NODE_ROLE = "tron:node_role"; + // witness flag of this node: 0 = not witness, 1 = witness + public static final String IS_WITNESS = "tron:is_witness"; private Gauge() { throw new IllegalStateException("Gauge"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index 10fac2a98e0..f576bf7936b 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -62,6 +62,9 @@ public static class Gauge { public static final String PEERS_VALID = "valid"; + // ADDRESS is the label of tron:witness_address (node_role and is_witness carry no label) + public static final String ADDRESS = "address"; + private Gauge() { throw new IllegalStateException("Gauge"); } diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index dc1e5447540..276d3cd89c2 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -19,6 +19,15 @@ class MetricsGauge { init(MetricKeys.Gauge.DB_SIZE_BYTES, "tron db size .", "type", "db", "level"); init(MetricKeys.Gauge.DB_SST_LEVEL, "tron db files .", "type", "db", "level"); init(MetricKeys.Gauge.TX_CACHE, "tron tx cache info.", "type"); + init(MetricKeys.Gauge.WITNESS_ADDRESS, + "1 for each consensus witness address this node holds a miner for.", + MetricLabels.Gauge.ADDRESS); + init(MetricKeys.Gauge.IS_WITNESS, + "Whether this node runs in witness mode, encoded as a number: 0 = not witness, " + + "1 = witness."); + init(MetricKeys.Gauge.NODE_ROLE, + "Backup-election role of this node, encoded as a number: 0 = init, " + + "1 = master, 2 = slaver."); } private MetricsGauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 0c547a38e2c..82516b78bd9 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -10,6 +10,39 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } + // is_witness: 0 = not witness, 1 = witness + public static final int NOT_WITNESS = 0; + public static final int WITNESS = 1; + + // node_role: 0 = init (pre-election / re-election), 1 = master, 2 = slaver + public static final int NODE_ROLE_INIT = 0; + public static final int NODE_ROLE_MASTER = 1; + public static final int NODE_ROLE_SLAVER = 2; + + /** Publishes is_witness as one label-less series; 0 = not witness, 1 = witness. */ + public static void recordIsWitness(int isWitness) { + if (!Metrics.enabled()) { + return; + } + Metrics.gaugeSet(MetricKeys.Gauge.IS_WITNESS, isWitness); + } + + /** Publishes node_role as one label-less series; role changes rewrite it in place. */ + public static void recordNodeRole(int role) { + if (!Metrics.enabled()) { + return; + } + Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, role); + } + + /** Publishes one tron:witness_address series (value 1) for a local witness address. */ + public static void recordWitnessAddress(String address) { + if (!Metrics.enabled()) { + return; + } + Metrics.gaugeSet(MetricKeys.Gauge.WITNESS_ADDRESS, 1, address); + } + public static void recordSrSetChange(List currentWits, List newWits) { if (!Metrics.enabled()) { return; diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 46c0402919a..971f6979e1c 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -10,6 +10,9 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:is_witness` (Gauge, no labels) — whether this node runs in witness mode, encoded as a number: `0` = `not witness` (started without witness mode; witness keys are not loaded and no `tron:witness_address` series is published), `1` = `witness` (started with `-w`). It is a process identity, published once at startup and independent of the backup-election role. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_role` (Gauge, no labels) — backup-election role of this node, encoded as a number: `0` = `init` (backup election configured but not yet run, and again while a re-election is running after the master lost keep-alive), `1` = `master` (the backup primary; also the standalone default for both node types when backup is not configured), `2` = `slaver` (standing by for a master). The role is a node-level status, independent of `tron:is_witness`, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:witness_address` (Gauge, label `address`) — value `1` for each consensus witness miner this node holds; the label is the Base58Check-encoded witness account address. The set is assembled once at startup from the finalized consensus miners: one series per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. With several witness keys, each address gets its own series. A plain fullnode without a witness miner publishes no `tron:witness_address` series at all. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_already_known` (Counter) — incremented when a block response matches an outstanding adv request whose exact block ID is already known before that response is processed (best-effort signal; concurrent arrivals may be missed). ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 497f38544bc..2352258cec4 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -14,6 +14,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -25,6 +26,8 @@ import org.tron.common.backup.socket.UdpEvent; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; import org.tron.p2p.utils.NetUtil; @Slf4j(topic = "backup") @@ -67,6 +70,41 @@ public class BackupManager implements EventHandler { public void setStatus(BackupStatusEnum status) { logger.info("Change backup status to {}", status); this.status = status; + recordNodeRole(); + } + + /** Publishes tron:node_role once at startup, and again on every backup-status change. */ + @PostConstruct + public synchronized void recordNodeRole() { + if (!Metrics.enabled()) { + return; + } + SRMetrics.recordNodeRole(currentRole()); + } + + private int currentRole() { + // no backup configured: default to master, for either identity + if (!isBackupConfigured()) { + return SRMetrics.NODE_ROLE_MASTER; + } + // status still defaults to MASTER before backup init; not a real election result yet + if (!isInit) { + return SRMetrics.NODE_ROLE_INIT; + } + // post-init INIT covers the pre-keep-alive window and master-loss re-election + switch (status) { + case MASTER: + return SRMetrics.NODE_ROLE_MASTER; + case SLAVER: + return SRMetrics.NODE_ROLE_SLAVER; + default: + return SRMetrics.NODE_ROLE_INIT; + } + } + + private boolean isBackupConfigured() { + // same condition BackupServer.initServer() uses to start the election transport + return parameter.getBackupPort() > 0 && !parameter.getBackupMembers().isEmpty(); } public void init() { diff --git a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java index b3921b1b657..9a0332e112f 100644 --- a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java +++ b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java @@ -11,6 +11,9 @@ import org.springframework.stereotype.Component; import org.tron.common.crypto.SignUtils; import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.StringUtil; import org.tron.consensus.Consensus; import org.tron.consensus.base.Param; import org.tron.consensus.base.Param.Miner; @@ -77,12 +80,47 @@ public void start() { } param.setMiners(miners); + recordWitnessAddresses(miners); + recordIsWitness(); param.setBlockHandle(blockHandle); param.setPbftInterface(pbftBaseImpl); consensus.start(param); logger.info("consensus service start success"); } + private void recordWitnessAddresses(List miners) { + if (!Metrics.enabled() || miners == null) { + return; + } + for (Miner miner : miners) { + ByteString witnessAddress = miner.getWitnessAddress(); + try { + String address = StringUtil.encode58Check(witnessAddress.toByteArray()); + SRMetrics.recordWitnessAddress(address); + } catch (Exception e) { + logger.warn("Failed to record witness address metrics.", e); + } + } + } + + /** + * Publishes tron:is_witness once at consensus startup; 0 = not witness, 1 = witness. It is + * published independently of the witness-address publication, and even when no miner is + * configured, so the flag series always exists. Catches its own failure so a metrics + * problem can neither skip the witness publication nor abort consensus startup. + */ + private void recordIsWitness() { + if (!Metrics.enabled()) { + return; + } + try { + SRMetrics.recordIsWitness(Args.getInstance().isWitness() + ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS); + } catch (Exception e) { + logger.warn("Failed to record is-witness metrics.", e); + } + } + public void stop() { logger.info("consensus service closed start."); consensus.stop(); diff --git a/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java b/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java index 0c754a0ac6d..b6326c72929 100644 --- a/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java +++ b/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java @@ -1,6 +1,7 @@ package org.tron.common.backup; import io.netty.channel.Channel; +import io.prometheus.client.CollectorRegistry; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -20,14 +21,19 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import org.tron.common.TestConstants; import org.tron.common.backup.BackupManager.BackupStatusEnum; import org.tron.common.backup.message.KeepAliveMessage; import org.tron.common.backup.socket.BackupServer; +import org.tron.common.backup.socket.MessageHandler; import org.tron.common.backup.socket.UdpEvent; import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.SRMetrics; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.ReflectUtils; +import org.tron.consensus.base.Param; import org.tron.core.config.args.Args; import org.tron.core.config.args.InetUtil; @@ -76,6 +82,9 @@ public void tearDown() throws Exception { }); InetUtil.dnsLookup = previousDnsLookup; Args.clearParam(); + // the consensus miner list is a process-wide singleton: reset it so this class + // cannot leak state into other tests of the same JVM + Param.getInstance().setMiners(new ArrayList<>()); BackupTestUtils.throwIfAnyError(errors); } @@ -183,6 +192,64 @@ public void testBackupServerLifecycleDuringKeepAliveInterval() throws Exception BackupTestUtils.assertExecutorsTerminated(manager, backupServer); } + /** + * A backup group whose only member is this node's own address is a "single-master" group: + * no peer ever answers a keep-alive, so the node must leave the pre-election INIT window and + * self-promote to MASTER once the keep-alive timeout elapses. + * + *

Drives the real {@link BackupManager#init()} and its real scheduled keep-alive task. The + * timeout condition is made deterministic by ageing the last keep-alive time, so the first + * (fixed 1s) scheduler tick is guaranteed to observe an expired keep-alive; recovery is then + * awaited by polling, so there is no fixed sleep. The published {@code tron:node_role} must + * track the transition (0 = init, 1 = master) and the manager executor must terminate. + */ + @Test(timeout = 30000) + public void singleMasterGroupRecoversFromInitToMaster() throws Exception { + CommonParameter parameter = CommonParameter.getInstance(); + parameter.setBackupPriority(8); + // the group's only member is ourselves, so init() drops it and no peer can answer + parameter.setBackupMembers( + Collections.singletonList(InetAddress.getLocalHost().getHostAddress())); + + boolean previousMetrics = Args.getInstance().isMetricsPrometheusEnable(); + Args.getInstance().setMetricsPrometheusEnable(true); + BackupManager single = new BackupManager(); + // absorb any keep-alive send in case the local IP lookup behaves differently + single.setMessageHandler(Mockito.mock(MessageHandler.class)); + try { + // backup configured but not yet init'ed: the pre-election window reports init (0) + Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, single.getStatus()); + single.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, nodeRoleGauge(), 0.0); + + single.init(); + Assert.assertEquals(BackupManager.BackupStatusEnum.INIT, single.getStatus()); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, nodeRoleGauge(), 0.0); + + // no keep-alive has ever arrived, so the next keep-alive tick must self-promote. + // setStatus() flips the volatile status before it republishes the gauge, so wait on + // the published series (the externally visible result), not on the status field alone. + ReflectUtils.setFieldValue(single, "lastKeepAliveTime", 0L); + BackupTestUtils.awaitCondition("lone master to recover from INIT", + () -> single.getStatus() == BackupManager.BackupStatusEnum.MASTER + && Double.valueOf(SRMetrics.NODE_ROLE_MASTER).equals(nodeRoleGauge())); + Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, single.getStatus()); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, nodeRoleGauge(), 0.0); + } finally { + single.stop(); + Args.getInstance().setMetricsPrometheusEnable(previousMetrics); + } + + ScheduledExecutorService executorService = + (ScheduledExecutorService) ReflectUtils.getFieldObject(single, "executorService"); + Assert.assertTrue("single-master executor must terminate", executorService.isTerminated()); + } + + private Double nodeRoleGauge() { + return CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_ROLE, new String[] {}, new String[] {}); + } + // ===== domain-handling tests for init() ===== @Test(timeout = 5000) diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 4c1404bb232..968b0e5363d 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -2,8 +2,12 @@ import com.google.protobuf.ByteString; import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.common.TextFormat; +import java.io.StringWriter; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -11,10 +15,18 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; import org.tron.common.BaseTest; import org.tron.common.TestConstants; +import org.tron.common.backup.BackupManager; +import org.tron.common.crypto.SignUtils; +import org.tron.common.utils.ByteArray; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.StringUtil; +import org.tron.consensus.Consensus; +import org.tron.consensus.base.Param; import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.VotesCapsule; @@ -34,6 +46,8 @@ public class SRMetricsTest extends BaseTest { private MaintenanceManager maintenanceManager; @Resource private ConsensusService consensusService; + @Resource + private BackupManager backupManager; static { Args.setParam(new String[]{"-d", dbPath()}, TestConstants.TEST_CONF); @@ -52,6 +66,11 @@ public void setUp() { @After public void tearDown() { Args.getInstance().setMetricsPrometheusEnable(true); + Args.getInstance().setBackupPort(0); + Args.getInstance().setBackupMembers(Collections.emptyList()); + Args.getInstance().witness = false; + Param.getInstance().setMiners(new ArrayList<>()); + backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); } /** @@ -169,6 +188,432 @@ public void testSrRemoveDirect() { Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, stableWit)); } + // ---- tron:is_witness: process flag, published at startup and independent of the role ---- + + /** + * Flag is published once at startup by ConsensusService: 0 for a node started without + * witness mode, 1 for a node started with -w. + */ + @Test + public void testIsWitnessBothValues() { + for (boolean witness : new boolean[] {false, true}) { + Args.getInstance().witness = witness; + publishIsWitness(); + int expected = witness ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS; + Assert.assertEquals(expected, isWitnessGauge(), 0.0); + } + } + + /** + * Metrics disabled: recordIsWitness() short-circuits, so the previously published value stands + * even when the flag would have changed. + */ + @Test + public void testIsWitnessNotPublishedWhenMetricsDisabled() { + Args.getInstance().witness = false; + publishIsWitness(); + Assert.assertEquals(SRMetrics.NOT_WITNESS, isWitnessGauge(), 0.0); + + Args.getInstance().setMetricsPrometheusEnable(false); + try { + Args.getInstance().witness = true; + publishIsWitness(); + Assert.assertEquals(SRMetrics.NOT_WITNESS, isWitnessGauge(), 0.0); + } finally { + Args.getInstance().setMetricsPrometheusEnable(true); + } + } + + /** + * Metrics disabled: recordNodeRole() short-circuits, so the previously published role stands + * even when the backup configuration would resolve to a different one. + */ + @Test + public void testNodeRoleNotPublishedWhenMetricsDisabled() { + Args.getInstance().witness = false; + // no backup configured: baseline master (1) + backupManager.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + + // a configuration that would resolve to init (0) if it were published + Args.getInstance().setBackupPort(PublicMethod.chooseRandomPort()); + Args.getInstance().setBackupMembers(Collections.singletonList("127.0.0.2")); + Args.getInstance().setMetricsPrometheusEnable(false); + try { + backupManager.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + } finally { + Args.getInstance().setMetricsPrometheusEnable(true); + Args.getInstance().setBackupPort(0); + Args.getInstance().setBackupMembers(Collections.emptyList()); + } + } + + /** + * A failure while publishing is_witness must not propagate: ConsensusService catches it, so it + * can neither skip the witness-address publication running right beside it nor abort startup. + */ + @Test + public void testIsWitnessFailureDoesNotBlockWitnessPublication() { + Args.getInstance().witness = false; + ByteString address = uniqueAddress(); + List miners = Collections.singletonList(miner(address, address)); + + try (MockedStatic mocked = Mockito.mockStatic(SRMetrics.class)) { + mocked.when(() -> SRMetrics.recordIsWitness(Mockito.anyInt())) + .thenThrow(new RuntimeException("is_witness failure")); + mocked.when(() -> SRMetrics.recordWitnessAddress(Mockito.anyString())) + .thenCallRealMethod(); + + // must not propagate the is_witness failure + publishIsWitness(); + + // the witness-address publication still runs independently on the same startup path + ReflectionTestUtils.invokeMethod(consensusService, "recordWitnessAddresses", miners); + } + + Assert.assertEquals(1.0, witnessGauge(encode(address)), 0.0); + } + + /** + * The flag metric must be published by {@link ConsensusService#start()} itself, not by a + * caller: with the election engine stubbed out, a real start() has to write tron:is_witness + * from Args.isWitness(), even for a node with zero miners. Pre-setting the opposite value pins + * the call site, so dropping the publication from start() fails here. + */ + @Test + public void testStartPublishesIsWitnessForZeroMiners() { + Consensus consensusMock = Mockito.mock(Consensus.class); + ConsensusService service = new ConsensusService(); + ReflectionTestUtils.setField(service, "consensus", consensusMock); + + for (boolean witness : new boolean[] {false, true}) { + Args.getInstance().witness = witness; + // opposite of the expected value, so a start() that skipped the publication cannot pass + SRMetrics.recordIsWitness(witness ? SRMetrics.NOT_WITNESS + : SRMetrics.WITNESS); + + service.start(); + + int expected = witness ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS; + Assert.assertEquals(expected, isWitnessGauge(), 0.0); + } + + Mockito.verify(consensusMock, Mockito.times(2)).start(Mockito.any(Param.class)); + } + + // ---- tron:node_role: the three states, the pre-election window and status transitions ---- + + /** + * No backup configured: both identities report master (1) as the standalone default; the + * backup status defaults to MASTER before any election runs. + */ + @Test + public void testNodeRoleMasterWhenNoBackupBothIdentities() { + Args.getInstance().witness = false; + backupManager.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + + Args.getInstance().witness = true; + backupManager.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + } + + /** + * A partial backup configuration — a usable port without members, or members without a usable + * port — is not an election setup (the same condition BackupServer.initServer() uses), so the + * node reports master (1) for both identities and no transport is started. + */ + @Test + public void testNodeRoleMasterWhenBackupPartiallyConfigured() { + for (boolean witness : new boolean[] {false, true}) { + for (boolean portUsable : new boolean[] {false, true}) { + Args.getInstance().witness = witness; + Args.getInstance().setBackupPort(portUsable ? PublicMethod.chooseRandomPort() : 0); + Args.getInstance().setBackupMembers(portUsable + ? Collections.emptyList() : Collections.singletonList("127.0.0.2")); + + // one half of the pair is always unset, so this is never a usable election config + BackupManager fresh = new BackupManager(); + try { + fresh.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, fresh.getStatus()); + } finally { + fresh.stop(); + } + } + } + } + + /** + * Backup configured but the election has not run yet: even though the status field still + * defaults to MASTER, the node reports init (0), for both identities; a status change before + * init() stays inside the window, and after init() the status maps to the elected role. + */ + @Test + public void testNodeRoleInitBeforeBackupElectionBothIdentities() throws Exception { + configureBackup(); + + for (boolean witness : new boolean[] {false, true}) { + BackupManager fresh = new BackupManager(); + try { + Args.getInstance().witness = witness; + + // a directly constructed manager has not gone through init(): default status is MASTER + Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, fresh.getStatus()); + fresh.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + + // a status change before init() stays inside the window + fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + + // once the election has run, the status maps to the elected role + markBackupInitiated(fresh); + fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); + } finally { + fresh.stop(); + } + } + } + + /** + * Flag and role are fully independent: for both flag values the startup publication is + * not-witness/witness + init, and the role then cycles INIT -> MASTER -> SLAVER -> MASTER while + * is_witness never changes. Every publication rewrites the single label-less series in place, + * so no duplicate registration or stale role series accumulates. + */ + @Test + public void testIsWitnessIndependentOfRoleBothValues() throws Exception { + configureBackup(); + + BackupManager.BackupStatusEnum[] statuses = { + BackupManager.BackupStatusEnum.INIT, + BackupManager.BackupStatusEnum.MASTER, + BackupManager.BackupStatusEnum.SLAVER, + BackupManager.BackupStatusEnum.MASTER}; + int[] roles = { + SRMetrics.NODE_ROLE_INIT, + SRMetrics.NODE_ROLE_MASTER, + SRMetrics.NODE_ROLE_SLAVER, + SRMetrics.NODE_ROLE_MASTER}; + + for (boolean witness : new boolean[] {false, true}) { + BackupManager fresh = new BackupManager(); + try { + Args.getInstance().witness = witness; + int expectedIsWitness = witness ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS; + + // startup: is_witness comes from ConsensusService, node_role from BackupManager; + // configured-but-not-init is INIT, not the MASTER default + publishIsWitness(); + fresh.recordNodeRole(); + Assert.assertEquals(expectedIsWitness, isWitnessGauge(), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + + // flip the startup flag: setStatus() must only republish the role, never recompute + // is_witness, so the originally published flag has to keep standing + Args.getInstance().witness = !witness; + markBackupInitiated(fresh); + for (int i = 0; i < statuses.length; i++) { + fresh.setStatus(statuses[i]); + Assert.assertEquals(roles[i], roleGauge(), 0.0); + Assert.assertEquals("is_witness must not change with the role", expectedIsWitness, + isWitnessGauge(), 0.0); + } + + Assert.assertEquals(1, sampleCount(MetricKeys.Gauge.NODE_ROLE)); + Assert.assertEquals(1, sampleCount(MetricKeys.Gauge.IS_WITNESS)); + } finally { + fresh.stop(); + } + } + } + + /** + * setStatus() itself republishes the single label-less node_role series, so a + * backup-election transition is reflected immediately. + */ + @Test + public void testNodeRoleRefreshOnBackupStatusChange() throws Exception { + Args.getInstance().witness = true; + configureBackup(); + markBackupInitiated(backupManager); + + backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); + + backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + } + + // ---- tron:witness_address: one series per consensus miner, value 1 ---- + + /** + * A single consensus miner publishes exactly one tron:witness_address series with value 1. + */ + @Test + public void testWitnessAddressValueOne() { + ByteString address = uniqueAddress(); + recordWitnessMetrics( + Collections.singletonList(miner(address, address))); + Assert.assertEquals(1.0, witnessGauge(encode(address)), 0.0); + } + + /** + * Multi-key setup: every consensus miner address gets its own series with value 1. + */ + @Test + public void testWitnessAddressMultipleMiners() { + ByteString address1 = uniqueAddress(); + ByteString address2 = uniqueAddress(); + List miners = new ArrayList<>(); + miners.add(miner(address1, address1)); + miners.add(miner(address2, address2)); + + recordWitnessMetrics(miners); + + Assert.assertEquals(1.0, witnessGauge(encode(address1)), 0.0); + Assert.assertEquals(1.0, witnessGauge(encode(address2)), 0.0); + } + + /** + * Single-key multisig: the witness account address may differ from the address derived + * from the signing key. Only the witness account address is published; the signing-key + * address must not appear. + */ + @Test + public void testWitnessAddressUsesWitnessAccountNotSigningKey() { + ByteString signingKeyAddress = addressOf(PublicMethod.getRandomPrivateKey()); + ByteString witnessAccountAddress = uniqueAddress(); + + recordWitnessMetrics( + Collections.singletonList(miner(signingKeyAddress, witnessAccountAddress))); + + Assert.assertEquals(1.0, witnessGauge(encode(witnessAccountAddress)), 0.0); + Assert.assertNull(witnessGauge(encode(signingKeyAddress))); + } + + /** + * No miner configured: a plain fullnode publishes no tron:witness_address series at all, + * not even one with an empty address label. + */ + @Test + public void testNoWitnessAddressSeriesForEmptyMiners() { + recordWitnessMetrics(new ArrayList<>()); + Assert.assertNull(witnessGauge("")); + } + + /** + * Metrics disabled: no tron:witness_address series is published. + */ + @Test + public void testWitnessAddressNotPublishedWhenMetricsDisabled() { + ByteString address = uniqueAddress(); + Args.getInstance().setMetricsPrometheusEnable(false); + try { + recordWitnessMetrics( + Collections.singletonList(miner(address, address))); + Assert.assertNull(witnessGauge(encode(address))); + } finally { + Args.getInstance().setMetricsPrometheusEnable(true); + } + } + + /** + * Backup must be seen as configured the same way BackupServer.initServer() decides to + * start the election transport, otherwise the pre-election window is unreachable. + */ + private void configureBackup() { + Args.getInstance().setBackupPort(PublicMethod.chooseRandomPort()); + Args.getInstance().setBackupMembers(Collections.singletonList("127.0.0.2")); + } + + /** + * Simulates that BackupManager.init() has run: in production the backup server only + * starts after the first fresh block, so tests drive the isInit flag directly instead + * of spinning up the election transport. + */ + private void markBackupInitiated(BackupManager manager) throws Exception { + java.lang.reflect.Field isInit = BackupManager.class.getDeclaredField("isInit"); + isInit.setAccessible(true); + isInit.setBoolean(manager, true); + } + + /** + * Drives the startup flag publication the way production does: ConsensusService.start() + * calls its private recordIsWitness() hook right after assembling the miners. + */ + private void publishIsWitness() { + ReflectionTestUtils.invokeMethod(consensusService, "recordIsWitness"); + } + + /** + * Drives the startup publication the way production does: the miners are assembled by + * ConsensusService, which publishes the witness identities once during startup. + */ + private void recordWitnessMetrics(List miners) { + Param.getInstance().setMiners(miners); + ReflectionTestUtils.invokeMethod(consensusService, "recordWitnessAddresses", miners); + } + + /** + * Builds the consensus-miner identity the node assembles: the signing-key address and the + * witness account address are passed explicitly, exactly as ConsensusService does. + */ + private Param.Miner miner(ByteString privateKeyAddress, ByteString witnessAddress) { + return Param.getInstance().new Miner(new byte[0], privateKeyAddress, witnessAddress); + } + + private ByteString addressOf(String privateKeyHex) { + return ByteString.copyFrom( + SignUtils.fromPrivate(ByteArray.fromHexString(privateKeyHex)).getAddress()); + } + + private String encode(ByteString address) { + return StringUtil.encode58Check(address.toByteArray()); + } + + private Double witnessGauge(String address) { + return CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.WITNESS_ADDRESS, + new String[] {MetricLabels.Gauge.ADDRESS}, + new String[] {address}); + } + + private Double roleGauge() { + return CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_ROLE, + new String[] {}, + new String[] {}); + } + + private Double isWitnessGauge() { + return CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.IS_WITNESS, + new String[] {}, + new String[] {}); + } + + /** Counts the exported sample lines of a label-less metric, to detect duplicate registration. */ + private int sampleCount(String metricName) { + StringWriter writer = new StringWriter(); + try { + TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples()); + } catch (Exception e) { + throw new IllegalStateException(e); + } + int count = 0; + for (String line : writer.toString().split("\n")) { + if (line.startsWith(metricName + " ") || line.startsWith(metricName + "{")) { + count++; + } + } + return count; + } + private ByteString registerWitness() { ByteString address = uniqueAddress(); chainBaseManager.getWitnessStore().put(address.toByteArray(), new WitnessCapsule(address)); diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index 77e9a95ad84..7dae98638ab 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -3,6 +3,8 @@ import com.google.common.collect.Maps; import com.google.protobuf.ByteString; import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.common.TextFormat; +import java.io.StringWriter; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -24,6 +26,7 @@ import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.MetricLabels; import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; import org.tron.common.utils.ByteArray; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.Sha256Hash; @@ -224,6 +227,50 @@ public void testNodeInfoMetric() { Assert.assertEquals(1.0, value, 0.0); } + @Test + public void testNodeRoleMetric() { + // a single label-less series per node; the value encodes the role (0 = init, + // 1 = master, 2 = slaver), here master after recording NODE_ROLE_MASTER + SRMetrics.recordNodeRole(SRMetrics.NODE_ROLE_MASTER); + Double value = CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_ROLE, + new String[] {}, + new String[] {}); + Assert.assertNotNull("tron:node_role sample should exist", value); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, value, 0.0); + } + + @Test + public void testIsWitnessMetric() { + // label-less flag series: 0 = not witness, 1 = witness + SRMetrics.recordIsWitness(SRMetrics.WITNESS); + Double value = CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.IS_WITNESS, + new String[] {}, + new String[] {}); + Assert.assertNotNull("tron:is_witness sample should exist", value); + Assert.assertEquals(SRMetrics.WITNESS, value, 0.0); + } + + @Test + public void testNodeGaugesTextExport() throws Exception { + SRMetrics.recordIsWitness(SRMetrics.NOT_WITNESS); + SRMetrics.recordNodeRole(SRMetrics.NODE_ROLE_INIT); + + StringWriter writer = new StringWriter(); + TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples()); + String text = writer.toString(); + + Assert.assertTrue("scrape output should expose tron:is_witness", + text.contains("tron:is_witness ")); + Assert.assertTrue("scrape output should expose tron:node_role", + text.contains("tron:node_role ")); + Assert.assertTrue("tron:is_witness should export the recorded value", + text.contains("tron:is_witness 0.0")); + Assert.assertTrue("tron:node_role should export the recorded value", + text.contains("tron:node_role 0.0")); + } + @Test public void testNodeInfoUnknownKey() { // unknown key exercises the null-guard branch in MetricsInfo.set diff --git a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java index 311e1887188..398e3595bb6 100644 --- a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java @@ -6,6 +6,7 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; @@ -20,6 +21,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.InOrder; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.tron.common.application.Application; @@ -100,8 +102,9 @@ private void assertStartup(String cryptoConfig) throws Exception { FullNode.main(new String[]{"-c", config.toString(), "-d", output.toString()}); assertEquals(1, contexts.constructed().size()); verify(contexts.constructed().get(0)).refresh(); - verify(application).startup(); - verify(application).blockUntilShutdown(); + InOrder inOrder = inOrder(application); + inOrder.verify(application).startup(); + inOrder.verify(application).blockUntilShutdown(); } } diff --git a/framework/src/test/java/org/tron/program/FullNodeTest.java b/framework/src/test/java/org/tron/program/FullNodeTest.java index faa2a99dafd..b7507b85220 100644 --- a/framework/src/test/java/org/tron/program/FullNodeTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeTest.java @@ -20,6 +20,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.InOrder; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -94,8 +95,9 @@ public void testNodeModeRetainsInitialization() { metrics.verify(Metrics::init); Assert.assertEquals(1, contexts.constructed().size()); Mockito.verify(contexts.constructed().get(0)).refresh(); - Mockito.verify(application).startup(); - Mockito.verify(application).blockUntilShutdown(); + InOrder order = Mockito.inOrder(application); + order.verify(application).startup(); + order.verify(application).blockUntilShutdown(); Assert.assertTrue(ipcClients.constructed().isEmpty()); } }