From a21b2a7c83798e0884b60c1ca02c1ecccf3ade1a Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 19 Oct 2026 15:19:52 +0800 Subject: [PATCH 01/26] feat(metrics): extend node_info and add node status gauges node_info gains address and role labels; add is_active, votes and rank gauges refreshed at startup and on maintenance blocks. --- .../tron/common/prometheus/MetricKeys.java | 5 + .../tron/common/prometheus/MetricLabels.java | 5 + .../tron/common/prometheus/MetricsGauge.java | 5 + .../tron/common/prometheus/MetricsInfo.java | 12 ++- .../org/tron/common/prometheus/SRMetrics.java | 17 ++++ docs/metrics-changelog.md | 5 +- .../main/java/org/tron/core/db/Manager.java | 5 + .../tron/core/metrics/SrMetricsService.java | 67 ++++++++++++++ .../main/java/org/tron/program/FullNode.java | 8 +- .../tron/common/prometheus/SRMetricsTest.java | 92 +++++++++++++++++++ .../prometheus/PrometheusApiServiceTest.java | 7 +- 11 files changed, 219 insertions(+), 9 deletions(-) create mode 100644 framework/src/main/java/org/tron/core/metrics/SrMetricsService.java diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index d392a47d200..55d1f301bd2 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -41,6 +41,11 @@ public static class Gauge { public static final String DB_SST_LEVEL = "tron:db_sst_level"; public static final String MANAGER_QUEUE = "tron:manager_queue_size"; public static final String TX_CACHE = "tron:tx_cache"; + // status of the SR configured locally (witness address), refreshed at startup and on + // maintenance blocks, when the vote counts and the active witness set are updated + public static final String SR_IS_ACTIVE = "tron:sr_is_active"; + public static final String SR_VOTES = "tron:sr_votes"; + public static final String SR_RANK = "tron:sr_rank"; private Gauge() { throw new IllegalStateException("Gauge"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index 7f3d5fa6076..85706a2c473 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -84,6 +84,11 @@ public static class Info { // identifies the genesis block: the label value is the chain id derived from the // genesis block hash, so the label is named genesis_block_id public static final String CHAIN_ID = "genesis_block_id"; + public static final String ADDRESS = "address"; + public static final String ROLE = "role"; + public static final String ROLE_PRIMARY = "primary"; + public static final String ROLE_BACKUP = "backup"; + public static final String ROLE_FULLNODE = "fullnode"; private Info() { throw new IllegalStateException("Info"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index dc1e5447540..73b27be5212 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -19,6 +19,11 @@ class MetricsGauge { init(MetricKeys.Gauge.DB_SIZE_BYTES, "tron db size .", "type", "db", "level"); init(MetricKeys.Gauge.DB_SST_LEVEL, "tron db files .", "type", "db", "level"); init(MetricKeys.Gauge.TX_CACHE, "tron tx cache info.", "type"); + init(MetricKeys.Gauge.SR_IS_ACTIVE, + "1 if this node's witness is in the active witness set."); + init(MetricKeys.Gauge.SR_VOTES, "Vote count of this node's witness."); + init(MetricKeys.Gauge.SR_RANK, + "Vote-count rank of this node's witness (1-based, 0 when not configured)."); } private MetricsGauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsInfo.java b/common/src/main/java/org/tron/common/prometheus/MetricsInfo.java index e3cd8b7ca65..6e2482b5e33 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsInfo.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsInfo.java @@ -1,6 +1,7 @@ package org.tron.common.prometheus; import io.prometheus.client.Info; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; @@ -10,9 +11,12 @@ class MetricsInfo { private static final Map container = new ConcurrentHashMap<>(); + private static final Map lastLabels = new ConcurrentHashMap<>(); + static { init(MetricKeys.Info.NODE_INFO, "tron node info.", - MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID); + MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID, + MetricLabels.Info.ADDRESS, MetricLabels.Info.ROLE); } private MetricsInfo() { @@ -34,6 +38,12 @@ static void set(String key, String... labels) { logger.info("{} not exist", key); return; } + String[] oldLabels = lastLabels.put(key, labels); + if (oldLabels != null && !Arrays.equals(oldLabels, labels)) { + // label values are part of the series identity: drop the previous one so an + // address or role change does not leave a stale series behind + info.remove(oldLabels); + } info.labels(labels); } } diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 0c547a38e2c..207fdcacc78 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -10,6 +10,23 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } + public static void recordNodeInfo(String version, String chainId, String address, + String role) { + if (!Metrics.enabled()) { + return; + } + Metrics.info(MetricKeys.Info.NODE_INFO, version, chainId, address, role); + } + + public static void recordSrStatus(boolean active, long votes, long rank) { + if (!Metrics.enabled()) { + return; + } + Metrics.gaugeSet(MetricKeys.Gauge.SR_IS_ACTIVE, active ? 1 : 0); + Metrics.gaugeSet(MetricKeys.Gauge.SR_VOTES, votes); + Metrics.gaugeSet(MetricKeys.Gauge.SR_RANK, rank); + } + public static void recordSrSetChange(List currentWits, List newWits) { if (!Metrics.enabled()) { return; diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 46c0402919a..b26f7305a6d 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -9,7 +9,10 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core -- `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_info` (Info, labels `version`, `genesis_block_id`, `address`, `role`) — static node identity: the running node version string, the full genesis block hash as the canonical chain identifier, the locally configured SR witness address in base58 (empty when the node is not configured as a witness), and the node role: `primary` when that witness is in the active witness set, `backup` when it is registered but not active, `fullnode` when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:sr_is_active` (Gauge) — 1 when the locally configured witness address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:sr_votes` (Gauge) — vote count of the locally configured witness, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:sr_rank` (Gauge) — 1-based rank of the locally configured witness by vote count, 0 when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_already_known` (Counter) — incremented when a block response matches an outstanding adv request whose exact block ID is already known before that response is processed (best-effort signal; concurrent arrivals may be missed). ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 0efa48dc20a..5d83e66904c 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -137,6 +137,7 @@ import org.tron.core.exception.ZksnarkException; import org.tron.core.metrics.MetricsKey; import org.tron.core.metrics.MetricsUtil; +import org.tron.core.metrics.SrMetricsService; import org.tron.core.service.MortgageService; import org.tron.core.service.RewardViCalService; import org.tron.core.services.event.exception.EventException; @@ -1900,6 +1901,10 @@ private void processBlock(BlockCapsule block, List txs) .initBlockSection(transactionRetCapsule); chainBaseManager.getSectionBloomStore().write(block.getNum()); block.setBloom(blockBloom); + + if (flag) { + SrMetricsService.refresh(chainBaseManager); + } } private void payReward(BlockCapsule block) { diff --git a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java new file mode 100644 index 00000000000..8d6ccc9b7f3 --- /dev/null +++ b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java @@ -0,0 +1,67 @@ +package org.tron.core.metrics; + +import com.google.protobuf.ByteString; +import java.util.List; +import org.tron.common.prometheus.MetricLabels; +import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.LocalWitnesses; +import org.tron.common.utils.StringUtil; +import org.tron.core.ChainBaseManager; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.store.WitnessStore; +import org.tron.program.Version; + +/** + * Publishes the status of the super representative (SR) configured locally: whether the node + * runs as a witness, whether that witness is in the active set (primary) or only registered as a + * candidate (backup), its vote count and vote-count rank, plus the local address itself. + */ +public final class SrMetricsService { + + private SrMetricsService() { + throw new IllegalStateException("SrMetricsService"); + } + + /** + * Recomputes and publishes the local SR status. Called once at startup and at maintenance + * blocks, where the active witness set and the witness vote counts are updated. + */ + public static void refresh(ChainBaseManager chainBaseManager) { + if (!Metrics.enabled()) { + return; + } + LocalWitnesses localWitnesses = Args.getLocalWitnesses(); + byte[] address = localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); + String address58 = ""; + String role = MetricLabels.Info.ROLE_FULLNODE; + boolean active = false; + long votes = 0; + long rank = 0; + if (address != null && address.length > 0) { + address58 = StringUtil.encode58Check(address); + ByteString witnessAddress = ByteString.copyFrom(address); + active = chainBaseManager.getWitnessScheduleStore().getActiveWitnesses() + .contains(witnessAddress); + WitnessCapsule witness = chainBaseManager.getWitnessStore().get(address); + votes = witness == null ? 0 : witness.getVoteCount(); + rank = rankOf(chainBaseManager.getWitnessStore(), witnessAddress); + role = active ? MetricLabels.Info.ROLE_PRIMARY : MetricLabels.Info.ROLE_BACKUP; + } + SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), + address58, role); + SRMetrics.recordSrStatus(active, votes, rank); + } + + private static long rankOf(WitnessStore witnessStore, ByteString address) { + List witnesses = witnessStore.getAllWitnesses(); + WitnessStore.sortWitnesses(witnesses, false); + for (int i = 0; i < witnesses.size(); i++) { + if (witnesses.get(i).getAddress().equals(address)) { + return i + 1L; + } + } + return 0L; + } +} diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 94d49fc43fd..f8af0995e3d 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -12,13 +12,14 @@ import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; -import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.Metrics; +import org.tron.core.ChainBaseManager; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.config.args.CLIParameter; import org.tron.core.config.args.CommandLineArguments; import org.tron.core.exception.TronError; +import org.tron.core.metrics.SrMetricsService; import org.tron.core.services.admin.ipc.client.IpcClient; public class FullNode { @@ -72,9 +73,8 @@ public static void main(String[] args) { context.registerShutdownHook(); appT.startup(); // the genesis block id (chainId) is only available after the context refresh - // (Manager.initGenesis) - Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), - Args.getInstance().getChainId()); + // (Manager.initGenesis); this also publishes the local SR identity and status + SrMetricsService.refresh(context.getBean(ChainBaseManager.class)); if (parameter.isSolidityNode()) { SolidityNode node = context.getBean(SolidityNode.class); node.run(); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 4c1404bb232..ebd0bdd638b 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -13,6 +13,7 @@ import org.junit.Test; import org.tron.common.BaseTest; import org.tron.common.TestConstants; +import org.tron.common.utils.LocalWitnesses; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.StringUtil; import org.tron.consensus.dpos.MaintenanceManager; @@ -21,6 +22,8 @@ import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.args.Args; import org.tron.core.consensus.ConsensusService; +import org.tron.core.metrics.SrMetricsService; +import org.tron.program.Version; import org.tron.protos.Protocol; import org.tron.protos.Protocol.Vote; @@ -52,6 +55,7 @@ public void setUp() { @After public void tearDown() { Args.getInstance().setMetricsPrometheusEnable(true); + Args.setLocalWitnesses(new LocalWitnesses()); } /** @@ -169,6 +173,94 @@ public void testSrRemoveDirect() { Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, stableWit)); } + /** + * No witness configured: role is fullnode, gauges are zero, address label is empty. + */ + @Test + public void testSrStatusFullNode() { + Args.setLocalWitnesses(new LocalWitnesses()); + + SrMetricsService.refresh(chainBaseManager); + + Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); + Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); + Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); + Assert.assertEquals(1.0, info("", MetricLabels.Info.ROLE_FULLNODE), 0.0); + } + + /** + * Witness in the active set: role is primary, gauges expose its votes and 1-based rank, and + * the previously published fullnode series is dropped when the label values change. + */ + @Test + public void testSrStatusPrimary() { + Args.setLocalWitnesses(new LocalWitnesses()); + SrMetricsService.refresh(chainBaseManager); + Assert.assertEquals(1.0, info("", MetricLabels.Info.ROLE_FULLNODE), 0.0); + + ByteString witness = registerWitnessWithVotes(1000); + registerWitnessWithVotes(50); + chainBaseManager.getWitnessScheduleStore() + .saveActiveWitnesses(Collections.singletonList(witness)); + + LocalWitnesses localWitnesses = new LocalWitnesses(); + localWitnesses.initWitnessAccountAddress(witness.toByteArray()); + Args.setLocalWitnesses(localWitnesses); + + SrMetricsService.refresh(chainBaseManager); + + String address = StringUtil.encode58Check(witness.toByteArray()); + Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); + Assert.assertEquals(1000.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); + Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); + Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_PRIMARY), 0.0); + Assert.assertNull(info("", MetricLabels.Info.ROLE_FULLNODE)); + } + + /** + * Witness registered but not in the active set: role is backup and is_active stays 0. + */ + @Test + public void testSrStatusBackup() { + ByteString activeWitness = registerWitnessWithVotes(10); + ByteString witness = registerWitnessWithVotes(7); + chainBaseManager.getWitnessScheduleStore() + .saveActiveWitnesses(Collections.singletonList(activeWitness)); + + LocalWitnesses localWitnesses = new LocalWitnesses(); + localWitnesses.initWitnessAccountAddress(witness.toByteArray()); + Args.setLocalWitnesses(localWitnesses); + + SrMetricsService.refresh(chainBaseManager); + + String address = StringUtil.encode58Check(witness.toByteArray()); + Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); + Assert.assertEquals(7.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); + Assert.assertTrue(gauge(MetricKeys.Gauge.SR_RANK) > 0); + Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_BACKUP), 0.0); + } + + private ByteString registerWitnessWithVotes(long votes) { + ByteString address = uniqueAddress(); + chainBaseManager.getWitnessStore().put(address.toByteArray(), + new WitnessCapsule(address, votes, "")); + chainBaseManager.getAccountStore().put(address.toByteArray(), + new AccountCapsule(Protocol.Account.newBuilder().setAddress(address).build())); + return address; + } + + private Double gauge(String name) { + return CollectorRegistry.defaultRegistry.getSampleValue(name); + } + + private Double info(String address, String role) { + return CollectorRegistry.defaultRegistry.getSampleValue( + "tron:node_info", + new String[] {MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID, + MetricLabels.Info.ADDRESS, MetricLabels.Info.ROLE}, + new String[] {Version.getVersion(), Args.getInstance().getChainId(), address, role}); + } + private ByteString registerWitness() { ByteString address = uniqueAddress(); chainBaseManager.getWitnessStore().put(address.toByteArray(), new WitnessCapsule(address)); diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index 77e9a95ad84..9d78dd1db77 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -214,12 +214,13 @@ public void testNodeInfoMetric() { String version = Version.getVersion(); String testGenesisBlockId = "00000000000000001ebf88508a03865c71d452e25f4d51194196a1d22b6653dc"; - Metrics.info(MetricKeys.Info.NODE_INFO, version, testGenesisBlockId); + Metrics.info(MetricKeys.Info.NODE_INFO, version, testGenesisBlockId, "", "fullnode"); // Prometheus Info collector appends "_info" to the sample name Double value = CollectorRegistry.defaultRegistry.getSampleValue( "tron:node_info", - new String[] {MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID}, - new String[] {version, testGenesisBlockId}); + new String[] {MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID, + MetricLabels.Info.ADDRESS, MetricLabels.Info.ROLE}, + new String[] {version, testGenesisBlockId, "", "fullnode"}); Assert.assertNotNull("tron:node_info sample should exist", value); Assert.assertEquals(1.0, value, 0.0); } From a9cc0abaf49fd44b63d992ecf1a793830e9ca451 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 19 Oct 2026 15:29:24 +0800 Subject: [PATCH 02/26] fix(metrics): use master/slaver for node role labels --- .../tron/common/prometheus/MetricLabels.java | 4 +-- docs/metrics-changelog.md | 2 +- .../main/java/org/tron/core/db/Manager.java | 5 +++- .../tron/core/metrics/SrMetricsService.java | 11 +++++--- .../main/java/org/tron/program/FullNode.java | 4 ++- .../tron/common/prometheus/SRMetricsTest.java | 26 ++++++++++++------- 6 files changed, 33 insertions(+), 19 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index 85706a2c473..df7afe7cba0 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -86,8 +86,8 @@ public static class Info { public static final String CHAIN_ID = "genesis_block_id"; public static final String ADDRESS = "address"; public static final String ROLE = "role"; - public static final String ROLE_PRIMARY = "primary"; - public static final String ROLE_BACKUP = "backup"; + public static final String ROLE_MASTER = "master"; + public static final String ROLE_SLAVER = "slaver"; public static final String ROLE_FULLNODE = "fullnode"; private Info() { diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index b26f7305a6d..05278e8b7cd 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -9,7 +9,7 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core -- `tron:node_info` (Info, labels `version`, `genesis_block_id`, `address`, `role`) — static node identity: the running node version string, the full genesis block hash as the canonical chain identifier, the locally configured SR witness address in base58 (empty when the node is not configured as a witness), and the node role: `primary` when that witness is in the active witness set, `backup` when it is registered but not active, `fullnode` when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_info` (Info, labels `version`, `genesis_block_id`, `address`, `role`) — static node identity: the running node version string, the full genesis block hash as the canonical chain identifier, the locally configured SR witness address in base58 (empty when the node is not configured as a witness), and the node role: `master` when the backup election has this node producing blocks, `slaver` when it stands by for a master, `fullnode` when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_is_active` (Gauge) — 1 when the locally configured witness address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_votes` (Gauge) — vote count of the locally configured witness, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_rank` (Gauge) — 1-based rank of the locally configured witness by vote count, 0 when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 5d83e66904c..0db506bd40d 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -50,6 +50,7 @@ import org.springframework.stereotype.Component; import org.tron.api.GrpcAPI.TransactionInfoList; import org.tron.common.args.GenesisBlock; +import org.tron.common.backup.BackupManager; import org.tron.common.bloom.Bloom; import org.tron.common.cron.CronExpression; import org.tron.common.es.ExecutorServiceManager; @@ -238,6 +239,8 @@ public class Manager { @Autowired @Getter private ChainBaseManager chainBaseManager; + @Autowired + private BackupManager backupManager; // transactions cache private BlockingQueue pendingTransactions; @Getter @@ -1903,7 +1906,7 @@ private void processBlock(BlockCapsule block, List txs) block.setBloom(blockBloom); if (flag) { - SrMetricsService.refresh(chainBaseManager); + SrMetricsService.refresh(chainBaseManager, backupManager); } } diff --git a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java index 8d6ccc9b7f3..f94784c3102 100644 --- a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java +++ b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java @@ -2,6 +2,7 @@ import com.google.protobuf.ByteString; import java.util.List; +import org.tron.common.backup.BackupManager; import org.tron.common.prometheus.MetricLabels; import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; @@ -15,8 +16,9 @@ /** * Publishes the status of the super representative (SR) configured locally: whether the node - * runs as a witness, whether that witness is in the active set (primary) or only registered as a - * candidate (backup), its vote count and vote-count rank, plus the local address itself. + * runs as a witness, its role in the backup election ({@code master} when it currently produces + * blocks, {@code slaver} when it stands by for a master), whether that witness is in the active + * witness set, its vote count and vote-count rank, plus the local address itself. */ public final class SrMetricsService { @@ -28,7 +30,7 @@ private SrMetricsService() { * Recomputes and publishes the local SR status. Called once at startup and at maintenance * blocks, where the active witness set and the witness vote counts are updated. */ - public static void refresh(ChainBaseManager chainBaseManager) { + public static void refresh(ChainBaseManager chainBaseManager, BackupManager backupManager) { if (!Metrics.enabled()) { return; } @@ -47,7 +49,8 @@ public static void refresh(ChainBaseManager chainBaseManager) { WitnessCapsule witness = chainBaseManager.getWitnessStore().get(address); votes = witness == null ? 0 : witness.getVoteCount(); rank = rankOf(chainBaseManager.getWitnessStore(), witnessAddress); - role = active ? MetricLabels.Info.ROLE_PRIMARY : MetricLabels.Info.ROLE_BACKUP; + role = backupManager.getStatus() == BackupManager.BackupStatusEnum.MASTER + ? MetricLabels.Info.ROLE_MASTER : MetricLabels.Info.ROLE_SLAVER; } SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), address58, role); diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index f8af0995e3d..a0e55524aa9 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -9,6 +9,7 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.arch.Arch; +import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; @@ -74,7 +75,8 @@ public static void main(String[] args) { appT.startup(); // the genesis block id (chainId) is only available after the context refresh // (Manager.initGenesis); this also publishes the local SR identity and status - SrMetricsService.refresh(context.getBean(ChainBaseManager.class)); + SrMetricsService.refresh(context.getBean(ChainBaseManager.class), + context.getBean(BackupManager.class)); if (parameter.isSolidityNode()) { SolidityNode node = context.getBean(SolidityNode.class); node.run(); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index ebd0bdd638b..a1ba25cb4d8 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -13,6 +13,7 @@ import org.junit.Test; import org.tron.common.BaseTest; import org.tron.common.TestConstants; +import org.tron.common.backup.BackupManager; import org.tron.common.utils.LocalWitnesses; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.StringUtil; @@ -37,6 +38,8 @@ public class SRMetricsTest extends BaseTest { private MaintenanceManager maintenanceManager; @Resource private ConsensusService consensusService; + @Resource + private BackupManager backupManager; static { Args.setParam(new String[]{"-d", dbPath()}, TestConstants.TEST_CONF); @@ -56,6 +59,7 @@ public void setUp() { public void tearDown() { Args.getInstance().setMetricsPrometheusEnable(true); Args.setLocalWitnesses(new LocalWitnesses()); + backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); } /** @@ -180,7 +184,7 @@ public void testSrRemoveDirect() { public void testSrStatusFullNode() { Args.setLocalWitnesses(new LocalWitnesses()); - SrMetricsService.refresh(chainBaseManager); + SrMetricsService.refresh(chainBaseManager, backupManager); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); @@ -189,13 +193,14 @@ public void testSrStatusFullNode() { } /** - * Witness in the active set: role is primary, gauges expose its votes and 1-based rank, and + * Witness in the active set: role is master, gauges expose its votes and 1-based rank, and * the previously published fullnode series is dropped when the label values change. */ @Test - public void testSrStatusPrimary() { + public void testSrStatusMaster() { + backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); Args.setLocalWitnesses(new LocalWitnesses()); - SrMetricsService.refresh(chainBaseManager); + SrMetricsService.refresh(chainBaseManager, backupManager); Assert.assertEquals(1.0, info("", MetricLabels.Info.ROLE_FULLNODE), 0.0); ByteString witness = registerWitnessWithVotes(1000); @@ -207,21 +212,22 @@ public void testSrStatusPrimary() { localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); - SrMetricsService.refresh(chainBaseManager); + SrMetricsService.refresh(chainBaseManager, backupManager); String address = StringUtil.encode58Check(witness.toByteArray()); Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(1000.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); - Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_PRIMARY), 0.0); + Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_MASTER), 0.0); Assert.assertNull(info("", MetricLabels.Info.ROLE_FULLNODE)); } /** - * Witness registered but not in the active set: role is backup and is_active stays 0. + * Backup election slaver: role is slaver and is_active reflects the active witness set. */ @Test - public void testSrStatusBackup() { + public void testSrStatusSlaver() { + backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); ByteString activeWitness = registerWitnessWithVotes(10); ByteString witness = registerWitnessWithVotes(7); chainBaseManager.getWitnessScheduleStore() @@ -231,13 +237,13 @@ public void testSrStatusBackup() { localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); - SrMetricsService.refresh(chainBaseManager); + SrMetricsService.refresh(chainBaseManager, backupManager); String address = StringUtil.encode58Check(witness.toByteArray()); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(7.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); Assert.assertTrue(gauge(MetricKeys.Gauge.SR_RANK) > 0); - Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_BACKUP), 0.0); + Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_SLAVER), 0.0); } private ByteString registerWitnessWithVotes(long votes) { From 9bd4f6bb53069138a5e2c2a28361baddee2bab48 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 19 Oct 2026 15:44:41 +0800 Subject: [PATCH 03/26] fix(metrics): refresh node role on backup status change --- .../org/tron/common/backup/BackupManager.java | 2 ++ .../tron/core/metrics/SrMetricsService.java | 22 +++++++++++++++++++ .../tron/common/prometheus/SRMetricsTest.java | 21 ++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 497f38544bc..b23799f4105 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -25,6 +25,7 @@ import org.tron.common.backup.socket.UdpEvent; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; +import org.tron.core.metrics.SrMetricsService; import org.tron.p2p.utils.NetUtil; @Slf4j(topic = "backup") @@ -67,6 +68,7 @@ public class BackupManager implements EventHandler { public void setStatus(BackupStatusEnum status) { logger.info("Change backup status to {}", status); this.status = status; + SrMetricsService.onRoleChanged(status); } public void init() { diff --git a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java index f94784c3102..45d4b54aacf 100644 --- a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java +++ b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java @@ -57,6 +57,28 @@ public static void refresh(ChainBaseManager chainBaseManager, BackupManager back SRMetrics.recordSrStatus(active, votes, rank); } + /** + * Publishes the {@code node_info} role immediately after a backup-election state change. + * Reads no store: votes, rank and active-set membership only change at maintenance blocks + * and are refreshed there. + */ + public static void onRoleChanged(BackupManager.BackupStatusEnum status) { + if (!Metrics.enabled()) { + return; + } + LocalWitnesses localWitnesses = Args.getLocalWitnesses(); + byte[] address = localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); + String address58 = ""; + String role = MetricLabels.Info.ROLE_FULLNODE; + if (address != null && address.length > 0) { + address58 = StringUtil.encode58Check(address); + role = status == BackupManager.BackupStatusEnum.MASTER + ? MetricLabels.Info.ROLE_MASTER : MetricLabels.Info.ROLE_SLAVER; + } + SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), + address58, role); + } + private static long rankOf(WitnessStore witnessStore, ByteString address) { List witnesses = witnessStore.getAllWitnesses(); WitnessStore.sortWitnesses(witnesses, false); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index a1ba25cb4d8..3428a618232 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -246,6 +246,27 @@ public void testSrStatusSlaver() { Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_SLAVER), 0.0); } + /** + * setStatus() itself republishes the node_info role without touching any store, so a + * backup-election transition is reflected immediately instead of at the next maintenance. + */ + @Test + public void testRoleRefreshOnBackupStatusChange() { + ByteString witness = registerWitnessWithVotes(9); + LocalWitnesses localWitnesses = new LocalWitnesses(); + localWitnesses.initWitnessAccountAddress(witness.toByteArray()); + Args.setLocalWitnesses(localWitnesses); + String address = StringUtil.encode58Check(witness.toByteArray()); + + backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_SLAVER), 0.0); + Assert.assertNull(info(address, MetricLabels.Info.ROLE_MASTER)); + + backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); + Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_MASTER), 0.0); + Assert.assertNull(info(address, MetricLabels.Info.ROLE_SLAVER)); + } + private ByteString registerWitnessWithVotes(long votes) { ByteString address = uniqueAddress(); chainBaseManager.getWitnessStore().put(address.toByteArray(), From 53f4e39cee8ca6257184543156786f5af7a0f6a1 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 19 Oct 2026 16:46:16 +0800 Subject: [PATCH 04/26] refactor(metrics): tidy naming in node status recording --- .../main/java/org/tron/core/db/Manager.java | 2 +- .../tron/core/metrics/SrMetricsService.java | 47 ++++++++++--------- .../main/java/org/tron/program/FullNode.java | 2 +- .../tron/common/prometheus/SRMetricsTest.java | 8 ++-- 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 0db506bd40d..83c23fbd301 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -1906,7 +1906,7 @@ private void processBlock(BlockCapsule block, List txs) block.setBloom(blockBloom); if (flag) { - SrMetricsService.refresh(chainBaseManager, backupManager); + SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); } } diff --git a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java index 45d4b54aacf..d4bd0bde56a 100644 --- a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java +++ b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java @@ -27,33 +27,34 @@ private SrMetricsService() { } /** - * Recomputes and publishes the local SR status. Called once at startup and at maintenance - * blocks, where the active witness set and the witness vote counts are updated. + * Records the local node status (witness identity, backup role, active-set membership, + * votes, rank) into the metrics. Called once at startup and at maintenance blocks, + * where the active witness set and the witness vote counts are updated. */ - public static void refresh(ChainBaseManager chainBaseManager, BackupManager backupManager) { + public static void recordNodeStatus(ChainBaseManager chainBaseManager, + BackupManager backupManager) { if (!Metrics.enabled()) { return; } - LocalWitnesses localWitnesses = Args.getLocalWitnesses(); - byte[] address = localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); - String address58 = ""; + byte[] witnessAddress = localWitnessAddress(); + String witnessAddress58 = ""; String role = MetricLabels.Info.ROLE_FULLNODE; boolean active = false; long votes = 0; long rank = 0; - if (address != null && address.length > 0) { - address58 = StringUtil.encode58Check(address); - ByteString witnessAddress = ByteString.copyFrom(address); + if (witnessAddress != null && witnessAddress.length > 0) { + witnessAddress58 = StringUtil.encode58Check(witnessAddress); + ByteString witnessAddressBs = ByteString.copyFrom(witnessAddress); active = chainBaseManager.getWitnessScheduleStore().getActiveWitnesses() - .contains(witnessAddress); - WitnessCapsule witness = chainBaseManager.getWitnessStore().get(address); + .contains(witnessAddressBs); + WitnessCapsule witness = chainBaseManager.getWitnessStore().get(witnessAddress); votes = witness == null ? 0 : witness.getVoteCount(); - rank = rankOf(chainBaseManager.getWitnessStore(), witnessAddress); + rank = witnessRank(chainBaseManager.getWitnessStore(), witnessAddressBs); role = backupManager.getStatus() == BackupManager.BackupStatusEnum.MASTER ? MetricLabels.Info.ROLE_MASTER : MetricLabels.Info.ROLE_SLAVER; } SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), - address58, role); + witnessAddress58, role); SRMetrics.recordSrStatus(active, votes, rank); } @@ -66,24 +67,28 @@ public static void onRoleChanged(BackupManager.BackupStatusEnum status) { if (!Metrics.enabled()) { return; } - LocalWitnesses localWitnesses = Args.getLocalWitnesses(); - byte[] address = localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); - String address58 = ""; + byte[] witnessAddress = localWitnessAddress(); + String witnessAddress58 = ""; String role = MetricLabels.Info.ROLE_FULLNODE; - if (address != null && address.length > 0) { - address58 = StringUtil.encode58Check(address); + if (witnessAddress != null && witnessAddress.length > 0) { + witnessAddress58 = StringUtil.encode58Check(witnessAddress); role = status == BackupManager.BackupStatusEnum.MASTER ? MetricLabels.Info.ROLE_MASTER : MetricLabels.Info.ROLE_SLAVER; } SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), - address58, role); + witnessAddress58, role); + } + + private static byte[] localWitnessAddress() { + LocalWitnesses localWitnesses = Args.getLocalWitnesses(); + return localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); } - private static long rankOf(WitnessStore witnessStore, ByteString address) { + private static long witnessRank(WitnessStore witnessStore, ByteString witnessAddress) { List witnesses = witnessStore.getAllWitnesses(); WitnessStore.sortWitnesses(witnesses, false); for (int i = 0; i < witnesses.size(); i++) { - if (witnesses.get(i).getAddress().equals(address)) { + if (witnesses.get(i).getAddress().equals(witnessAddress)) { return i + 1L; } } diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index a0e55524aa9..b9771fcf67d 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -75,7 +75,7 @@ public static void main(String[] args) { appT.startup(); // the genesis block id (chainId) is only available after the context refresh // (Manager.initGenesis); this also publishes the local SR identity and status - SrMetricsService.refresh(context.getBean(ChainBaseManager.class), + SrMetricsService.recordNodeStatus(context.getBean(ChainBaseManager.class), context.getBean(BackupManager.class)); if (parameter.isSolidityNode()) { SolidityNode node = context.getBean(SolidityNode.class); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 3428a618232..21fe303f82e 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -184,7 +184,7 @@ public void testSrRemoveDirect() { public void testSrStatusFullNode() { Args.setLocalWitnesses(new LocalWitnesses()); - SrMetricsService.refresh(chainBaseManager, backupManager); + SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); @@ -200,7 +200,7 @@ public void testSrStatusFullNode() { public void testSrStatusMaster() { backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); Args.setLocalWitnesses(new LocalWitnesses()); - SrMetricsService.refresh(chainBaseManager, backupManager); + SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); Assert.assertEquals(1.0, info("", MetricLabels.Info.ROLE_FULLNODE), 0.0); ByteString witness = registerWitnessWithVotes(1000); @@ -212,7 +212,7 @@ public void testSrStatusMaster() { localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); - SrMetricsService.refresh(chainBaseManager, backupManager); + SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); String address = StringUtil.encode58Check(witness.toByteArray()); Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); @@ -237,7 +237,7 @@ public void testSrStatusSlaver() { localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); - SrMetricsService.refresh(chainBaseManager, backupManager); + SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); String address = StringUtil.encode58Check(witness.toByteArray()); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); From cd2ec205d7dc6997ba69a08fd9aa7192f926ddf2 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 19 Oct 2026 16:56:58 +0800 Subject: [PATCH 05/26] refactor(metrics): refresh only witness gauges at maintenance blocks --- .../main/java/org/tron/core/db/Manager.java | 5 +-- .../tron/core/metrics/SrMetricsService.java | 38 ++++++++++++------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 83c23fbd301..f2897810d36 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -50,7 +50,6 @@ import org.springframework.stereotype.Component; import org.tron.api.GrpcAPI.TransactionInfoList; import org.tron.common.args.GenesisBlock; -import org.tron.common.backup.BackupManager; import org.tron.common.bloom.Bloom; import org.tron.common.cron.CronExpression; import org.tron.common.es.ExecutorServiceManager; @@ -239,8 +238,6 @@ public class Manager { @Autowired @Getter private ChainBaseManager chainBaseManager; - @Autowired - private BackupManager backupManager; // transactions cache private BlockingQueue pendingTransactions; @Getter @@ -1906,7 +1903,7 @@ private void processBlock(BlockCapsule block, List txs) block.setBloom(blockBloom); if (flag) { - SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); + SrMetricsService.recordWitnessStatus(chainBaseManager); } } diff --git a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java index d4bd0bde56a..ea3c510ba30 100644 --- a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java +++ b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java @@ -27,46 +27,58 @@ private SrMetricsService() { } /** - * Records the local node status (witness identity, backup role, active-set membership, - * votes, rank) into the metrics. Called once at startup and at maintenance blocks, - * where the active witness set and the witness vote counts are updated. + * Records the full node status. Called once at startup; afterwards each part refreshes + * at its own point of change: the role on backup-status switches + * ({@link #onRoleChanged}), the witness gauges at maintenance blocks + * ({@link #recordWitnessStatus}). */ public static void recordNodeStatus(ChainBaseManager chainBaseManager, BackupManager backupManager) { if (!Metrics.enabled()) { return; } + recordNodeInfoLabels(backupManager.getStatus()); + recordWitnessStatus(chainBaseManager); + } + + /** + * Records the witness gauges (active-set membership, settled votes, settled rank). + * Called at maintenance blocks, where the active witness set and the settled vote + * counts are updated. Reads no backup state: the role only changes on backup-status + * switches and is refreshed by {@link #onRoleChanged}. + */ + public static void recordWitnessStatus(ChainBaseManager chainBaseManager) { + if (!Metrics.enabled()) { + return; + } byte[] witnessAddress = localWitnessAddress(); - String witnessAddress58 = ""; - String role = MetricLabels.Info.ROLE_FULLNODE; boolean active = false; long votes = 0; long rank = 0; if (witnessAddress != null && witnessAddress.length > 0) { - witnessAddress58 = StringUtil.encode58Check(witnessAddress); ByteString witnessAddressBs = ByteString.copyFrom(witnessAddress); active = chainBaseManager.getWitnessScheduleStore().getActiveWitnesses() .contains(witnessAddressBs); WitnessCapsule witness = chainBaseManager.getWitnessStore().get(witnessAddress); votes = witness == null ? 0 : witness.getVoteCount(); rank = witnessRank(chainBaseManager.getWitnessStore(), witnessAddressBs); - role = backupManager.getStatus() == BackupManager.BackupStatusEnum.MASTER - ? MetricLabels.Info.ROLE_MASTER : MetricLabels.Info.ROLE_SLAVER; } - SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), - witnessAddress58, role); SRMetrics.recordSrStatus(active, votes, rank); } /** - * Publishes the {@code node_info} role immediately after a backup-election state change. - * Reads no store: votes, rank and active-set membership only change at maintenance blocks - * and are refreshed there. + * Publishes the {@code node_info} labels immediately after a backup-election state + * change. Reads no store: votes, rank and active-set membership only change at + * maintenance blocks and are refreshed there. */ public static void onRoleChanged(BackupManager.BackupStatusEnum status) { if (!Metrics.enabled()) { return; } + recordNodeInfoLabels(status); + } + + private static void recordNodeInfoLabels(BackupManager.BackupStatusEnum status) { byte[] witnessAddress = localWitnessAddress(); String witnessAddress58 = ""; String role = MetricLabels.Info.ROLE_FULLNODE; From c526e6897f3dc87d44584daadf76c4df53782a60 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 19 Oct 2026 17:15:02 +0800 Subject: [PATCH 06/26] refactor(metrics): record witness gauges in the maintenance flow --- .../org/tron/common/prometheus/SRMetrics.java | 14 +++ .../consensus/dpos/MaintenanceManager.java | 43 +++++++ .../org/tron/common/backup/BackupManager.java | 37 +++++- .../main/java/org/tron/core/db/Manager.java | 5 - .../tron/core/metrics/SrMetricsService.java | 109 ------------------ .../main/java/org/tron/program/FullNode.java | 10 +- .../tron/common/prometheus/SRMetricsTest.java | 12 +- 7 files changed, 104 insertions(+), 126 deletions(-) delete mode 100644 framework/src/main/java/org/tron/core/metrics/SrMetricsService.java diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 207fdcacc78..16ebaeef380 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -10,6 +10,20 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } + private static volatile byte[] localWitnessAddress; + + /** + * Caches the local witness address once it is resolved at startup, so maintenance-time + * recording can read it without reaching back into node configuration. + */ + public static void setLocalWitnessAddress(byte[] address) { + localWitnessAddress = address; + } + + public static byte[] getLocalWitnessAddress() { + return localWitnessAddress; + } + public static void recordNodeInfo(String version, String chainId, String address, String role) { if (!Metrics.enabled()) { diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index fd5e4364d0d..80d2461a70b 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -16,6 +16,7 @@ import org.bouncycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; import org.tron.consensus.ConsensusDelegate; import org.tron.consensus.pbft.PbftManager; @@ -26,6 +27,7 @@ import org.tron.core.store.DelegationStore; import org.tron.core.store.DynamicPropertiesStore; import org.tron.core.store.VotesStore; +import org.tron.core.store.WitnessStore; @Slf4j(topic = "consensus") @Component @@ -160,6 +162,47 @@ public void doMaintenance() { delegationStore.setWitnessVote(nextCycle, witness.createDbKey(), witness.getVoteCount()); }); } + + recordWitnessMetrics(); + } + + /** + * Records the witness gauges (active-set membership, settled votes, settled rank) for + * the locally configured witness. Runs at the end of every maintenance and once at + * node startup. Never throws: a metrics failure must not affect maintenance or block + * processing. + */ + public void recordWitnessMetrics() { + if (!Metrics.enabled()) { + return; + } + try { + byte[] witnessAddress = SRMetrics.getLocalWitnessAddress(); + boolean active = false; + long votes = 0; + long rank = 0; + if (witnessAddress != null && witnessAddress.length > 0) { + ByteString witnessAddressBs = ByteString.copyFrom(witnessAddress); + active = consensusDelegate.getActiveWitnesses().contains(witnessAddressBs); + WitnessCapsule witness = consensusDelegate.getWitness(witnessAddress); + votes = witness == null ? 0 : witness.getVoteCount(); + rank = witnessRank(witnessAddressBs); + } + SRMetrics.recordSrStatus(active, votes, rank); + } catch (Exception e) { + logger.warn("Failed to record witness metrics", e); + } + } + + private long witnessRank(ByteString witnessAddress) { + List witnesses = consensusDelegate.getAllWitnesses(); + WitnessStore.sortWitnesses(witnesses, false); + for (int i = 0; i < witnesses.size(); i++) { + if (witnesses.get(i).getAddress().equals(witnessAddress)) { + return i + 1L; + } + } + return 0L; } private Map countVote(VotesStore votesStore) { diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index b23799f4105..1d086dbf5eb 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -25,8 +25,14 @@ import org.tron.common.backup.socket.UdpEvent; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; -import org.tron.core.metrics.SrMetricsService; +import org.tron.common.prometheus.MetricLabels; +import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.LocalWitnesses; +import org.tron.common.utils.StringUtil; +import org.tron.core.config.args.Args; import org.tron.p2p.utils.NetUtil; +import org.tron.program.Version; @Slf4j(topic = "backup") @Component @@ -68,7 +74,34 @@ public class BackupManager implements EventHandler { public void setStatus(BackupStatusEnum status) { logger.info("Change backup status to {}", status); this.status = status; - SrMetricsService.onRoleChanged(status); + recordNodeInfo(); + } + + /** + * Publishes the {@code node_info} labels (version, chain id, local witness address, + * role derived from the current backup status) and caches the local witness address + * for maintenance-time gauge recording. Called once at node startup and on every + * backup-status change. + */ + public void recordNodeInfo() { + if (!Metrics.enabled()) { + return; + } + byte[] witnessAddress = localWitnessAddress(); + SRMetrics.setLocalWitnessAddress(witnessAddress); + String witnessAddress58 = ""; + String role = MetricLabels.Info.ROLE_FULLNODE; + if (witnessAddress != null && witnessAddress.length > 0) { + witnessAddress58 = StringUtil.encode58Check(witnessAddress); + role = status == MASTER ? MetricLabels.Info.ROLE_MASTER : MetricLabels.Info.ROLE_SLAVER; + } + SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), + witnessAddress58, role); + } + + private byte[] localWitnessAddress() { + LocalWitnesses localWitnesses = Args.getLocalWitnesses(); + return localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); } public void init() { diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index f2897810d36..0efa48dc20a 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -137,7 +137,6 @@ import org.tron.core.exception.ZksnarkException; import org.tron.core.metrics.MetricsKey; import org.tron.core.metrics.MetricsUtil; -import org.tron.core.metrics.SrMetricsService; import org.tron.core.service.MortgageService; import org.tron.core.service.RewardViCalService; import org.tron.core.services.event.exception.EventException; @@ -1901,10 +1900,6 @@ private void processBlock(BlockCapsule block, List txs) .initBlockSection(transactionRetCapsule); chainBaseManager.getSectionBloomStore().write(block.getNum()); block.setBloom(blockBloom); - - if (flag) { - SrMetricsService.recordWitnessStatus(chainBaseManager); - } } private void payReward(BlockCapsule block) { diff --git a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java b/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java deleted file mode 100644 index ea3c510ba30..00000000000 --- a/framework/src/main/java/org/tron/core/metrics/SrMetricsService.java +++ /dev/null @@ -1,109 +0,0 @@ -package org.tron.core.metrics; - -import com.google.protobuf.ByteString; -import java.util.List; -import org.tron.common.backup.BackupManager; -import org.tron.common.prometheus.MetricLabels; -import org.tron.common.prometheus.Metrics; -import org.tron.common.prometheus.SRMetrics; -import org.tron.common.utils.LocalWitnesses; -import org.tron.common.utils.StringUtil; -import org.tron.core.ChainBaseManager; -import org.tron.core.capsule.WitnessCapsule; -import org.tron.core.config.args.Args; -import org.tron.core.store.WitnessStore; -import org.tron.program.Version; - -/** - * Publishes the status of the super representative (SR) configured locally: whether the node - * runs as a witness, its role in the backup election ({@code master} when it currently produces - * blocks, {@code slaver} when it stands by for a master), whether that witness is in the active - * witness set, its vote count and vote-count rank, plus the local address itself. - */ -public final class SrMetricsService { - - private SrMetricsService() { - throw new IllegalStateException("SrMetricsService"); - } - - /** - * Records the full node status. Called once at startup; afterwards each part refreshes - * at its own point of change: the role on backup-status switches - * ({@link #onRoleChanged}), the witness gauges at maintenance blocks - * ({@link #recordWitnessStatus}). - */ - public static void recordNodeStatus(ChainBaseManager chainBaseManager, - BackupManager backupManager) { - if (!Metrics.enabled()) { - return; - } - recordNodeInfoLabels(backupManager.getStatus()); - recordWitnessStatus(chainBaseManager); - } - - /** - * Records the witness gauges (active-set membership, settled votes, settled rank). - * Called at maintenance blocks, where the active witness set and the settled vote - * counts are updated. Reads no backup state: the role only changes on backup-status - * switches and is refreshed by {@link #onRoleChanged}. - */ - public static void recordWitnessStatus(ChainBaseManager chainBaseManager) { - if (!Metrics.enabled()) { - return; - } - byte[] witnessAddress = localWitnessAddress(); - boolean active = false; - long votes = 0; - long rank = 0; - if (witnessAddress != null && witnessAddress.length > 0) { - ByteString witnessAddressBs = ByteString.copyFrom(witnessAddress); - active = chainBaseManager.getWitnessScheduleStore().getActiveWitnesses() - .contains(witnessAddressBs); - WitnessCapsule witness = chainBaseManager.getWitnessStore().get(witnessAddress); - votes = witness == null ? 0 : witness.getVoteCount(); - rank = witnessRank(chainBaseManager.getWitnessStore(), witnessAddressBs); - } - SRMetrics.recordSrStatus(active, votes, rank); - } - - /** - * Publishes the {@code node_info} labels immediately after a backup-election state - * change. Reads no store: votes, rank and active-set membership only change at - * maintenance blocks and are refreshed there. - */ - public static void onRoleChanged(BackupManager.BackupStatusEnum status) { - if (!Metrics.enabled()) { - return; - } - recordNodeInfoLabels(status); - } - - private static void recordNodeInfoLabels(BackupManager.BackupStatusEnum status) { - byte[] witnessAddress = localWitnessAddress(); - String witnessAddress58 = ""; - String role = MetricLabels.Info.ROLE_FULLNODE; - if (witnessAddress != null && witnessAddress.length > 0) { - witnessAddress58 = StringUtil.encode58Check(witnessAddress); - role = status == BackupManager.BackupStatusEnum.MASTER - ? MetricLabels.Info.ROLE_MASTER : MetricLabels.Info.ROLE_SLAVER; - } - SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), - witnessAddress58, role); - } - - private static byte[] localWitnessAddress() { - LocalWitnesses localWitnesses = Args.getLocalWitnesses(); - return localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); - } - - private static long witnessRank(WitnessStore witnessStore, ByteString witnessAddress) { - List witnesses = witnessStore.getAllWitnesses(); - WitnessStore.sortWitnesses(witnesses, false); - for (int i = 0; i < witnesses.size(); i++) { - if (witnesses.get(i).getAddress().equals(witnessAddress)) { - return i + 1L; - } - } - return 0L; - } -} diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index b9771fcf67d..23c40c777a0 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -14,13 +14,12 @@ import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.Metrics; -import org.tron.core.ChainBaseManager; +import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.config.args.CLIParameter; import org.tron.core.config.args.CommandLineArguments; import org.tron.core.exception.TronError; -import org.tron.core.metrics.SrMetricsService; import org.tron.core.services.admin.ipc.client.IpcClient; public class FullNode { @@ -74,9 +73,10 @@ public static void main(String[] args) { context.registerShutdownHook(); appT.startup(); // the genesis block id (chainId) is only available after the context refresh - // (Manager.initGenesis); this also publishes the local SR identity and status - SrMetricsService.recordNodeStatus(context.getBean(ChainBaseManager.class), - context.getBean(BackupManager.class)); + // (Manager.initGenesis); this also publishes the local node identity and the + // initial witness gauges + context.getBean(BackupManager.class).recordNodeInfo(); + context.getBean(MaintenanceManager.class).recordWitnessMetrics(); if (parameter.isSolidityNode()) { SolidityNode node = context.getBean(SolidityNode.class); node.run(); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 21fe303f82e..54298946214 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -23,7 +23,6 @@ import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.args.Args; import org.tron.core.consensus.ConsensusService; -import org.tron.core.metrics.SrMetricsService; import org.tron.program.Version; import org.tron.protos.Protocol; import org.tron.protos.Protocol.Vote; @@ -184,7 +183,8 @@ public void testSrRemoveDirect() { public void testSrStatusFullNode() { Args.setLocalWitnesses(new LocalWitnesses()); - SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); + backupManager.recordNodeInfo(); + maintenanceManager.recordWitnessMetrics(); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); @@ -200,7 +200,7 @@ public void testSrStatusFullNode() { public void testSrStatusMaster() { backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); Args.setLocalWitnesses(new LocalWitnesses()); - SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); + backupManager.recordNodeInfo(); Assert.assertEquals(1.0, info("", MetricLabels.Info.ROLE_FULLNODE), 0.0); ByteString witness = registerWitnessWithVotes(1000); @@ -212,7 +212,8 @@ public void testSrStatusMaster() { localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); - SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); + backupManager.recordNodeInfo(); + maintenanceManager.recordWitnessMetrics(); String address = StringUtil.encode58Check(witness.toByteArray()); Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); @@ -237,7 +238,8 @@ public void testSrStatusSlaver() { localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); - SrMetricsService.recordNodeStatus(chainBaseManager, backupManager); + backupManager.recordNodeInfo(); + maintenanceManager.recordWitnessMetrics(); String address = StringUtil.encode58Check(witness.toByteArray()); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); From 2cb285fe187b1d9e34d5109fbf7fbe0be5416754 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 20 Oct 2026 14:06:33 +0800 Subject: [PATCH 07/26] test(framework): stub metrics beans in startup fixtures Stub BackupManager and MaintenanceManager lookups in both mocked startup contexts. Verify startup, node info, witness gauges and shutdown ordering without changing production code. --- .../program/FullNodeCryptoEngineTest.java | 19 ++++++++++++++++--- .../java/org/tron/program/FullNodeTest.java | 17 ++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java index 311e1887188..42dc1fdbbe6 100644 --- a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java @@ -6,10 +6,12 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.typesafe.config.ConfigUtil; import java.io.File; @@ -20,14 +22,17 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.InOrder; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; +import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.prometheus.Metrics; +import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.args.Args; import org.tron.core.config.args.WitnessInitializer; import org.tron.core.exception.TronError; @@ -89,19 +94,27 @@ private void assertStartup(String cryptoConfig) throws Exception { File config = writeConfig(cryptoConfig); File output = new File(temporaryFolder.newFolder(), "chain"); Application application = mock(Application.class); + BackupManager backupManager = mock(BackupManager.class); + MaintenanceManager maintenanceManager = mock(MaintenanceManager.class); // Exercise the real entry point and config loader, while isolating services and chain stores. try (MockedStatic exit = mockStatic(ExitManager.class); MockedStatic logs = mockStatic(LogService.class); MockedStatic metrics = mockStatic(Metrics.class); MockedStatic factory = mockStatic(ApplicationFactory.class); MockedConstruction contexts = - mockConstruction(TronApplicationContext.class)) { + mockConstruction(TronApplicationContext.class, (context, constructionContext) -> { + when(context.getBean(BackupManager.class)).thenReturn(backupManager); + when(context.getBean(MaintenanceManager.class)).thenReturn(maintenanceManager); + })) { factory.when(() -> ApplicationFactory.create(any())).thenReturn(application); FullNode.main(new String[]{"-c", config.toString(), "-d", output.toString()}); assertEquals(1, contexts.constructed().size()); verify(contexts.constructed().get(0)).refresh(); - verify(application).startup(); - verify(application).blockUntilShutdown(); + InOrder inOrder = inOrder(application, backupManager, maintenanceManager); + inOrder.verify(application).startup(); + inOrder.verify(backupManager).recordNodeInfo(); + inOrder.verify(maintenanceManager).recordWitnessMetrics(); + inOrder.verify(application).blockUntilShutdown(); } } diff --git a/framework/src/test/java/org/tron/program/FullNodeTest.java b/framework/src/test/java/org/tron/program/FullNodeTest.java index bd2dce64088..b0db116f8df 100644 --- a/framework/src/test/java/org/tron/program/FullNodeTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeTest.java @@ -20,6 +20,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.InOrder; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -30,10 +31,12 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.arch.Arch; +import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.Metrics; +import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.args.Args; import org.tron.core.config.args.CommandLineArguments; import org.tron.core.exception.TronError; @@ -70,6 +73,8 @@ public void testNodeModeRetainsInitialization() { CommonParameter parameter = new CommonParameter(); parameter.logbackPath = "node-logback.xml"; Application application = Mockito.mock(Application.class); + BackupManager backupManager = Mockito.mock(BackupManager.class); + MaintenanceManager maintenanceManager = Mockito.mock(MaintenanceManager.class); try (MockedStatic exitManager = Mockito.mockStatic(ExitManager.class); MockedStatic arch = Mockito.mockStatic(Arch.class); MockedStatic args = Mockito.mockStatic(Args.class); @@ -78,7 +83,10 @@ public void testNodeModeRetainsInitialization() { MockedStatic metrics = Mockito.mockStatic(Metrics.class); MockedStatic factory = Mockito.mockStatic(ApplicationFactory.class); MockedConstruction contexts = - Mockito.mockConstruction(TronApplicationContext.class); + Mockito.mockConstruction(TronApplicationContext.class, (context, constructionContext) -> { + Mockito.when(context.getBean(BackupManager.class)).thenReturn(backupManager); + Mockito.when(context.getBean(MaintenanceManager.class)).thenReturn(maintenanceManager); + }); MockedConstruction ipcClients = Mockito.mockConstruction(IpcClient.class)) { parameters.when(CommonParameter::getInstance).thenReturn(parameter); factory.when(() -> ApplicationFactory.create(Mockito.any())).thenReturn(application); @@ -94,8 +102,11 @@ public void testNodeModeRetainsInitialization() { metrics.verify(Metrics::init); Assert.assertEquals(1, contexts.constructed().size()); Mockito.verify(contexts.constructed().get(0)).refresh(); - Mockito.verify(application).startup(); - Mockito.verify(application).blockUntilShutdown(); + InOrder order = Mockito.inOrder(application, backupManager, maintenanceManager); + order.verify(application).startup(); + order.verify(backupManager).recordNodeInfo(); + order.verify(maintenanceManager).recordWitnessMetrics(); + order.verify(application).blockUntilShutdown(); Assert.assertTrue(ipcClients.constructed().isEmpty()); } } From a0fa17f5d269f24c4d9d7d9307ac18fdb177f6cc Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 20 Oct 2026 15:01:44 +0800 Subject: [PATCH 08/26] refactor(metrics): expose witness role as one-hot gauge Address review feedback: keep node_info in its original form (version, genesis_block_id) recorded in FullNode, and move the witness address and role into a dedicated one-hot gauge tron:node_role{address, role} where the full role set (init/master/slaver) is always written and only the current role is 1. The gauge is exposed on witness-configured nodes only. BackupManager now derives the role: with no backup election configured the node is master; with backup configured but the election not yet initialized it reports init, covering the pre-init window where the status field default previously read as an elected master. MetricsInfo is reverted to its original two-label implementation, removing the mutable-label mechanism and the label-swap race with concurrent scrapes. --- .../tron/common/prometheus/MetricKeys.java | 3 + .../tron/common/prometheus/MetricLabels.java | 17 ++- .../tron/common/prometheus/MetricsGauge.java | 5 + .../tron/common/prometheus/MetricsInfo.java | 12 +- .../org/tron/common/prometheus/SRMetrics.java | 17 ++- docs/metrics-changelog.md | 3 +- .../org/tron/common/backup/BackupManager.java | 45 ++++-- .../main/java/org/tron/program/FullNode.java | 8 +- .../tron/common/prometheus/SRMetricsTest.java | 137 ++++++++++++++---- .../prometheus/PrometheusApiServiceTest.java | 30 +++- .../program/FullNodeCryptoEngineTest.java | 6 +- .../java/org/tron/program/FullNodeTest.java | 14 +- 12 files changed, 214 insertions(+), 83 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 55d1f301bd2..8be19d48cec 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -46,6 +46,9 @@ public static class Gauge { public static final String SR_IS_ACTIVE = "tron:sr_is_active"; public static final String SR_VOTES = "tron:sr_votes"; public static final String SR_RANK = "tron:sr_rank"; + // one-hot role indicator of the local node in the backup election, refreshed at + // startup and on every backup-status change; witness-configured nodes only + public static final String NODE_ROLE = "tron:node_role"; private Gauge() { throw new IllegalStateException("Gauge"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index df7afe7cba0..ef0af9a9acd 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -59,6 +59,18 @@ public static class Gauge { public static final String PEERS_VALID = "valid"; + // node_role labels: the gauge publishes one series per backup-election role and + // sets the current one to 1, so dashboards filter on role="..." value == 1 + public static final String ADDRESS = "address"; + + public static final String ROLE = "role"; + + public static final String ROLE_INIT = "init"; + + public static final String ROLE_MASTER = "master"; + + public static final String ROLE_SLAVER = "slaver"; + private Gauge() { throw new IllegalStateException("Gauge"); } @@ -84,11 +96,6 @@ public static class Info { // identifies the genesis block: the label value is the chain id derived from the // genesis block hash, so the label is named genesis_block_id public static final String CHAIN_ID = "genesis_block_id"; - public static final String ADDRESS = "address"; - public static final String ROLE = "role"; - public static final String ROLE_MASTER = "master"; - public static final String ROLE_SLAVER = "slaver"; - public static final String ROLE_FULLNODE = "fullnode"; private Info() { throw new IllegalStateException("Info"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index 73b27be5212..5dce48b0f61 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -24,6 +24,11 @@ class MetricsGauge { init(MetricKeys.Gauge.SR_VOTES, "Vote count of this node's witness."); init(MetricKeys.Gauge.SR_RANK, "Vote-count rank of this node's witness (1-based, 0 when not configured)."); + init(MetricKeys.Gauge.NODE_ROLE, + "Role of this node in the backup election: the current role reports 1 and the " + + "other two report 0 (init: before the first election round; master: " + + "producing blocks; slaver: standing by). Witness-configured nodes only.", + MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE); } private MetricsGauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsInfo.java b/common/src/main/java/org/tron/common/prometheus/MetricsInfo.java index 6e2482b5e33..e3cd8b7ca65 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsInfo.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsInfo.java @@ -1,7 +1,6 @@ package org.tron.common.prometheus; import io.prometheus.client.Info; -import java.util.Arrays; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; @@ -11,12 +10,9 @@ class MetricsInfo { private static final Map container = new ConcurrentHashMap<>(); - private static final Map lastLabels = new ConcurrentHashMap<>(); - static { init(MetricKeys.Info.NODE_INFO, "tron node info.", - MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID, - MetricLabels.Info.ADDRESS, MetricLabels.Info.ROLE); + MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID); } private MetricsInfo() { @@ -38,12 +34,6 @@ static void set(String key, String... labels) { logger.info("{} not exist", key); return; } - String[] oldLabels = lastLabels.put(key, labels); - if (oldLabels != null && !Arrays.equals(oldLabels, labels)) { - // label values are part of the series identity: drop the previous one so an - // address or role change does not leave a stale series behind - info.remove(oldLabels); - } info.labels(labels); } } diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 16ebaeef380..ee9c62c83f4 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -24,12 +24,21 @@ public static byte[] getLocalWitnessAddress() { return localWitnessAddress; } - public static void recordNodeInfo(String version, String chainId, String address, - String role) { - if (!Metrics.enabled()) { + /** + * Publishes the node-role gauge as three one-hot series (init, master, slaver) for the + * given witness address, with the current role set to 1, so a role change rewrites the + * same series instead of leaving stale ones behind. A no-op when no witness address is + * given: nodes without a local witness expose no role series. + */ + public static void recordNodeRole(String address, String role) { + if (!Metrics.enabled() || address.isEmpty()) { return; } - Metrics.info(MetricKeys.Info.NODE_INFO, version, chainId, address, role); + String[] roles = {MetricLabels.Gauge.ROLE_INIT, MetricLabels.Gauge.ROLE_MASTER, + MetricLabels.Gauge.ROLE_SLAVER}; + for (String r : roles) { + Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, r.equals(role) ? 1 : 0, address, r); + } } public static void recordSrStatus(boolean active, long votes, long rank) { diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 05278e8b7cd..e8604fa3353 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -9,7 +9,8 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core -- `tron:node_info` (Info, labels `version`, `genesis_block_id`, `address`, `role`) — static node identity: the running node version string, the full genesis block hash as the canonical chain identifier, the locally configured SR witness address in base58 (empty when the node is not configured as a witness), and the node role: `master` when the backup election has this node producing blocks, `slaver` when it stands by for a master, `fullnode` when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_role` (Gauge, labels `address`, `role`) — one-hot role indicator for the locally configured witness: the current role reports 1 and the other two roles report 0. `role` is `init` from node startup until the backup election first runs, `master` when this node produces blocks (always, when backup is not configured), and `slaver` when it stands by for a master. Only witness-configured nodes expose this metric. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_is_active` (Gauge) — 1 when the locally configured witness address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_votes` (Gauge) — vote count of the locally configured witness, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_rank` (Gauge) — 1-based rank of the locally configured witness by vote count, 0 when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 1d086dbf5eb..cd5b663d9d3 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -14,6 +14,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -32,7 +33,6 @@ import org.tron.common.utils.StringUtil; import org.tron.core.config.args.Args; import org.tron.p2p.utils.NetUtil; -import org.tron.program.Version; @Slf4j(topic = "backup") @Component @@ -74,29 +74,46 @@ public class BackupManager implements EventHandler { public void setStatus(BackupStatusEnum status) { logger.info("Change backup status to {}", status); this.status = status; - recordNodeInfo(); + recordNodeRole(); } /** - * Publishes the {@code node_info} labels (version, chain id, local witness address, - * role derived from the current backup status) and caches the local witness address - * for maintenance-time gauge recording. Called once at node startup and on every - * backup-status change. + * Publishes the {@code tron:node_role} gauge for the locally configured witness and + * caches that address for maintenance-time gauge recording. Runs once when the bean is + * created — covering the window before the backup election first runs — and again on + * every backup-status change. No role series is written when no witness is configured. */ - public void recordNodeInfo() { + @PostConstruct + public void recordNodeRole() { if (!Metrics.enabled()) { return; } byte[] witnessAddress = localWitnessAddress(); SRMetrics.setLocalWitnessAddress(witnessAddress); - String witnessAddress58 = ""; - String role = MetricLabels.Info.ROLE_FULLNODE; - if (witnessAddress != null && witnessAddress.length > 0) { - witnessAddress58 = StringUtil.encode58Check(witnessAddress); - role = status == MASTER ? MetricLabels.Info.ROLE_MASTER : MetricLabels.Info.ROLE_SLAVER; + if (witnessAddress == null || witnessAddress.length == 0) { + return; + } + SRMetrics.recordNodeRole(StringUtil.encode58Check(witnessAddress), currentRole()); + } + + private String currentRole() { + if (!isBackupConfigured()) { + // no backup election to take part in: this node produces blocks itself + return MetricLabels.Gauge.ROLE_MASTER; } - SRMetrics.recordNodeInfo(Version.getVersion(), Args.getInstance().getChainId(), - witnessAddress58, role); + if (!isInit) { + // the backup server only starts after the first fresh block, so between startup + // and the first election round the status default (MASTER) is not a real result; + // report init instead of pretending the election chose a master + return MetricLabels.Gauge.ROLE_INIT; + } + return status == MASTER ? MetricLabels.Gauge.ROLE_MASTER : MetricLabels.Gauge.ROLE_SLAVER; + } + + private boolean isBackupConfigured() { + // same condition BackupServer.initServer() uses to start the election transport, + // i.e. the only way init() -- and with it the first real status -- can be reached + return parameter.getBackupPort() > 0 && !parameter.getBackupMembers().isEmpty(); } private byte[] localWitnessAddress() { diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 23c40c777a0..7655d148b11 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -9,10 +9,10 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.arch.Arch; -import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.Metrics; import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.DefaultConfig; @@ -73,9 +73,9 @@ public static void main(String[] args) { context.registerShutdownHook(); appT.startup(); // the genesis block id (chainId) is only available after the context refresh - // (Manager.initGenesis); this also publishes the local node identity and the - // initial witness gauges - context.getBean(BackupManager.class).recordNodeInfo(); + // (Manager.initGenesis) + Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), + Args.getInstance().getChainId()); context.getBean(MaintenanceManager.class).recordWitnessMetrics(); if (parameter.isSolidityNode()) { SolidityNode node = context.getBean(SolidityNode.class); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 54298946214..b1fc3e7b5b3 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -57,6 +57,8 @@ public void setUp() { @After public void tearDown() { Args.getInstance().setMetricsPrometheusEnable(true); + Args.getInstance().setBackupPort(0); + Args.getInstance().setBackupMembers(Collections.emptyList()); Args.setLocalWitnesses(new LocalWitnesses()); backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); } @@ -177,32 +179,34 @@ public void testSrRemoveDirect() { } /** - * No witness configured: role is fullnode, gauges are zero, address label is empty. + * No witness configured: no node_role series is exposed at all, while the sr gauges + * still report zeros and node_info keeps its two static labels. */ @Test public void testSrStatusFullNode() { Args.setLocalWitnesses(new LocalWitnesses()); - backupManager.recordNodeInfo(); + backupManager.recordNodeRole(); maintenanceManager.recordWitnessMetrics(); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); - Assert.assertEquals(1.0, info("", MetricLabels.Info.ROLE_FULLNODE), 0.0); + for (String role : roles()) { + Assert.assertNull("no " + role + " series without a witness", roleGauge("", role)); + } + Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), + Args.getInstance().getChainId()); + Assert.assertEquals(1.0, info(), 0.0); } /** - * Witness in the active set: role is master, gauges expose its votes and 1-based rank, and - * the previously published fullnode series is dropped when the label values change. + * Witness configured without a backup election (backup members unset): the node always + * produces blocks, so the role gauge reports master; node_info stays a static + * two-label identity record. */ @Test public void testSrStatusMaster() { - backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); - Args.setLocalWitnesses(new LocalWitnesses()); - backupManager.recordNodeInfo(); - Assert.assertEquals(1.0, info("", MetricLabels.Info.ROLE_FULLNODE), 0.0); - ByteString witness = registerWitnessWithVotes(1000); registerWitnessWithVotes(50); chainBaseManager.getWitnessScheduleStore() @@ -212,22 +216,28 @@ public void testSrStatusMaster() { localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); - backupManager.recordNodeInfo(); + backupManager.recordNodeRole(); maintenanceManager.recordWitnessMetrics(); String address = StringUtil.encode58Check(witness.toByteArray()); Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(1000.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); - Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_MASTER), 0.0); - Assert.assertNull(info("", MetricLabels.Info.ROLE_FULLNODE)); + Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), + Args.getInstance().getChainId()); + Assert.assertEquals(1.0, info(), 0.0); } /** - * Backup election slaver: role is slaver and is_active reflects the active witness set. + * Backup election slaver: the role gauge reports slaver while is_active reflects the + * active witness set. */ @Test - public void testSrStatusSlaver() { + public void testSrStatusSlaver() throws Exception { + configureBackup(); backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); ByteString activeWitness = registerWitnessWithVotes(10); ByteString witness = registerWitnessWithVotes(7); @@ -237,36 +247,103 @@ public void testSrStatusSlaver() { LocalWitnesses localWitnesses = new LocalWitnesses(); localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); + markBackupInitiated(backupManager); - backupManager.recordNodeInfo(); + backupManager.recordNodeRole(); maintenanceManager.recordWitnessMetrics(); String address = StringUtil.encode58Check(witness.toByteArray()); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(7.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); Assert.assertTrue(gauge(MetricKeys.Gauge.SR_RANK) > 0); - Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_SLAVER), 0.0); + Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + } + + /** + * Before the first election round (backup configured, BackupManager.init() not yet run, + * which in production only happens after the first fresh block) the node must report + * init instead of falling back to the MASTER field default; the first real status + * after init() switches the same series to the elected role. + */ + @Test + public void testRoleInitBeforeBackupElectionRuns() throws Exception { + configureBackup(); + ByteString witness = registerWitnessWithVotes(1); + LocalWitnesses localWitnesses = new LocalWitnesses(); + localWitnesses.initWitnessAccountAddress(witness.toByteArray()); + Args.setLocalWitnesses(localWitnesses); + String address = StringUtil.encode58Check(witness.toByteArray()); + + // a directly constructed manager has not gone through init(): pre-election window + BackupManager fresh = new BackupManager(); + try { + fresh.recordNodeRole(); + Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); + + // a status change while init() has not run stays inside the window + fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + + // once the election has run, the status maps to the elected role + markBackupInitiated(fresh); + fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + } finally { + fresh.stop(); + } } /** - * setStatus() itself republishes the node_info role without touching any store, so a + * setStatus() itself republishes the node_role gauge without touching any store, so a * backup-election transition is reflected immediately instead of at the next maintenance. */ @Test - public void testRoleRefreshOnBackupStatusChange() { + public void testRoleRefreshOnBackupStatusChange() throws Exception { + configureBackup(); ByteString witness = registerWitnessWithVotes(9); LocalWitnesses localWitnesses = new LocalWitnesses(); localWitnesses.initWitnessAccountAddress(witness.toByteArray()); Args.setLocalWitnesses(localWitnesses); + markBackupInitiated(backupManager); String address = StringUtil.encode58Check(witness.toByteArray()); backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_SLAVER), 0.0); - Assert.assertNull(info(address, MetricLabels.Info.ROLE_MASTER)); + Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); - Assert.assertEquals(1.0, info(address, MetricLabels.Info.ROLE_MASTER), 0.0); - Assert.assertNull(info(address, MetricLabels.Info.ROLE_SLAVER)); + Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); + Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); + } + + /** + * Backup must be seen as configured the same way BackupServer.initServer() decides to + * start the election transport, otherwise the pre-election window is unreachable. + */ + private void configureBackup() { + Args.getInstance().setBackupPort(PublicMethod.chooseRandomPort()); + Args.getInstance().setBackupMembers(Collections.singletonList("127.0.0.2")); + } + + /** + * Simulates that BackupManager.init() has run: in production the backup server only + * starts after the first fresh block, so tests drive the isInit flag directly instead + * of spinning up the election transport. + */ + private void markBackupInitiated(BackupManager manager) throws Exception { + java.lang.reflect.Field isInit = BackupManager.class.getDeclaredField("isInit"); + isInit.setAccessible(true); + isInit.setBoolean(manager, true); + } + + private String[] roles() { + return new String[] {MetricLabels.Gauge.ROLE_INIT, MetricLabels.Gauge.ROLE_MASTER, + MetricLabels.Gauge.ROLE_SLAVER}; } private ByteString registerWitnessWithVotes(long votes) { @@ -282,12 +359,18 @@ private Double gauge(String name) { return CollectorRegistry.defaultRegistry.getSampleValue(name); } - private Double info(String address, String role) { + private Double info() { return CollectorRegistry.defaultRegistry.getSampleValue( "tron:node_info", - new String[] {MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID, - MetricLabels.Info.ADDRESS, MetricLabels.Info.ROLE}, - new String[] {Version.getVersion(), Args.getInstance().getChainId(), address, role}); + new String[] {MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID}, + new String[] {Version.getVersion(), Args.getInstance().getChainId()}); + } + + private Double roleGauge(String address, String role) { + return CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_ROLE, + new String[] {MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE}, + new String[] {address, role}); } private ByteString registerWitness() { diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index 9d78dd1db77..31f3315ded9 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -24,6 +24,7 @@ import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.MetricLabels; import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; import org.tron.common.utils.ByteArray; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.Sha256Hash; @@ -214,17 +215,38 @@ public void testNodeInfoMetric() { String version = Version.getVersion(); String testGenesisBlockId = "00000000000000001ebf88508a03865c71d452e25f4d51194196a1d22b6653dc"; - Metrics.info(MetricKeys.Info.NODE_INFO, version, testGenesisBlockId, "", "fullnode"); + Metrics.info(MetricKeys.Info.NODE_INFO, version, testGenesisBlockId); // Prometheus Info collector appends "_info" to the sample name Double value = CollectorRegistry.defaultRegistry.getSampleValue( "tron:node_info", - new String[] {MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID, - MetricLabels.Info.ADDRESS, MetricLabels.Info.ROLE}, - new String[] {version, testGenesisBlockId, "", "fullnode"}); + new String[] {MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID}, + new String[] {version, testGenesisBlockId}); Assert.assertNotNull("tron:node_info sample should exist", value); Assert.assertEquals(1.0, value, 0.0); } + @Test + public void testNodeRoleMetric() { + String address = "TLyqz8V6CR3s9nNote9u1NoxKZ8NqBSbn1"; + SRMetrics.recordNodeRole(address, MetricLabels.Gauge.ROLE_MASTER); + // the current role reports 1, the other two series report 0 + Double master = CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_ROLE, + new String[] {MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE}, + new String[] {address, MetricLabels.Gauge.ROLE_MASTER}); + Double slaver = CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_ROLE, + new String[] {MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE}, + new String[] {address, MetricLabels.Gauge.ROLE_SLAVER}); + Double init = CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_ROLE, + new String[] {MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE}, + new String[] {address, MetricLabels.Gauge.ROLE_INIT}); + Assert.assertEquals(1.0, master, 0.0); + Assert.assertEquals(0.0, slaver, 0.0); + Assert.assertEquals(0.0, init, 0.0); + } + @Test public void testNodeInfoUnknownKey() { // unknown key exercises the null-guard branch in MetricsInfo.set diff --git a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java index 42dc1fdbbe6..1919afd3488 100644 --- a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java @@ -28,7 +28,6 @@ import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.prometheus.Metrics; @@ -94,7 +93,6 @@ private void assertStartup(String cryptoConfig) throws Exception { File config = writeConfig(cryptoConfig); File output = new File(temporaryFolder.newFolder(), "chain"); Application application = mock(Application.class); - BackupManager backupManager = mock(BackupManager.class); MaintenanceManager maintenanceManager = mock(MaintenanceManager.class); // Exercise the real entry point and config loader, while isolating services and chain stores. try (MockedStatic exit = mockStatic(ExitManager.class); @@ -103,16 +101,14 @@ private void assertStartup(String cryptoConfig) throws Exception { MockedStatic factory = mockStatic(ApplicationFactory.class); MockedConstruction contexts = mockConstruction(TronApplicationContext.class, (context, constructionContext) -> { - when(context.getBean(BackupManager.class)).thenReturn(backupManager); when(context.getBean(MaintenanceManager.class)).thenReturn(maintenanceManager); })) { factory.when(() -> ApplicationFactory.create(any())).thenReturn(application); FullNode.main(new String[]{"-c", config.toString(), "-d", output.toString()}); assertEquals(1, contexts.constructed().size()); verify(contexts.constructed().get(0)).refresh(); - InOrder inOrder = inOrder(application, backupManager, maintenanceManager); + InOrder inOrder = inOrder(application, maintenanceManager); inOrder.verify(application).startup(); - inOrder.verify(backupManager).recordNodeInfo(); inOrder.verify(maintenanceManager).recordWitnessMetrics(); inOrder.verify(application).blockUntilShutdown(); } diff --git a/framework/src/test/java/org/tron/program/FullNodeTest.java b/framework/src/test/java/org/tron/program/FullNodeTest.java index b0db116f8df..6d89bdb76c3 100644 --- a/framework/src/test/java/org/tron/program/FullNodeTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeTest.java @@ -31,7 +31,6 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.arch.Arch; -import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; @@ -73,7 +72,6 @@ public void testNodeModeRetainsInitialization() { CommonParameter parameter = new CommonParameter(); parameter.logbackPath = "node-logback.xml"; Application application = Mockito.mock(Application.class); - BackupManager backupManager = Mockito.mock(BackupManager.class); MaintenanceManager maintenanceManager = Mockito.mock(MaintenanceManager.class); try (MockedStatic exitManager = Mockito.mockStatic(ExitManager.class); MockedStatic arch = Mockito.mockStatic(Arch.class); @@ -83,10 +81,11 @@ public void testNodeModeRetainsInitialization() { MockedStatic metrics = Mockito.mockStatic(Metrics.class); MockedStatic factory = Mockito.mockStatic(ApplicationFactory.class); MockedConstruction contexts = - Mockito.mockConstruction(TronApplicationContext.class, (context, constructionContext) -> { - Mockito.when(context.getBean(BackupManager.class)).thenReturn(backupManager); - Mockito.when(context.getBean(MaintenanceManager.class)).thenReturn(maintenanceManager); - }); + Mockito.mockConstruction(TronApplicationContext.class, + (context, constructionContext) -> { + Mockito.when(context.getBean(MaintenanceManager.class)) + .thenReturn(maintenanceManager); + }); MockedConstruction ipcClients = Mockito.mockConstruction(IpcClient.class)) { parameters.when(CommonParameter::getInstance).thenReturn(parameter); factory.when(() -> ApplicationFactory.create(Mockito.any())).thenReturn(application); @@ -102,9 +101,8 @@ public void testNodeModeRetainsInitialization() { metrics.verify(Metrics::init); Assert.assertEquals(1, contexts.constructed().size()); Mockito.verify(contexts.constructed().get(0)).refresh(); - InOrder order = Mockito.inOrder(application, backupManager, maintenanceManager); + InOrder order = Mockito.inOrder(application, maintenanceManager); order.verify(application).startup(); - order.verify(backupManager).recordNodeInfo(); order.verify(maintenanceManager).recordWitnessMetrics(); order.verify(application).blockUntilShutdown(); Assert.assertTrue(ipcClients.constructed().isEmpty()); From 65159f74cb61d70c1255170ae51e12858980dd9a Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 20 Oct 2026 17:03:33 +0800 Subject: [PATCH 09/26] refactor(metrics): encode node role as single numeric gauge tron:node_role now exposes one series per witness address whose value encodes the role: 0=init, 1=master, 2=slaver. This replaces the previous one-hot form that published three series per address with a role label. - SRMetrics owns the mapping as package-visible constants shared with tests; unknown role names fall back to init so a misderived state cannot masquerade as an elected one. - The gauge is registered with the single label 'address' and its help text documents the numeric mapping. - docs/metrics-changelog.md updated with the 0/1/2 mapping table. - Tests updated to assert the numeric encoding; a plain fullnode with no local witness still exposes no series. --- .../tron/common/prometheus/MetricLabels.java | 8 ++-- .../tron/common/prometheus/MetricsGauge.java | 8 ++-- .../org/tron/common/prometheus/SRMetrics.java | 34 +++++++++---- docs/metrics-changelog.md | 2 +- .../tron/common/prometheus/SRMetricsTest.java | 48 +++++++------------ .../prometheus/PrometheusApiServiceTest.java | 22 +++------ 6 files changed, 58 insertions(+), 64 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index ef0af9a9acd..d3f8c90a41f 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -59,12 +59,12 @@ public static class Gauge { public static final String PEERS_VALID = "valid"; - // node_role labels: the gauge publishes one series per backup-election role and - // sets the current one to 1, so dashboards filter on role="..." value == 1 + // node_role label: one gauge series per witness address, whose value encodes the + // current backup-election role as a number (0 = init, 1 = master, 2 = slaver); + // ROLE_INIT/ROLE_MASTER/ROLE_SLAVER are the canonical role names produced by + // BackupManager and mapped to numbers in SRMetrics.recordNodeRole() public static final String ADDRESS = "address"; - public static final String ROLE = "role"; - public static final String ROLE_INIT = "init"; public static final String ROLE_MASTER = "master"; diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index 5dce48b0f61..a3eaf06c3c2 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -25,10 +25,10 @@ class MetricsGauge { init(MetricKeys.Gauge.SR_RANK, "Vote-count rank of this node's witness (1-based, 0 when not configured)."); init(MetricKeys.Gauge.NODE_ROLE, - "Role of this node in the backup election: the current role reports 1 and the " - + "other two report 0 (init: before the first election round; master: " - + "producing blocks; slaver: standing by). Witness-configured nodes only.", - MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE); + "Role of this node in the backup election, encoded as a number: 0 = init " + + "(before the first election round), 1 = master (producing blocks), " + + "2 = slaver (standing by). Witness-configured nodes only.", + MetricLabels.Gauge.ADDRESS); } private MetricsGauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index ee9c62c83f4..9f2c09b2ea3 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -24,21 +24,39 @@ public static byte[] getLocalWitnessAddress() { return localWitnessAddress; } + // numeric encoding of the backup-election role in the tron:node_role gauge: + // 0 = init (before the first election round), 1 = master (producing blocks), + // 2 = slaver (standing by). Kept package-visible for tests; the mapping is + // documented for operators in docs/metrics-changelog.md + static final int NODE_ROLE_INIT = 0; + static final int NODE_ROLE_MASTER = 1; + static final int NODE_ROLE_SLAVER = 2; + /** - * Publishes the node-role gauge as three one-hot series (init, master, slaver) for the - * given witness address, with the current role set to 1, so a role change rewrites the - * same series instead of leaving stale ones behind. A no-op when no witness address is - * given: nodes without a local witness expose no role series. + * Publishes the node-role gauge as a single series per witness address, encoding the + * current role as a number (0 = init, 1 = master, 2 = slaver). A role change rewrites + * the value of the same series, so no stale series are left behind. A no-op when no + * witness address is given: nodes without a local witness expose no role series. */ public static void recordNodeRole(String address, String role) { if (!Metrics.enabled() || address.isEmpty()) { return; } - String[] roles = {MetricLabels.Gauge.ROLE_INIT, MetricLabels.Gauge.ROLE_MASTER, - MetricLabels.Gauge.ROLE_SLAVER}; - for (String r : roles) { - Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, r.equals(role) ? 1 : 0, address, r); + Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, roleValue(role), address); + } + + /** + * Maps a role name to its {@code tron:node_role} value; an unknown role falls back to + * init so a bad argument can never fake an elected state. + */ + private static int roleValue(String role) { + if (MetricLabels.Gauge.ROLE_MASTER.equals(role)) { + return NODE_ROLE_MASTER; + } + if (MetricLabels.Gauge.ROLE_SLAVER.equals(role)) { + return NODE_ROLE_SLAVER; } + return NODE_ROLE_INIT; } public static void recordSrStatus(boolean active, long votes, long rank) { diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index e8604fa3353..9fcc5179358 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -10,7 +10,7 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:node_role` (Gauge, labels `address`, `role`) — one-hot role indicator for the locally configured witness: the current role reports 1 and the other two roles report 0. `role` is `init` from node startup until the backup election first runs, `master` when this node produces blocks (always, when backup is not configured), and `slaver` when it stands by for a master. Only witness-configured nodes expose this metric. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_role` (Gauge, label `address`) — role of this node in the backup election, encoded as a number: `0` = `init` (from node startup until the backup election first runs), `1` = `master` (producing blocks; always the value when backup is not configured), `2` = `slaver` (standing by for a master). A role change rewrites the value of the same series, so no stale series remain. Only witness-configured nodes expose this metric. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_is_active` (Gauge) — 1 when the locally configured witness address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_votes` (Gauge) — vote count of the locally configured witness, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_rank` (Gauge) — 1-based rank of the locally configured witness by vote count, 0 when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index b1fc3e7b5b3..3a5aef0565d 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -192,9 +192,7 @@ public void testSrStatusFullNode() { Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); - for (String role : roles()) { - Assert.assertNull("no " + role + " series without a witness", roleGauge("", role)); - } + Assert.assertNull("no node_role series without a witness", roleGauge("")); Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); Assert.assertEquals(1.0, info(), 0.0); @@ -202,7 +200,7 @@ public void testSrStatusFullNode() { /** * Witness configured without a backup election (backup members unset): the node always - * produces blocks, so the role gauge reports master; node_info stays a static + * produces blocks, so the role gauge reports master (1); node_info stays a static * two-label identity record. */ @Test @@ -223,17 +221,15 @@ public void testSrStatusMaster() { Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(1000.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); - Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address), 0.0); Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); Assert.assertEquals(1.0, info(), 0.0); } /** - * Backup election slaver: the role gauge reports slaver while is_active reflects the - * active witness set. + * Backup election slaver: the role gauge reports slaver (2) while is_active reflects + * the active witness set. */ @Test public void testSrStatusSlaver() throws Exception { @@ -256,16 +252,14 @@ public void testSrStatusSlaver() throws Exception { Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); Assert.assertEquals(7.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); Assert.assertTrue(gauge(MetricKeys.Gauge.SR_RANK) > 0); - Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address), 0.0); } /** * Before the first election round (backup configured, BackupManager.init() not yet run, * which in production only happens after the first fresh block) the node must report - * init instead of falling back to the MASTER field default; the first real status - * after init() switches the same series to the elected role. + * init (0) instead of falling back to the MASTER field default; the first real status + * after init() rewrites the same series to the elected role. */ @Test public void testRoleInitBeforeBackupElectionRuns() throws Exception { @@ -280,19 +274,16 @@ public void testRoleInitBeforeBackupElectionRuns() throws Exception { BackupManager fresh = new BackupManager(); try { fresh.recordNodeRole(); - Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(address), 0.0); // a status change while init() has not run stays inside the window fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(address), 0.0); // once the election has run, the status maps to the elected role markBackupInitiated(fresh); fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_INIT), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address), 0.0); } finally { fresh.stop(); } @@ -313,12 +304,10 @@ public void testRoleRefreshOnBackupStatusChange() throws Exception { String address = StringUtil.encode58Check(witness.toByteArray()); backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address), 0.0); backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); - Assert.assertEquals(1.0, roleGauge(address, MetricLabels.Gauge.ROLE_MASTER), 0.0); - Assert.assertEquals(0.0, roleGauge(address, MetricLabels.Gauge.ROLE_SLAVER), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address), 0.0); } /** @@ -341,11 +330,6 @@ private void markBackupInitiated(BackupManager manager) throws Exception { isInit.setBoolean(manager, true); } - private String[] roles() { - return new String[] {MetricLabels.Gauge.ROLE_INIT, MetricLabels.Gauge.ROLE_MASTER, - MetricLabels.Gauge.ROLE_SLAVER}; - } - private ByteString registerWitnessWithVotes(long votes) { ByteString address = uniqueAddress(); chainBaseManager.getWitnessStore().put(address.toByteArray(), @@ -366,11 +350,11 @@ private Double info() { new String[] {Version.getVersion(), Args.getInstance().getChainId()}); } - private Double roleGauge(String address, String role) { + private Double roleGauge(String address) { return CollectorRegistry.defaultRegistry.getSampleValue( MetricKeys.Gauge.NODE_ROLE, - new String[] {MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE}, - new String[] {address, role}); + new String[] {MetricLabels.Gauge.ADDRESS}, + new String[] {address}); } private ByteString registerWitness() { diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index 31f3315ded9..489c6e1a707 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -229,22 +229,14 @@ public void testNodeInfoMetric() { public void testNodeRoleMetric() { String address = "TLyqz8V6CR3s9nNote9u1NoxKZ8NqBSbn1"; SRMetrics.recordNodeRole(address, MetricLabels.Gauge.ROLE_MASTER); - // the current role reports 1, the other two series report 0 - Double master = CollectorRegistry.defaultRegistry.getSampleValue( - MetricKeys.Gauge.NODE_ROLE, - new String[] {MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE}, - new String[] {address, MetricLabels.Gauge.ROLE_MASTER}); - Double slaver = CollectorRegistry.defaultRegistry.getSampleValue( - MetricKeys.Gauge.NODE_ROLE, - new String[] {MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE}, - new String[] {address, MetricLabels.Gauge.ROLE_SLAVER}); - Double init = CollectorRegistry.defaultRegistry.getSampleValue( + // a single series per node; the value encodes the role (0 = init, 1 = master, + // 2 = slaver), here master after recording ROLE_MASTER + Double value = CollectorRegistry.defaultRegistry.getSampleValue( MetricKeys.Gauge.NODE_ROLE, - new String[] {MetricLabels.Gauge.ADDRESS, MetricLabels.Gauge.ROLE}, - new String[] {address, MetricLabels.Gauge.ROLE_INIT}); - Assert.assertEquals(1.0, master, 0.0); - Assert.assertEquals(0.0, slaver, 0.0); - Assert.assertEquals(0.0, init, 0.0); + new String[] {MetricLabels.Gauge.ADDRESS}, + new String[] {address}); + Assert.assertNotNull("tron:node_role sample should exist", value); + Assert.assertEquals(1.0, value, 0.0); } @Test From c992ddabaff290b32f87d72ddbe5bf2c5367f303 Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 21 Oct 2026 16:23:33 +0800 Subject: [PATCH 10/26] feat(metrics): publish node_role per configured witness address Review follow-ups on the backup role gauge: - Post-init the election status is a real outcome, so map it 1:1: INIT -> init(0), MASTER -> master(1), SLAVER -> slaver(2). Previously a post-init INIT was misreported as slaver(2). The pre-init window is still gated by isInit, since the status field initializer (MASTER) is not an election result and cannot change: BlockHandleImpl.getState() relies on the MASTER default to decide whether this node may produce a block, and a node without backup configured never runs init(). - Multi-key setups (testnets) get one node_role series per configured witness identity: the witness account address when set, otherwise one address derived per private key, all carrying the same node-level role value. --- docs/metrics-changelog.md | 2 +- .../org/tron/common/backup/BackupManager.java | 58 ++++++++++++--- .../tron/common/prometheus/SRMetricsTest.java | 72 +++++++++++++++++++ 3 files changed, 123 insertions(+), 9 deletions(-) diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 9fcc5179358..9a20a817fa2 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -10,7 +10,7 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:node_role` (Gauge, label `address`) — role of this node in the backup election, encoded as a number: `0` = `init` (from node startup until the backup election first runs), `1` = `master` (producing blocks; always the value when backup is not configured), `2` = `slaver` (standing by for a master). A role change rewrites the value of the same series, so no stale series remain. Only witness-configured nodes expose this metric. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_role` (Gauge, label `address`) — role of this node in the backup election, encoded as a number: `0` = `init` (from node startup until the backup election first runs), `1` = `master` (producing blocks; always the value when backup is not configured), `2` = `slaver` (standing by for a master). A role change rewrites the value of the same series, so no stale series remain. Only witness-configured nodes expose this metric. Refreshed at startup and on every backup-status change. When several witness private keys are configured (testnet setups without a witness account address), one series is written per derived address, all carrying the same value — the role is a node-level status, not a per-identity one. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_is_active` (Gauge) — 1 when the locally configured witness address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_votes` (Gauge) — vote count of the locally configured witness, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_rank` (Gauge) — 1-based rank of the locally configured witness by vote count, 0 when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index cd5b663d9d3..4772f9f64d7 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -9,6 +9,9 @@ import io.netty.util.internal.ConcurrentSet; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -24,11 +27,14 @@ import org.tron.common.backup.socket.EventHandler; import org.tron.common.backup.socket.MessageHandler; import org.tron.common.backup.socket.UdpEvent; +import org.tron.common.crypto.SignInterface; +import org.tron.common.crypto.SignUtils; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.MetricLabels; import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.ByteArray; import org.tron.common.utils.LocalWitnesses; import org.tron.common.utils.StringUtil; import org.tron.core.config.args.Args; @@ -78,10 +84,11 @@ public void setStatus(BackupStatusEnum status) { } /** - * Publishes the {@code tron:node_role} gauge for the locally configured witness and - * caches that address for maintenance-time gauge recording. Runs once when the bean is - * created — covering the window before the backup election first runs — and again on - * every backup-status change. No role series is written when no witness is configured. + * Publishes the {@code tron:node_role} gauge for every locally configured witness + * address and caches the primary witness address for maintenance-time gauge recording. + * Runs once when the bean is created — covering the window before the backup election + * first runs — and again on every backup-status change. No role series is written when + * no witness is configured. */ @PostConstruct public void recordNodeRole() { @@ -90,10 +97,9 @@ public void recordNodeRole() { } byte[] witnessAddress = localWitnessAddress(); SRMetrics.setLocalWitnessAddress(witnessAddress); - if (witnessAddress == null || witnessAddress.length == 0) { - return; + for (String address : localWitnessAddresses()) { + SRMetrics.recordNodeRole(address, currentRole()); } - SRMetrics.recordNodeRole(StringUtil.encode58Check(witnessAddress), currentRole()); } private String currentRole() { @@ -107,7 +113,17 @@ private String currentRole() { // report init instead of pretending the election chose a master return MetricLabels.Gauge.ROLE_INIT; } - return status == MASTER ? MetricLabels.Gauge.ROLE_MASTER : MetricLabels.Gauge.ROLE_SLAVER; + // post-init the status is a real election outcome, so map every state 1:1; the + // pre-init window is already gated by the isInit check above, so INIT here only + // means the election transport is up but no keep-alive timeout has been judged yet + switch (status) { + case MASTER: + return MetricLabels.Gauge.ROLE_MASTER; + case SLAVER: + return MetricLabels.Gauge.ROLE_SLAVER; + default: + return MetricLabels.Gauge.ROLE_INIT; + } } private boolean isBackupConfigured() { @@ -121,6 +137,32 @@ private byte[] localWitnessAddress() { return localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); } + /** + * Resolves every locally configured witness identity to a 58check-encoded address: + * the witness account address when set, otherwise one address per configured private + * key, derived the same way {@link LocalWitnesses#initWitnessAccountAddress(byte[])} + * derives the default witness account address. Returns an empty list when the node has + * no witness configured. Multi-key setups are a testnet convenience: the role is a + * node-level status, so every configured identity gets its own series carrying the + * same value. + */ + private List localWitnessAddresses() { + LocalWitnesses localWitnesses = Args.getLocalWitnesses(); + if (localWitnesses == null) { + return Collections.emptyList(); + } + byte[] accountAddress = localWitnesses.getWitnessAccountAddress(); + if (accountAddress != null && accountAddress.length > 0) { + return Collections.singletonList(StringUtil.encode58Check(accountAddress)); + } + List addresses = new ArrayList<>(); + for (String privateKey : localWitnesses.getPrivateKeys()) { + SignInterface ecKey = SignUtils.fromPrivate(ByteArray.fromHexString(privateKey)); + addresses.add(StringUtil.encode58Check(ecKey.getAddress())); + } + return addresses; + } + public void init() { if (isInit) { diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 3a5aef0565d..a7fa1d9f807 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -14,6 +14,8 @@ import org.tron.common.BaseTest; import org.tron.common.TestConstants; import org.tron.common.backup.BackupManager; +import org.tron.common.crypto.SignUtils; +import org.tron.common.utils.ByteArray; import org.tron.common.utils.LocalWitnesses; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.StringUtil; @@ -310,6 +312,67 @@ public void testRoleRefreshOnBackupStatusChange() throws Exception { Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address), 0.0); } + /** + * After init() has run, status INIT is a real election state — the transport is up but + * no keep-alive timeout has been judged yet — and maps 1:1 to init (0). It must not be + * misreported as slaver (2), the pre-fix behavior. + */ + @Test + public void testRoleInitAfterElectionStarted() throws Exception { + configureBackup(); + ByteString witness = registerWitnessWithVotes(3); + LocalWitnesses localWitnesses = new LocalWitnesses(); + localWitnesses.initWitnessAccountAddress(witness.toByteArray()); + Args.setLocalWitnesses(localWitnesses); + markBackupInitiated(backupManager); + String address = StringUtil.encode58Check(witness.toByteArray()); + + backupManager.setStatus(BackupManager.BackupStatusEnum.INIT); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(address), 0.0); + } + + /** + * Testnet multi-key setup: with several witness private keys configured and no witness + * account address, node_role writes one series per derived address, all carrying the + * same node-level role value, and a status change flips every series together. Role + * recording never touches the single-identity sr_* gauges, and maintenance-time + * recording never touches the role series. + */ + @Test + public void testRoleMultiWitnessKeys() throws Exception { + configureBackup(); + String key1 = PublicMethod.getRandomPrivateKey(); + String key2 = PublicMethod.getRandomPrivateKey(); + LocalWitnesses localWitnesses = new LocalWitnesses(Arrays.asList(key1, key2)); + Args.setLocalWitnesses(localWitnesses); + String address1 = derivedAddress(key1); + String address2 = derivedAddress(key2); + markBackupInitiated(backupManager); + + Double activeBefore = gauge(MetricKeys.Gauge.SR_IS_ACTIVE); + Double votesBefore = gauge(MetricKeys.Gauge.SR_VOTES); + Double rankBefore = gauge(MetricKeys.Gauge.SR_RANK); + + backupManager.recordNodeRole(); + Assert.assertEquals(roleGauge(address1), roleGauge(address2)); + + backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address1), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address2), 0.0); + + backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address1), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address2), 0.0); + + Assert.assertEquals(activeBefore, gauge(MetricKeys.Gauge.SR_IS_ACTIVE)); + Assert.assertEquals(votesBefore, gauge(MetricKeys.Gauge.SR_VOTES)); + Assert.assertEquals(rankBefore, gauge(MetricKeys.Gauge.SR_RANK)); + + maintenanceManager.recordWitnessMetrics(); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address1), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address2), 0.0); + } + /** * Backup must be seen as configured the same way BackupServer.initServer() decides to * start the election transport, otherwise the pre-election window is unreachable. @@ -330,6 +393,15 @@ private void markBackupInitiated(BackupManager manager) throws Exception { isInit.setBoolean(manager, true); } + /** + * Derives the 58check address the same way BackupManager.localWitnessAddresses() + * derives one from a configured private key. + */ + private String derivedAddress(String privateKeyHex) { + return StringUtil.encode58Check( + SignUtils.fromPrivate(ByteArray.fromHexString(privateKeyHex)).getAddress()); + } + private ByteString registerWitnessWithVotes(long votes) { ByteString address = uniqueAddress(); chainBaseManager.getWitnessStore().put(address.toByteArray(), From 98719c65962771f1fe53dff789cecd1ed12cb463 Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 21 Oct 2026 17:50:48 +0800 Subject: [PATCH 11/26] feat(metrics): bind sr gauges to witness addresses - sr_is_active / sr_votes / sr_rank now carry an address label and emit one series per configured local witness address, mirroring tron:node_role; a node without a local witness keeps the existing empty-address zero-value series - SRMetrics caches the full address set resolved by BackupManager (explicit witness account address, or one address per configured private key); MaintenanceManager scans the witness list per address - a malformed address no longer skips recording for the remaining addresses until the next maintenance round (per-address catch + warn) - gauge help text: rank is 0 when the witness is unknown --- .../tron/common/prometheus/MetricLabels.java | 5 +- .../tron/common/prometheus/MetricsGauge.java | 10 +- .../org/tron/common/prometheus/SRMetrics.java | 36 +++++-- .../consensus/dpos/MaintenanceManager.java | 44 +++++--- docs/metrics-changelog.md | 8 +- .../org/tron/common/backup/BackupManager.java | 13 +-- .../tron/common/prometheus/SRMetricsTest.java | 102 ++++++++++++------ 7 files changed, 143 insertions(+), 75 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index d3f8c90a41f..6d1fcc48da1 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -61,10 +61,11 @@ public static class Gauge { // node_role label: one gauge series per witness address, whose value encodes the // current backup-election role as a number (0 = init, 1 = master, 2 = slaver); - // ROLE_INIT/ROLE_MASTER/ROLE_SLAVER are the canonical role names produced by - // BackupManager and mapped to numbers in SRMetrics.recordNodeRole() + // ADDRESS labels both tron:node_role and the tron:sr_* gauges public static final String ADDRESS = "address"; + // ROLE_INIT/ROLE_MASTER/ROLE_SLAVER are the canonical role names produced by + // BackupManager and mapped to numbers in SRMetrics.recordNodeRole() public static final String ROLE_INIT = "init"; public static final String ROLE_MASTER = "master"; diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index a3eaf06c3c2..ac90359618e 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -20,10 +20,14 @@ class MetricsGauge { init(MetricKeys.Gauge.DB_SST_LEVEL, "tron db files .", "type", "db", "level"); init(MetricKeys.Gauge.TX_CACHE, "tron tx cache info.", "type"); init(MetricKeys.Gauge.SR_IS_ACTIVE, - "1 if this node's witness is in the active witness set."); - init(MetricKeys.Gauge.SR_VOTES, "Vote count of this node's witness."); + "1 if the witness with this address is in the active witness set.", + MetricLabels.Gauge.ADDRESS); + init(MetricKeys.Gauge.SR_VOTES, "Vote count of the witness with this address.", + MetricLabels.Gauge.ADDRESS); init(MetricKeys.Gauge.SR_RANK, - "Vote-count rank of this node's witness (1-based, 0 when not configured)."); + "Vote-count rank of the witness with this address (1-based, 0 when the " + + "witness is unknown).", + MetricLabels.Gauge.ADDRESS); init(MetricKeys.Gauge.NODE_ROLE, "Role of this node in the backup election, encoded as a number: 0 = init " + "(before the first election round), 1 = master (producing blocks), " diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 9f2c09b2ea3..2fed1ddf1b4 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -1,7 +1,11 @@ package org.tron.common.prometheus; import com.google.protobuf.ByteString; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.tron.common.utils.StringUtil; public class SRMetrics { @@ -10,18 +14,24 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } - private static volatile byte[] localWitnessAddress; + private static volatile Set localWitnessAddresses = Collections.emptySet(); /** - * Caches the local witness address once it is resolved at startup, so maintenance-time - * recording can read it without reaching back into node configuration. + * Caches the local witness addresses (58check-encoded) once they are resolved at + * startup, so maintenance-time recording can read them without reaching back into node + * configuration. Written once by BackupManager's {@code @PostConstruct} hook: an empty + * collection (no witness configured) is cached as an empty set, a non-empty one as an + * immutable snapshot covering the witness account address or every derived private-key + * address. */ - public static void setLocalWitnessAddress(byte[] address) { - localWitnessAddress = address; + public static void setLocalWitnessAddresses(Collection addresses) { + localWitnessAddresses = addresses.isEmpty() + ? Collections.emptySet() + : Collections.unmodifiableSet(new HashSet<>(addresses)); } - public static byte[] getLocalWitnessAddress() { - return localWitnessAddress; + public static Set getLocalWitnessAddresses() { + return localWitnessAddresses; } // numeric encoding of the backup-election role in the tron:node_role gauge: @@ -59,13 +69,17 @@ private static int roleValue(String role) { return NODE_ROLE_INIT; } - public static void recordSrStatus(boolean active, long votes, long rank) { + /** + * Publishes the sr_* gauges as one series per local witness address. An empty address + * records the node-level zero series a fullnode exposes when no witness is configured. + */ + public static void recordSrStatus(String address, boolean active, long votes, long rank) { if (!Metrics.enabled()) { return; } - Metrics.gaugeSet(MetricKeys.Gauge.SR_IS_ACTIVE, active ? 1 : 0); - Metrics.gaugeSet(MetricKeys.Gauge.SR_VOTES, votes); - Metrics.gaugeSet(MetricKeys.Gauge.SR_RANK, rank); + Metrics.gaugeSet(MetricKeys.Gauge.SR_IS_ACTIVE, active ? 1 : 0, address); + Metrics.gaugeSet(MetricKeys.Gauge.SR_VOTES, votes, address); + Metrics.gaugeSet(MetricKeys.Gauge.SR_RANK, rank, address); } public static void recordSrSetChange(List currentWits, List newWits) { diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index 80d2461a70b..1774e53c6fd 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -18,6 +19,7 @@ import org.springframework.stereotype.Component; import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.Commons; import org.tron.consensus.ConsensusDelegate; import org.tron.consensus.pbft.PbftManager; import org.tron.core.capsule.AccountCapsule; @@ -168,29 +170,39 @@ public void doMaintenance() { /** * Records the witness gauges (active-set membership, settled votes, settled rank) for - * the locally configured witness. Runs at the end of every maintenance and once at - * node startup. Never throws: a metrics failure must not affect maintenance or block + * every locally configured witness address, one series per address. With several + * private keys configured (testnet setups without a witness account address) each + * derived address gets its own series; a node without a witness keeps the zero series + * with an empty address label. Runs at the end of every maintenance and once at node + * startup. Never throws: a metrics failure must not affect maintenance or block * processing. */ public void recordWitnessMetrics() { if (!Metrics.enabled()) { return; } - try { - byte[] witnessAddress = SRMetrics.getLocalWitnessAddress(); - boolean active = false; - long votes = 0; - long rank = 0; - if (witnessAddress != null && witnessAddress.length > 0) { - ByteString witnessAddressBs = ByteString.copyFrom(witnessAddress); - active = consensusDelegate.getActiveWitnesses().contains(witnessAddressBs); - WitnessCapsule witness = consensusDelegate.getWitness(witnessAddress); - votes = witness == null ? 0 : witness.getVoteCount(); - rank = witnessRank(witnessAddressBs); + Set addresses = SRMetrics.getLocalWitnessAddresses(); + if (addresses.isEmpty()) { + SRMetrics.recordSrStatus("", false, 0, 0); + return; + } + for (String address : addresses) { + try { + boolean active = false; + long votes = 0; + long rank = 0; + byte[] witnessAddress = Commons.decode58Check(address); + if (witnessAddress != null && witnessAddress.length > 0) { + ByteString witnessAddressBs = ByteString.copyFrom(witnessAddress); + active = consensusDelegate.getActiveWitnesses().contains(witnessAddressBs); + WitnessCapsule witness = consensusDelegate.getWitness(witnessAddress); + votes = witness == null ? 0 : witness.getVoteCount(); + rank = witnessRank(witnessAddressBs); + } + SRMetrics.recordSrStatus(address, active, votes, rank); + } catch (Exception e) { + logger.warn("Failed to record witness metrics for {}", address, e); } - SRMetrics.recordSrStatus(active, votes, rank); - } catch (Exception e) { - logger.warn("Failed to record witness metrics", e); } } diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 9a20a817fa2..7e34a12e473 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -11,9 +11,11 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:node_role` (Gauge, label `address`) — role of this node in the backup election, encoded as a number: `0` = `init` (from node startup until the backup election first runs), `1` = `master` (producing blocks; always the value when backup is not configured), `2` = `slaver` (standing by for a master). A role change rewrites the value of the same series, so no stale series remain. Only witness-configured nodes expose this metric. Refreshed at startup and on every backup-status change. When several witness private keys are configured (testnet setups without a witness account address), one series is written per derived address, all carrying the same value — the role is a node-level status, not a per-identity one. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:sr_is_active` (Gauge) — 1 when the locally configured witness address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:sr_votes` (Gauge) — vote count of the locally configured witness, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:sr_rank` (Gauge) — 1-based rank of the locally configured witness by vote count, 0 when no witness is configured. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:sr_is_active` (Gauge, label `address`) — 1 when the witness with this address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:sr_votes` (Gauge, label `address`) — vote count of the witness with this address, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:sr_rank` (Gauge, label `address`) — 1-based vote-count rank of the witness with this address, 0 when the witness is unknown. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) + + The `address` label carries the 58check-encoded locally configured witness address. When several witness private keys are configured (testnet setups without a witness account address), each derived address gets its own series with its own values. A node without a witness keeps the zero series under an empty `address` label. - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_already_known` (Counter) — incremented when a block response matches an outstanding adv request whose exact block ID is already known before that response is processed (best-effort signal; concurrent arrivals may be missed). ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 4772f9f64d7..b1914ca2f27 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -85,7 +85,7 @@ public void setStatus(BackupStatusEnum status) { /** * Publishes the {@code tron:node_role} gauge for every locally configured witness - * address and caches the primary witness address for maintenance-time gauge recording. + * address and caches the full address set for maintenance-time sr gauge recording. * Runs once when the bean is created — covering the window before the backup election * first runs — and again on every backup-status change. No role series is written when * no witness is configured. @@ -95,9 +95,9 @@ public void recordNodeRole() { if (!Metrics.enabled()) { return; } - byte[] witnessAddress = localWitnessAddress(); - SRMetrics.setLocalWitnessAddress(witnessAddress); - for (String address : localWitnessAddresses()) { + List addresses = localWitnessAddresses(); + SRMetrics.setLocalWitnessAddresses(addresses); + for (String address : addresses) { SRMetrics.recordNodeRole(address, currentRole()); } } @@ -132,11 +132,6 @@ private boolean isBackupConfigured() { return parameter.getBackupPort() > 0 && !parameter.getBackupMembers().isEmpty(); } - private byte[] localWitnessAddress() { - LocalWitnesses localWitnesses = Args.getLocalWitnesses(); - return localWitnesses == null ? null : localWitnesses.getWitnessAccountAddress(); - } - /** * Resolves every locally configured witness identity to a 58check-encoded address: * the witness account address when set, otherwise one address per configured private diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index a7fa1d9f807..142008d0dcc 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -182,7 +182,8 @@ public void testSrRemoveDirect() { /** * No witness configured: no node_role series is exposed at all, while the sr gauges - * still report zeros and node_info keeps its two static labels. + * keep their zero series under an empty address label and node_info keeps its two + * static labels. */ @Test public void testSrStatusFullNode() { @@ -191,9 +192,9 @@ public void testSrStatusFullNode() { backupManager.recordNodeRole(); maintenanceManager.recordWitnessMetrics(); - Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); - Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); - Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); + Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, ""), 0.0); + Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_VOTES, ""), 0.0); + Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_RANK, ""), 0.0); Assert.assertNull("no node_role series without a witness", roleGauge("")); Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); @@ -220,9 +221,9 @@ public void testSrStatusMaster() { maintenanceManager.recordWitnessMetrics(); String address = StringUtil.encode58Check(witness.toByteArray()); - Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); - Assert.assertEquals(1000.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); - Assert.assertEquals(1.0, gauge(MetricKeys.Gauge.SR_RANK), 0.0); + Assert.assertEquals(1.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address), 0.0); + Assert.assertEquals(1000.0, srGauge(MetricKeys.Gauge.SR_VOTES, address), 0.0); + Assert.assertEquals(1.0, srGauge(MetricKeys.Gauge.SR_RANK, address), 0.0); Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address), 0.0); Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); @@ -251,9 +252,9 @@ public void testSrStatusSlaver() throws Exception { maintenanceManager.recordWitnessMetrics(); String address = StringUtil.encode58Check(witness.toByteArray()); - Assert.assertEquals(0.0, gauge(MetricKeys.Gauge.SR_IS_ACTIVE), 0.0); - Assert.assertEquals(7.0, gauge(MetricKeys.Gauge.SR_VOTES), 0.0); - Assert.assertTrue(gauge(MetricKeys.Gauge.SR_RANK) > 0); + Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address), 0.0); + Assert.assertEquals(7.0, srGauge(MetricKeys.Gauge.SR_VOTES, address), 0.0); + Assert.assertTrue(srGauge(MetricKeys.Gauge.SR_RANK, address) > 0); Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address), 0.0); } @@ -333,10 +334,11 @@ public void testRoleInitAfterElectionStarted() throws Exception { /** * Testnet multi-key setup: with several witness private keys configured and no witness - * account address, node_role writes one series per derived address, all carrying the - * same node-level role value, and a status change flips every series together. Role - * recording never touches the single-identity sr_* gauges, and maintenance-time - * recording never touches the role series. + * account address, every sr gauge and the node_role gauge are written as one series + * per derived address. Role recording only touches the role series; maintenance-time + * recording gives each address its own sr values (here distinct votes, activity and + * rank) and leaves the role series alone. A status change flips every role series + * together without touching the sr series. */ @Test public void testRoleMultiWitnessKeys() throws Exception { @@ -347,30 +349,43 @@ public void testRoleMultiWitnessKeys() throws Exception { Args.setLocalWitnesses(localWitnesses); String address1 = derivedAddress(key1); String address2 = derivedAddress(key2); + ByteString witness1 = registerWitnessWithVotes( + ByteArray.fromHexString(hexAddress(key1)), 100); + ByteString witness2 = registerWitnessWithVotes( + ByteArray.fromHexString(hexAddress(key2)), 40); + chainBaseManager.getWitnessScheduleStore() + .saveActiveWitnesses(Collections.singletonList(witness1)); markBackupInitiated(backupManager); - Double activeBefore = gauge(MetricKeys.Gauge.SR_IS_ACTIVE); - Double votesBefore = gauge(MetricKeys.Gauge.SR_VOTES); - Double rankBefore = gauge(MetricKeys.Gauge.SR_RANK); - backupManager.recordNodeRole(); - Assert.assertEquals(roleGauge(address1), roleGauge(address2)); - - backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address1), 0.0); Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address2), 0.0); + // role recording never writes sr_* series + Assert.assertNull(srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address1)); + Assert.assertNull(srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address2)); - backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address1), 0.0); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address2), 0.0); - - Assert.assertEquals(activeBefore, gauge(MetricKeys.Gauge.SR_IS_ACTIVE)); - Assert.assertEquals(votesBefore, gauge(MetricKeys.Gauge.SR_VOTES)); - Assert.assertEquals(rankBefore, gauge(MetricKeys.Gauge.SR_RANK)); - + // one sr series per address, with per-address values: witness1 is active with the + // higher vote count, witness2 stands inactive on the lower rank maintenanceManager.recordWitnessMetrics(); + Assert.assertEquals(1.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address1), 0.0); + Assert.assertEquals(100.0, srGauge(MetricKeys.Gauge.SR_VOTES, address1), 0.0); + Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address2), 0.0); + Assert.assertEquals(40.0, srGauge(MetricKeys.Gauge.SR_VOTES, address2), 0.0); + // rank is absolute within the full witness store, which also holds genesis + // witnesses; the two local series must at least order by their votes + Assert.assertTrue(srGauge(MetricKeys.Gauge.SR_RANK, address1) > 0); + Assert.assertTrue(srGauge(MetricKeys.Gauge.SR_RANK, address2) > 0); + Assert.assertTrue("higher votes must give the better rank", + srGauge(MetricKeys.Gauge.SR_RANK, address1) + < srGauge(MetricKeys.Gauge.SR_RANK, address2)); + + // sr recording never touches the role series; a status change flips every + // role series together and leaves the sr series alone + backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address1), 0.0); Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address2), 0.0); + Assert.assertEquals(100.0, srGauge(MetricKeys.Gauge.SR_VOTES, address1), 0.0); + Assert.assertEquals(40.0, srGauge(MetricKeys.Gauge.SR_VOTES, address2), 0.0); } /** @@ -411,8 +426,33 @@ private ByteString registerWitnessWithVotes(long votes) { return address; } - private Double gauge(String name) { - return CollectorRegistry.defaultRegistry.getSampleValue(name); + /** + * Registers a witness for a locally configured private key's derived address so the + * multi-key test can assert distinguishable per-address sr values. + */ + private ByteString registerWitnessWithVotes(byte[] addressBytes, long votes) { + ByteString address = ByteString.copyFrom(addressBytes); + chainBaseManager.getWitnessStore().put(address.toByteArray(), + new WitnessCapsule(address, votes, "")); + chainBaseManager.getAccountStore().put(address.toByteArray(), + new AccountCapsule(Protocol.Account.newBuilder().setAddress(address).build())); + return address; + } + + /** + * Hex form of the address derived from a private key, matching what the node derives + * when no witness account address is configured. + */ + private String hexAddress(String privateKeyHex) { + return ByteArray.toHexString( + SignUtils.fromPrivate(ByteArray.fromHexString(privateKeyHex)).getAddress()); + } + + private Double srGauge(String name, String address) { + return CollectorRegistry.defaultRegistry.getSampleValue( + name, + new String[] {MetricLabels.Gauge.ADDRESS}, + new String[] {address}); } private Double info() { From 97eeaebd651852539eccae46fb19bb39d7ee3040 Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 21 Oct 2026 18:31:47 +0800 Subject: [PATCH 12/26] fix(metrics): derive per-key witness addresses for multi-key nodes localWitnessAddresses() preferred getWitnessAccountAddress(), but the production assembly path (WitnessInitializer -> LocalWitnesses.initWitnessAccountAddress) always derives it from the first private key, leaving it non-null even when multiple keys are configured. Multi-key nodes therefore published role/sr series only for the first key, while every key is its own SR identity. Branch on the private key count instead: with several keys (an explicit witness account address is rejected at startup in that case) derive one address per key; with a single key keep using the witness account address, covering multisig setups where it differs from the derived one. Add a regression test that builds LocalWitnesses through the production assembly path (WitnessInitializer.initFromCFGPrivateKey) with two keys and asserts both derived addresses get role series. Also fix a stale NODE_ROLE comment that still said one-hot after the numeric encoding change. --- .../tron/common/prometheus/MetricKeys.java | 5 +-- .../org/tron/common/backup/BackupManager.java | 35 ++++++++++++++----- .../tron/common/prometheus/SRMetricsTest.java | 21 +++++++++++ 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 8be19d48cec..8cdd1c6778c 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -46,8 +46,9 @@ public static class Gauge { public static final String SR_IS_ACTIVE = "tron:sr_is_active"; public static final String SR_VOTES = "tron:sr_votes"; public static final String SR_RANK = "tron:sr_rank"; - // one-hot role indicator of the local node in the backup election, refreshed at - // startup and on every backup-status change; witness-configured nodes only + // numeric role gauge of the local node in the backup election (0=init, 1=master, + // 2=slaver), one series per local witness address, refreshed at startup and on + // every backup-status change; witness-configured nodes only public static final String NODE_ROLE = "tron:node_role"; private Gauge() { diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index b1914ca2f27..98a1dd37d08 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -134,30 +134,47 @@ private boolean isBackupConfigured() { /** * Resolves every locally configured witness identity to a 58check-encoded address: - * the witness account address when set, otherwise one address per configured private - * key, derived the same way {@link LocalWitnesses#initWitnessAccountAddress(byte[])} - * derives the default witness account address. Returns an empty list when the node has - * no witness configured. Multi-key setups are a testnet convenience: the role is a - * node-level status, so every configured identity gets its own series carrying the - * same value. + * one address per configured private key when several keys are set, otherwise the + * witness account address (explicit, or derived from the single key). Returns an + * empty list when the node has no witness configured. Multi-key setups are a testnet + * convenience: the role is a node-level status, so every configured identity gets + * its own series carrying the same value. */ private List localWitnessAddresses() { LocalWitnesses localWitnesses = Args.getLocalWitnesses(); if (localWitnesses == null) { return Collections.emptyList(); } + List privateKeys = localWitnesses.getPrivateKeys(); + if (privateKeys.size() > 1) { + // A multi-key setup is only valid without an explicit witness account address + // (WitnessInitializer rejects the combination), so every key is its own SR + // identity. getWitnessAccountAddress() cannot detect this case: the production + // assembly path always derives it from the first key + // (LocalWitnesses.initWitnessAccountAddress), leaving it non-null. + List addresses = new ArrayList<>(privateKeys.size()); + for (String privateKey : privateKeys) { + addresses.add(deriveAddress(privateKey)); + } + return addresses; + } byte[] accountAddress = localWitnesses.getWitnessAccountAddress(); if (accountAddress != null && accountAddress.length > 0) { return Collections.singletonList(StringUtil.encode58Check(accountAddress)); } + // single key without an account address (unit-test construction) List addresses = new ArrayList<>(); - for (String privateKey : localWitnesses.getPrivateKeys()) { - SignInterface ecKey = SignUtils.fromPrivate(ByteArray.fromHexString(privateKey)); - addresses.add(StringUtil.encode58Check(ecKey.getAddress())); + for (String privateKey : privateKeys) { + addresses.add(deriveAddress(privateKey)); } return addresses; } + private static String deriveAddress(String privateKey) { + SignInterface ecKey = SignUtils.fromPrivate(ByteArray.fromHexString(privateKey)); + return StringUtil.encode58Check(ecKey.getAddress()); + } + public void init() { if (isInit) { diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 142008d0dcc..9c704074d9d 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -24,6 +24,7 @@ import org.tron.core.capsule.VotesCapsule; import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.args.Args; +import org.tron.core.config.args.WitnessInitializer; import org.tron.core.consensus.ConsensusService; import org.tron.program.Version; import org.tron.protos.Protocol; @@ -388,6 +389,26 @@ public void testRoleMultiWitnessKeys() throws Exception { Assert.assertEquals(40.0, srGauge(MetricKeys.Gauge.SR_VOTES, address2), 0.0); } + /** + * Regression for the production assembly path: multi-key nodes are built through + * WitnessInitializer.initFromCFGPrivateKey, which always derives the witness account + * address from the first key and therefore leaves it non-null. localWitnessAddresses + * must still publish one role series per key instead of stopping at that derived + * address. + */ + @Test + public void testRoleMultiWitnessKeysProductionAssembly() { + String key1 = PublicMethod.getRandomPrivateKey(); + String key2 = PublicMethod.getRandomPrivateKey(); + Args.setLocalWitnesses( + WitnessInitializer.initFromCFGPrivateKey(Arrays.asList(key1, key2), null)); + + backupManager.recordNodeRole(); + + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(derivedAddress(key1)), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(derivedAddress(key2)), 0.0); + } + /** * Backup must be seen as configured the same way BackupServer.initServer() decides to * start the election transport, otherwise the pre-election window is unreachable. From 04271d5d0bddb3b3574f7feb00610bc8c9c968de Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 22 Oct 2026 11:43:17 +0800 Subject: [PATCH 13/26] refactor(metrics): single-series node_role and miner-sourced sr addresses --- .../tron/common/prometheus/MetricLabels.java | 13 +- .../tron/common/prometheus/MetricsGauge.java | 3 +- .../org/tron/common/prometheus/SRMetrics.java | 65 ++----- .../consensus/dpos/MaintenanceManager.java | 44 ++--- docs/metrics-changelog.md | 4 +- .../org/tron/common/backup/BackupManager.java | 82 ++------ .../main/java/org/tron/program/FullNode.java | 11 ++ .../tron/common/backup/BackupManagerTest.java | 4 + .../tron/common/prometheus/SRMetricsTest.java | 179 ++++++++---------- .../prometheus/PrometheusApiServiceTest.java | 11 +- .../program/FullNodeCryptoEngineTest.java | 12 +- .../java/org/tron/program/FullNodeTest.java | 12 +- 12 files changed, 168 insertions(+), 272 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index 6d1fcc48da1..ba5ee6a81eb 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -59,19 +59,10 @@ public static class Gauge { public static final String PEERS_VALID = "valid"; - // node_role label: one gauge series per witness address, whose value encodes the - // current backup-election role as a number (0 = init, 1 = master, 2 = slaver); - // ADDRESS labels both tron:node_role and the tron:sr_* gauges + // ADDRESS is the label of the tron:sr_* gauges; tron:node_role carries no label + // (single series, the value encodes the role) public static final String ADDRESS = "address"; - // ROLE_INIT/ROLE_MASTER/ROLE_SLAVER are the canonical role names produced by - // BackupManager and mapped to numbers in SRMetrics.recordNodeRole() - public static final String ROLE_INIT = "init"; - - public static final String ROLE_MASTER = "master"; - - public static final String ROLE_SLAVER = "slaver"; - private Gauge() { throw new IllegalStateException("Gauge"); } diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index ac90359618e..bffe102eb01 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -31,8 +31,7 @@ class MetricsGauge { init(MetricKeys.Gauge.NODE_ROLE, "Role of this node in the backup election, encoded as a number: 0 = init " + "(before the first election round), 1 = master (producing blocks), " - + "2 = slaver (standing by). Witness-configured nodes only.", - MetricLabels.Gauge.ADDRESS); + + "2 = slaver (standing by)."); } private MetricsGauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 2fed1ddf1b4..afad6ff8171 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -1,11 +1,7 @@ package org.tron.common.prometheus; import com.google.protobuf.ByteString; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import org.tron.common.utils.StringUtil; public class SRMetrics { @@ -14,59 +10,24 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } - private static volatile Set localWitnessAddresses = Collections.emptySet(); + // numeric encoding of the tron:node_role gauge value: 0 = init (before the first + // election round), 1 = master (producing blocks), 2 = slaver (standing by). Read by + // BackupManager when it reports the election state; documented for operators in + // docs/metrics-changelog.md + public static final int NODE_ROLE_INIT = 0; + public static final int NODE_ROLE_MASTER = 1; + public static final int NODE_ROLE_SLAVER = 2; /** - * Caches the local witness addresses (58check-encoded) once they are resolved at - * startup, so maintenance-time recording can read them without reaching back into node - * configuration. Written once by BackupManager's {@code @PostConstruct} hook: an empty - * collection (no witness configured) is cached as an empty set, a non-empty one as an - * immutable snapshot covering the witness account address or every derived private-key - * address. + * Publishes the node-role gauge as a single label-less series, encoding the current + * role as a number (0 = init, 1 = master, 2 = slaver). A role change rewrites the + * value of the same series, so no stale series are left behind. */ - public static void setLocalWitnessAddresses(Collection addresses) { - localWitnessAddresses = addresses.isEmpty() - ? Collections.emptySet() - : Collections.unmodifiableSet(new HashSet<>(addresses)); - } - - public static Set getLocalWitnessAddresses() { - return localWitnessAddresses; - } - - // numeric encoding of the backup-election role in the tron:node_role gauge: - // 0 = init (before the first election round), 1 = master (producing blocks), - // 2 = slaver (standing by). Kept package-visible for tests; the mapping is - // documented for operators in docs/metrics-changelog.md - static final int NODE_ROLE_INIT = 0; - static final int NODE_ROLE_MASTER = 1; - static final int NODE_ROLE_SLAVER = 2; - - /** - * Publishes the node-role gauge as a single series per witness address, encoding the - * current role as a number (0 = init, 1 = master, 2 = slaver). A role change rewrites - * the value of the same series, so no stale series are left behind. A no-op when no - * witness address is given: nodes without a local witness expose no role series. - */ - public static void recordNodeRole(String address, String role) { - if (!Metrics.enabled() || address.isEmpty()) { + public static void recordNodeRole(int role) { + if (!Metrics.enabled()) { return; } - Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, roleValue(role), address); - } - - /** - * Maps a role name to its {@code tron:node_role} value; an unknown role falls back to - * init so a bad argument can never fake an elected state. - */ - private static int roleValue(String role) { - if (MetricLabels.Gauge.ROLE_MASTER.equals(role)) { - return NODE_ROLE_MASTER; - } - if (MetricLabels.Gauge.ROLE_SLAVER.equals(role)) { - return NODE_ROLE_SLAVER; - } - return NODE_ROLE_INIT; + Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, role); } /** diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index 1774e53c6fd..76c8f341fa0 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -9,7 +9,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Set; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -19,8 +18,10 @@ import org.springframework.stereotype.Component; import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; -import org.tron.common.utils.Commons; +import org.tron.common.utils.StringUtil; import org.tron.consensus.ConsensusDelegate; +import org.tron.consensus.base.Param; +import org.tron.consensus.base.Param.Miner; import org.tron.consensus.pbft.PbftManager; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.BlockCapsule; @@ -170,35 +171,32 @@ public void doMaintenance() { /** * Records the witness gauges (active-set membership, settled votes, settled rank) for - * every locally configured witness address, one series per address. With several - * private keys configured (testnet setups without a witness account address) each - * derived address gets its own series; a node without a witness keeps the zero series - * with an empty address label. Runs at the end of every maintenance and once at node - * startup. Never throws: a metrics failure must not affect maintenance or block - * processing. + * every consensus miner the node assembles from its configured witness keys, one + * series per address. A node without a witness keeps the zero series with an empty + * address label, published once at startup by the caller. Runs at the end of every + * maintenance and once at node startup. Never throws: a metrics failure must not + * affect maintenance or block processing. */ public void recordWitnessMetrics() { if (!Metrics.enabled()) { return; } - Set addresses = SRMetrics.getLocalWitnessAddresses(); - if (addresses.isEmpty()) { - SRMetrics.recordSrStatus("", false, 0, 0); + List miners = Param.getInstance().getMiners(); + if (miners == null) { + // consensus never started: nothing to publish (never the case on a running node) return; } - for (String address : addresses) { + for (Miner miner : miners) { + ByteString witnessAddressBs = miner.getWitnessAddress(); + if (witnessAddressBs == null) { + continue; + } + String address = StringUtil.encode58Check(witnessAddressBs.toByteArray()); try { - boolean active = false; - long votes = 0; - long rank = 0; - byte[] witnessAddress = Commons.decode58Check(address); - if (witnessAddress != null && witnessAddress.length > 0) { - ByteString witnessAddressBs = ByteString.copyFrom(witnessAddress); - active = consensusDelegate.getActiveWitnesses().contains(witnessAddressBs); - WitnessCapsule witness = consensusDelegate.getWitness(witnessAddress); - votes = witness == null ? 0 : witness.getVoteCount(); - rank = witnessRank(witnessAddressBs); - } + boolean active = consensusDelegate.getActiveWitnesses().contains(witnessAddressBs); + WitnessCapsule witness = consensusDelegate.getWitness(witnessAddressBs.toByteArray()); + long votes = witness == null ? 0 : witness.getVoteCount(); + long rank = witnessRank(witnessAddressBs); SRMetrics.recordSrStatus(address, active, votes, rank); } catch (Exception e) { logger.warn("Failed to record witness metrics for {}", address, e); diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 7e34a12e473..bda73b4a759 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -10,12 +10,12 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:node_role` (Gauge, label `address`) — role of this node in the backup election, encoded as a number: `0` = `init` (from node startup until the backup election first runs), `1` = `master` (producing blocks; always the value when backup is not configured), `2` = `slaver` (standing by for a master). A role change rewrites the value of the same series, so no stale series remain. Only witness-configured nodes expose this metric. Refreshed at startup and on every backup-status change. When several witness private keys are configured (testnet setups without a witness account address), one series is written per derived address, all carrying the same value — the role is a node-level status, not a per-identity one. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_role` (Gauge, no labels) — role of this node in the backup election, encoded as a number: `0` = `init` (from node startup until the backup election first runs), `1` = `master` (producing blocks; also the value when backup is not configured), `2` = `slaver` (standing by for a master). The role is a node-level status, not a per-identity one, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_is_active` (Gauge, label `address`) — 1 when the witness with this address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_votes` (Gauge, label `address`) — vote count of the witness with this address, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_rank` (Gauge, label `address`) — 1-based vote-count rank of the witness with this address, 0 when the witness is unknown. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - The `address` label carries the 58check-encoded locally configured witness address. When several witness private keys are configured (testnet setups without a witness account address), each derived address gets its own series with its own values. A node without a witness keeps the zero series under an empty `address` label. + The `address` label carries the 58check-encoded address of one of the node's consensus witness miners. The miners are assembled once at startup: one miner per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. When several witness private keys are configured, each key address gets its own series with its own values. A node without a witness keeps the zero series under an empty `address` label. - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_already_known` (Counter) — incremented when a block response matches an outstanding adv request whose exact block ID is already known before that response is processed (best-effort signal; concurrent arrivals may be missed). ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 98a1dd37d08..219dc28bf17 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -9,15 +9,11 @@ import io.netty.util.internal.ConcurrentSet; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import javax.annotation.PostConstruct; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -27,17 +23,10 @@ import org.tron.common.backup.socket.EventHandler; import org.tron.common.backup.socket.MessageHandler; import org.tron.common.backup.socket.UdpEvent; -import org.tron.common.crypto.SignInterface; -import org.tron.common.crypto.SignUtils; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; -import org.tron.common.prometheus.MetricLabels; import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.LocalWitnesses; -import org.tron.common.utils.StringUtil; -import org.tron.core.config.args.Args; import org.tron.p2p.utils.NetUtil; @Slf4j(topic = "backup") @@ -84,45 +73,39 @@ public void setStatus(BackupStatusEnum status) { } /** - * Publishes the {@code tron:node_role} gauge for every locally configured witness - * address and caches the full address set for maintenance-time sr gauge recording. - * Runs once when the bean is created — covering the window before the backup election - * first runs — and again on every backup-status change. No role series is written when - * no witness is configured. + * Publishes the {@code tron:node_role} gauge — a single label-less series encoding + * the current backup-election role as a number (0 = init, 1 = master, 2 = slaver). + * Runs once after the application has started — covering the window before the backup + * election first runs — and again on every backup-status change. */ - @PostConstruct public void recordNodeRole() { if (!Metrics.enabled()) { return; } - List addresses = localWitnessAddresses(); - SRMetrics.setLocalWitnessAddresses(addresses); - for (String address : addresses) { - SRMetrics.recordNodeRole(address, currentRole()); - } + SRMetrics.recordNodeRole(currentRole()); } - private String currentRole() { + private int currentRole() { if (!isBackupConfigured()) { // no backup election to take part in: this node produces blocks itself - return MetricLabels.Gauge.ROLE_MASTER; + return SRMetrics.NODE_ROLE_MASTER; } if (!isInit) { // the backup server only starts after the first fresh block, so between startup // and the first election round the status default (MASTER) is not a real result; // report init instead of pretending the election chose a master - return MetricLabels.Gauge.ROLE_INIT; + return SRMetrics.NODE_ROLE_INIT; } // post-init the status is a real election outcome, so map every state 1:1; the // pre-init window is already gated by the isInit check above, so INIT here only // means the election transport is up but no keep-alive timeout has been judged yet switch (status) { case MASTER: - return MetricLabels.Gauge.ROLE_MASTER; + return SRMetrics.NODE_ROLE_MASTER; case SLAVER: - return MetricLabels.Gauge.ROLE_SLAVER; + return SRMetrics.NODE_ROLE_SLAVER; default: - return MetricLabels.Gauge.ROLE_INIT; + return SRMetrics.NODE_ROLE_INIT; } } @@ -132,49 +115,6 @@ private boolean isBackupConfigured() { return parameter.getBackupPort() > 0 && !parameter.getBackupMembers().isEmpty(); } - /** - * Resolves every locally configured witness identity to a 58check-encoded address: - * one address per configured private key when several keys are set, otherwise the - * witness account address (explicit, or derived from the single key). Returns an - * empty list when the node has no witness configured. Multi-key setups are a testnet - * convenience: the role is a node-level status, so every configured identity gets - * its own series carrying the same value. - */ - private List localWitnessAddresses() { - LocalWitnesses localWitnesses = Args.getLocalWitnesses(); - if (localWitnesses == null) { - return Collections.emptyList(); - } - List privateKeys = localWitnesses.getPrivateKeys(); - if (privateKeys.size() > 1) { - // A multi-key setup is only valid without an explicit witness account address - // (WitnessInitializer rejects the combination), so every key is its own SR - // identity. getWitnessAccountAddress() cannot detect this case: the production - // assembly path always derives it from the first key - // (LocalWitnesses.initWitnessAccountAddress), leaving it non-null. - List addresses = new ArrayList<>(privateKeys.size()); - for (String privateKey : privateKeys) { - addresses.add(deriveAddress(privateKey)); - } - return addresses; - } - byte[] accountAddress = localWitnesses.getWitnessAccountAddress(); - if (accountAddress != null && accountAddress.length > 0) { - return Collections.singletonList(StringUtil.encode58Check(accountAddress)); - } - // single key without an account address (unit-test construction) - List addresses = new ArrayList<>(); - for (String privateKey : privateKeys) { - addresses.add(deriveAddress(privateKey)); - } - return addresses; - } - - private static String deriveAddress(String privateKey) { - SignInterface ecKey = SignUtils.fromPrivate(ByteArray.fromHexString(privateKey)); - return StringUtil.encode58Check(ecKey.getAddress()); - } - public void init() { if (isInit) { diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 7655d148b11..01ea9f8d681 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -9,11 +9,14 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.arch.Arch; +import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; +import org.tron.consensus.base.Param; import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; @@ -76,7 +79,15 @@ public static void main(String[] args) { // (Manager.initGenesis) Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); + // the sr snapshot and the initial node_role series read the consensus miners, which + // are assembled in ApplicationImpl.startup() via ConsensusService.start(), so they + // are available here; an empty miner list means a fullnode, whose zero sr series is + // published once at startup — gauge values persist until rewritten + if (Param.getInstance().getMiners().isEmpty()) { + SRMetrics.recordSrStatus("", false, 0, 0); + } context.getBean(MaintenanceManager.class).recordWitnessMetrics(); + context.getBean(BackupManager.class).recordNodeRole(); if (parameter.isSolidityNode()) { SolidityNode node = context.getBean(SolidityNode.class); node.run(); diff --git a/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java b/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java index 0c754a0ac6d..cd2a18ac594 100644 --- a/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java +++ b/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java @@ -28,6 +28,7 @@ import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.ReflectUtils; +import org.tron.consensus.base.Param; import org.tron.core.config.args.Args; import org.tron.core.config.args.InetUtil; @@ -76,6 +77,9 @@ public void tearDown() throws Exception { }); InetUtil.dnsLookup = previousDnsLookup; Args.clearParam(); + // the consensus miner list is a process-wide singleton: reset it so this class + // cannot leak state into other tests of the same JVM + Param.getInstance().setMiners(new ArrayList<>()); BackupTestUtils.throwIfAnyError(errors); } diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 9c704074d9d..071ca8a03dd 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -2,8 +2,10 @@ import com.google.protobuf.ByteString; import io.prometheus.client.CollectorRegistry; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -16,15 +18,14 @@ import org.tron.common.backup.BackupManager; import org.tron.common.crypto.SignUtils; import org.tron.common.utils.ByteArray; -import org.tron.common.utils.LocalWitnesses; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.StringUtil; +import org.tron.consensus.base.Param; import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.VotesCapsule; import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.args.Args; -import org.tron.core.config.args.WitnessInitializer; import org.tron.core.consensus.ConsensusService; import org.tron.program.Version; import org.tron.protos.Protocol; @@ -62,7 +63,7 @@ public void tearDown() { Args.getInstance().setMetricsPrometheusEnable(true); Args.getInstance().setBackupPort(0); Args.getInstance().setBackupMembers(Collections.emptyList()); - Args.setLocalWitnesses(new LocalWitnesses()); + Param.getInstance().setMiners(new ArrayList<>()); backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); } @@ -182,21 +183,29 @@ public void testSrRemoveDirect() { } /** - * No witness configured: no node_role series is exposed at all, while the sr gauges - * keep their zero series under an empty address label and node_info keeps its two - * static labels. + * No witness configured: with an empty miner list, recordWitnessMetrics() writes + * nothing and FullNode publishes the node-level zero sr series once at startup (the + * direct recordSrStatus call below — gauge values persist until rewritten); node_role + * is a single label-less series — this node takes no part in a backup election, so it + * reports master (1) — while node_info keeps its two static labels. */ @Test public void testSrStatusFullNode() { - Args.setLocalWitnesses(new LocalWitnesses()); + Param.getInstance().setMiners(new ArrayList<>()); - backupManager.recordNodeRole(); maintenanceManager.recordWitnessMetrics(); + Assert.assertNull("empty miner list must not write an empty-address sr series", + srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, "")); + + backupManager.recordNodeRole(); + Assert.assertEquals("node_role is one label-less series", + SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + // the startup zero series, exactly what FullNode publishes when miners are empty + SRMetrics.recordSrStatus("", false, 0, 0); Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, ""), 0.0); Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_VOTES, ""), 0.0); Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_RANK, ""), 0.0); - Assert.assertNull("no node_role series without a witness", roleGauge("")); Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); Assert.assertEquals(1.0, info(), 0.0); @@ -204,8 +213,8 @@ public void testSrStatusFullNode() { /** * Witness configured without a backup election (backup members unset): the node always - * produces blocks, so the role gauge reports master (1); node_info stays a static - * two-label identity record. + * produces blocks, so the single role series reports master (1); node_info stays a + * static two-label identity record. */ @Test public void testSrStatusMaster() { @@ -214,9 +223,7 @@ public void testSrStatusMaster() { chainBaseManager.getWitnessScheduleStore() .saveActiveWitnesses(Collections.singletonList(witness)); - LocalWitnesses localWitnesses = new LocalWitnesses(); - localWitnesses.initWitnessAccountAddress(witness.toByteArray()); - Args.setLocalWitnesses(localWitnesses); + seedMiners(witness); backupManager.recordNodeRole(); maintenanceManager.recordWitnessMetrics(); @@ -225,15 +232,15 @@ public void testSrStatusMaster() { Assert.assertEquals(1.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address), 0.0); Assert.assertEquals(1000.0, srGauge(MetricKeys.Gauge.SR_VOTES, address), 0.0); Assert.assertEquals(1.0, srGauge(MetricKeys.Gauge.SR_RANK, address), 0.0); - Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); Assert.assertEquals(1.0, info(), 0.0); } /** - * Backup election slaver: the role gauge reports slaver (2) while is_active reflects - * the active witness set. + * Backup election slaver: the single role series reports slaver (2) while is_active + * reflects the active witness set. */ @Test public void testSrStatusSlaver() throws Exception { @@ -244,9 +251,7 @@ public void testSrStatusSlaver() throws Exception { chainBaseManager.getWitnessScheduleStore() .saveActiveWitnesses(Collections.singletonList(activeWitness)); - LocalWitnesses localWitnesses = new LocalWitnesses(); - localWitnesses.initWitnessAccountAddress(witness.toByteArray()); - Args.setLocalWitnesses(localWitnesses); + seedMiners(witness); markBackupInitiated(backupManager); backupManager.recordNodeRole(); @@ -256,62 +261,53 @@ public void testSrStatusSlaver() throws Exception { Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address), 0.0); Assert.assertEquals(7.0, srGauge(MetricKeys.Gauge.SR_VOTES, address), 0.0); Assert.assertTrue(srGauge(MetricKeys.Gauge.SR_RANK, address) > 0); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); } /** * Before the first election round (backup configured, BackupManager.init() not yet run, * which in production only happens after the first fresh block) the node must report * init (0) instead of falling back to the MASTER field default; the first real status - * after init() rewrites the same series to the elected role. + * after init() rewrites the same single series to the elected role. */ @Test public void testRoleInitBeforeBackupElectionRuns() throws Exception { configureBackup(); - ByteString witness = registerWitnessWithVotes(1); - LocalWitnesses localWitnesses = new LocalWitnesses(); - localWitnesses.initWitnessAccountAddress(witness.toByteArray()); - Args.setLocalWitnesses(localWitnesses); - String address = StringUtil.encode58Check(witness.toByteArray()); // a directly constructed manager has not gone through init(): pre-election window BackupManager fresh = new BackupManager(); try { fresh.recordNodeRole(); - Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(address), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); // a status change while init() has not run stays inside the window fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(address), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); // once the election has run, the status maps to the elected role markBackupInitiated(fresh); fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); } finally { fresh.stop(); } } /** - * setStatus() itself republishes the node_role gauge without touching any store, so a - * backup-election transition is reflected immediately instead of at the next maintenance. + * setStatus() itself republishes the single label-less node_role series without + * touching any store, so a backup-election transition is reflected immediately instead + * of at the next maintenance. */ @Test public void testRoleRefreshOnBackupStatusChange() throws Exception { configureBackup(); - ByteString witness = registerWitnessWithVotes(9); - LocalWitnesses localWitnesses = new LocalWitnesses(); - localWitnesses.initWitnessAccountAddress(witness.toByteArray()); - Args.setLocalWitnesses(localWitnesses); markBackupInitiated(backupManager); - String address = StringUtil.encode58Check(witness.toByteArray()); backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); - Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); } /** @@ -322,45 +318,38 @@ public void testRoleRefreshOnBackupStatusChange() throws Exception { @Test public void testRoleInitAfterElectionStarted() throws Exception { configureBackup(); - ByteString witness = registerWitnessWithVotes(3); - LocalWitnesses localWitnesses = new LocalWitnesses(); - localWitnesses.initWitnessAccountAddress(witness.toByteArray()); - Args.setLocalWitnesses(localWitnesses); markBackupInitiated(backupManager); - String address = StringUtil.encode58Check(witness.toByteArray()); backupManager.setStatus(BackupManager.BackupStatusEnum.INIT); - Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(address), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); } /** - * Testnet multi-key setup: with several witness private keys configured and no witness - * account address, every sr gauge and the node_role gauge are written as one series - * per derived address. Role recording only touches the role series; maintenance-time - * recording gives each address its own sr values (here distinct votes, activity and - * rank) and leaves the role series alone. A status change flips every role series - * together without touching the sr series. + * Testnet multi-key setup: the consensus layer assembles one miner per configured + * witness private key, so every sr gauge is written as one series per key address while + * node_role stays a single series regardless of the key count. Role recording only + * touches the role series; maintenance-time recording gives each address its own sr + * values (here distinct votes, activity and rank) and leaves the role series alone. A + * status change rewrites the single role series without touching the sr series. */ @Test public void testRoleMultiWitnessKeys() throws Exception { configureBackup(); String key1 = PublicMethod.getRandomPrivateKey(); String key2 = PublicMethod.getRandomPrivateKey(); - LocalWitnesses localWitnesses = new LocalWitnesses(Arrays.asList(key1, key2)); - Args.setLocalWitnesses(localWitnesses); - String address1 = derivedAddress(key1); - String address2 = derivedAddress(key2); - ByteString witness1 = registerWitnessWithVotes( - ByteArray.fromHexString(hexAddress(key1)), 100); - ByteString witness2 = registerWitnessWithVotes( - ByteArray.fromHexString(hexAddress(key2)), 40); + ByteString miner1 = addressOf(key1); + ByteString miner2 = addressOf(key2); + seedMiners(miner1, miner2); + String address1 = StringUtil.encode58Check(miner1.toByteArray()); + String address2 = StringUtil.encode58Check(miner2.toByteArray()); + ByteString witness1 = registerWitnessWithVotes(miner1.toByteArray(), 100); + ByteString witness2 = registerWitnessWithVotes(miner2.toByteArray(), 40); chainBaseManager.getWitnessScheduleStore() .saveActiveWitnesses(Collections.singletonList(witness1)); markBackupInitiated(backupManager); backupManager.recordNodeRole(); - Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address1), 0.0); - Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(address2), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); // role recording never writes sr_* series Assert.assertNull(srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address1)); Assert.assertNull(srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address2)); @@ -380,35 +369,14 @@ public void testRoleMultiWitnessKeys() throws Exception { srGauge(MetricKeys.Gauge.SR_RANK, address1) < srGauge(MetricKeys.Gauge.SR_RANK, address2)); - // sr recording never touches the role series; a status change flips every - // role series together and leaves the sr series alone + // sr recording never touches the role series; a status change flips the single + // role series and leaves the sr series alone backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address1), 0.0); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(address2), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); Assert.assertEquals(100.0, srGauge(MetricKeys.Gauge.SR_VOTES, address1), 0.0); Assert.assertEquals(40.0, srGauge(MetricKeys.Gauge.SR_VOTES, address2), 0.0); } - /** - * Regression for the production assembly path: multi-key nodes are built through - * WitnessInitializer.initFromCFGPrivateKey, which always derives the witness account - * address from the first key and therefore leaves it non-null. localWitnessAddresses - * must still publish one role series per key instead of stopping at that derived - * address. - */ - @Test - public void testRoleMultiWitnessKeysProductionAssembly() { - String key1 = PublicMethod.getRandomPrivateKey(); - String key2 = PublicMethod.getRandomPrivateKey(); - Args.setLocalWitnesses( - WitnessInitializer.initFromCFGPrivateKey(Arrays.asList(key1, key2), null)); - - backupManager.recordNodeRole(); - - Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(derivedAddress(key1)), 0.0); - Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(derivedAddress(key2)), 0.0); - } - /** * Backup must be seen as configured the same way BackupServer.initServer() decides to * start the election transport, otherwise the pre-election window is unreachable. @@ -430,14 +398,28 @@ private void markBackupInitiated(BackupManager manager) throws Exception { } /** - * Derives the 58check address the same way BackupManager.localWitnessAddresses() - * derives one from a configured private key. + * Builds the consensus-miner identity the node assembles from a configured private + * key: the address derived from that key. */ - private String derivedAddress(String privateKeyHex) { - return StringUtil.encode58Check( + private ByteString addressOf(String privateKeyHex) { + return ByteString.copyFrom( SignUtils.fromPrivate(ByteArray.fromHexString(privateKeyHex)).getAddress()); } + /** + * Seeds the consensus miners the metrics path reads its identities from, one miner + * per address. The private key bytes are never read by the metrics path; tearDown + * clears the miner list again. + */ + private void seedMiners(ByteString... addresses) { + Param param = Param.getInstance(); + List miners = new ArrayList<>(); + for (ByteString address : addresses) { + miners.add(param.new Miner(new byte[0], address, address)); + } + param.setMiners(miners); + } + private ByteString registerWitnessWithVotes(long votes) { ByteString address = uniqueAddress(); chainBaseManager.getWitnessStore().put(address.toByteArray(), @@ -448,8 +430,8 @@ private ByteString registerWitnessWithVotes(long votes) { } /** - * Registers a witness for a locally configured private key's derived address so the - * multi-key test can assert distinguishable per-address sr values. + * Registers a witness for a locally configured miner's address so the multi-key test + * can assert distinguishable per-address sr values. */ private ByteString registerWitnessWithVotes(byte[] addressBytes, long votes) { ByteString address = ByteString.copyFrom(addressBytes); @@ -460,15 +442,6 @@ private ByteString registerWitnessWithVotes(byte[] addressBytes, long votes) { return address; } - /** - * Hex form of the address derived from a private key, matching what the node derives - * when no witness account address is configured. - */ - private String hexAddress(String privateKeyHex) { - return ByteArray.toHexString( - SignUtils.fromPrivate(ByteArray.fromHexString(privateKeyHex)).getAddress()); - } - private Double srGauge(String name, String address) { return CollectorRegistry.defaultRegistry.getSampleValue( name, @@ -483,11 +456,11 @@ private Double info() { new String[] {Version.getVersion(), Args.getInstance().getChainId()}); } - private Double roleGauge(String address) { + private Double roleGauge() { return CollectorRegistry.defaultRegistry.getSampleValue( MetricKeys.Gauge.NODE_ROLE, - new String[] {MetricLabels.Gauge.ADDRESS}, - new String[] {address}); + new String[] {}, + new String[] {}); } private ByteString registerWitness() { diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index 489c6e1a707..4ad3b7af377 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -227,14 +227,13 @@ public void testNodeInfoMetric() { @Test public void testNodeRoleMetric() { - String address = "TLyqz8V6CR3s9nNote9u1NoxKZ8NqBSbn1"; - SRMetrics.recordNodeRole(address, MetricLabels.Gauge.ROLE_MASTER); - // a single series per node; the value encodes the role (0 = init, 1 = master, - // 2 = slaver), here master after recording ROLE_MASTER + // a single label-less series per node; the value encodes the role (0 = init, + // 1 = master, 2 = slaver), here master after recording NODE_ROLE_MASTER + SRMetrics.recordNodeRole(SRMetrics.NODE_ROLE_MASTER); Double value = CollectorRegistry.defaultRegistry.getSampleValue( MetricKeys.Gauge.NODE_ROLE, - new String[] {MetricLabels.Gauge.ADDRESS}, - new String[] {address}); + new String[] {}, + new String[] {}); Assert.assertNotNull("tron:node_role sample should exist", value); Assert.assertEquals(1.0, value, 0.0); } diff --git a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java index 1919afd3488..f144496559e 100644 --- a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java @@ -17,6 +17,7 @@ import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.ArrayList; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -28,9 +29,11 @@ import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; +import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.prometheus.Metrics; +import org.tron.consensus.base.Param; import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.args.Args; import org.tron.core.config.args.WitnessInitializer; @@ -94,6 +97,11 @@ private void assertStartup(String cryptoConfig) throws Exception { File output = new File(temporaryFolder.newFolder(), "chain"); Application application = mock(Application.class); MaintenanceManager maintenanceManager = mock(MaintenanceManager.class); + BackupManager backupManager = mock(BackupManager.class); + // the application startup is mocked, so ConsensusService.start() never fills the + // consensus miners; FullNode reads them right after startup to publish the fullnode + // zero series, so hand it the empty miner list of a witness-less node + Param.getInstance().setMiners(new ArrayList<>()); // Exercise the real entry point and config loader, while isolating services and chain stores. try (MockedStatic exit = mockStatic(ExitManager.class); MockedStatic logs = mockStatic(LogService.class); @@ -102,14 +110,16 @@ private void assertStartup(String cryptoConfig) throws Exception { MockedConstruction contexts = mockConstruction(TronApplicationContext.class, (context, constructionContext) -> { when(context.getBean(MaintenanceManager.class)).thenReturn(maintenanceManager); + when(context.getBean(BackupManager.class)).thenReturn(backupManager); })) { factory.when(() -> ApplicationFactory.create(any())).thenReturn(application); FullNode.main(new String[]{"-c", config.toString(), "-d", output.toString()}); assertEquals(1, contexts.constructed().size()); verify(contexts.constructed().get(0)).refresh(); - InOrder inOrder = inOrder(application, maintenanceManager); + InOrder inOrder = inOrder(application, maintenanceManager, backupManager); inOrder.verify(application).startup(); inOrder.verify(maintenanceManager).recordWitnessMetrics(); + inOrder.verify(backupManager).recordNodeRole(); inOrder.verify(application).blockUntilShutdown(); } } diff --git a/framework/src/test/java/org/tron/program/FullNodeTest.java b/framework/src/test/java/org/tron/program/FullNodeTest.java index 6d89bdb76c3..19882c9e0ab 100644 --- a/framework/src/test/java/org/tron/program/FullNodeTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeTest.java @@ -31,10 +31,12 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.arch.Arch; +import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.Metrics; +import org.tron.consensus.base.Param; import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.args.Args; import org.tron.core.config.args.CommandLineArguments; @@ -73,6 +75,11 @@ public void testNodeModeRetainsInitialization() { parameter.logbackPath = "node-logback.xml"; Application application = Mockito.mock(Application.class); MaintenanceManager maintenanceManager = Mockito.mock(MaintenanceManager.class); + BackupManager backupManager = Mockito.mock(BackupManager.class); + // the application startup is mocked, so ConsensusService.start() never fills the + // consensus miners; FullNode reads them right after startup to publish the fullnode + // zero series, so hand it the empty miner list of a witness-less node + Param.getInstance().setMiners(new ArrayList<>()); try (MockedStatic exitManager = Mockito.mockStatic(ExitManager.class); MockedStatic arch = Mockito.mockStatic(Arch.class); MockedStatic args = Mockito.mockStatic(Args.class); @@ -85,6 +92,8 @@ public void testNodeModeRetainsInitialization() { (context, constructionContext) -> { Mockito.when(context.getBean(MaintenanceManager.class)) .thenReturn(maintenanceManager); + Mockito.when(context.getBean(BackupManager.class)) + .thenReturn(backupManager); }); MockedConstruction ipcClients = Mockito.mockConstruction(IpcClient.class)) { parameters.when(CommonParameter::getInstance).thenReturn(parameter); @@ -101,9 +110,10 @@ public void testNodeModeRetainsInitialization() { metrics.verify(Metrics::init); Assert.assertEquals(1, contexts.constructed().size()); Mockito.verify(contexts.constructed().get(0)).refresh(); - InOrder order = Mockito.inOrder(application, maintenanceManager); + InOrder order = Mockito.inOrder(application, maintenanceManager, backupManager); order.verify(application).startup(); order.verify(maintenanceManager).recordWitnessMetrics(); + order.verify(backupManager).recordNodeRole(); order.verify(application).blockUntilShutdown(); Assert.assertTrue(ipcClients.constructed().isEmpty()); } From 16f15a2172791bac34e6d6f01c805b5bc60f9aa5 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 22 Oct 2026 12:51:15 +0800 Subject: [PATCH 14/26] refactor(metrics): four-state node_role and fullnode semantics --- .../org/tron/common/prometheus/SRMetrics.java | 18 ++++---- docs/metrics-changelog.md | 4 +- .../org/tron/common/backup/BackupManager.java | 17 ++++--- .../main/java/org/tron/program/FullNode.java | 8 +--- .../tron/common/prometheus/SRMetricsTest.java | 44 ++++++++++--------- .../prometheus/PrometheusApiServiceTest.java | 6 +-- 6 files changed, 51 insertions(+), 46 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index afad6ff8171..a6942fdfb04 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -10,18 +10,20 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } - // numeric encoding of the tron:node_role gauge value: 0 = init (before the first - // election round), 1 = master (producing blocks), 2 = slaver (standing by). Read by - // BackupManager when it reports the election state; documented for operators in + // numeric encoding of the tron:node_role gauge value: 0 = fullnode (started without + // witness mode), 1 = init (before the first election round, or during a re-election + // after the master lost keep-alive), 2 = master, 3 = slaver. Read by BackupManager + // when it reports the election state; documented for operators in // docs/metrics-changelog.md - public static final int NODE_ROLE_INIT = 0; - public static final int NODE_ROLE_MASTER = 1; - public static final int NODE_ROLE_SLAVER = 2; + public static final int NODE_ROLE_FULLNODE = 0; + public static final int NODE_ROLE_INIT = 1; + public static final int NODE_ROLE_MASTER = 2; + public static final int NODE_ROLE_SLAVER = 3; /** * Publishes the node-role gauge as a single label-less series, encoding the current - * role as a number (0 = init, 1 = master, 2 = slaver). A role change rewrites the - * value of the same series, so no stale series are left behind. + * role as a number (0 = fullnode, 1 = init, 2 = master, 3 = slaver). A role change + * rewrites the value of the same series, so no stale series are left behind. */ public static void recordNodeRole(int role) { if (!Metrics.enabled()) { diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index bda73b4a759..08da47952e7 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -10,12 +10,12 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:node_role` (Gauge, no labels) — role of this node in the backup election, encoded as a number: `0` = `init` (from node startup until the backup election first runs), `1` = `master` (producing blocks; also the value when backup is not configured), `2` = `slaver` (standing by for a master). The role is a node-level status, not a per-identity one, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_role` (Gauge, no labels) — role of this node, encoded as a number: `0` = `fullnode` (started without witness mode; witness keys, if any, are not loaded on such a node and no sr_* series are published), `1` = `init` (from node startup until the backup election first runs, and again while a re-election is running after the master lost keep-alive), `2` = `master` (holds the primary witness key; also the value when backup is not configured), `3` = `slaver` (standing by for a master). The role is a node-level status, not a per-identity one, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_is_active` (Gauge, label `address`) — 1 when the witness with this address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_votes` (Gauge, label `address`) — vote count of the witness with this address, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:sr_rank` (Gauge, label `address`) — 1-based vote-count rank of the witness with this address, 0 when the witness is unknown. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - The `address` label carries the 58check-encoded address of one of the node's consensus witness miners. The miners are assembled once at startup: one miner per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. When several witness private keys are configured, each key address gets its own series with its own values. A node without a witness keeps the zero series under an empty `address` label. + The `address` label carries the 58check-encoded address of one of the node's consensus witness miners. The miners are assembled once at startup: one miner per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. When several witness private keys are configured, each key address gets its own series with its own values. A node without a witness publishes no `sr_*` series at all. - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_already_known` (Counter) — incremented when a block response matches an outstanding adv request whose exact block ID is already known before that response is processed (best-effort signal; concurrent arrivals may be missed). ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 219dc28bf17..40525f417d3 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -27,6 +27,7 @@ import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; +import org.tron.core.config.args.Args; import org.tron.p2p.utils.NetUtil; @Slf4j(topic = "backup") @@ -74,7 +75,7 @@ public void setStatus(BackupStatusEnum status) { /** * Publishes the {@code tron:node_role} gauge — a single label-less series encoding - * the current backup-election role as a number (0 = init, 1 = master, 2 = slaver). + * the current role as a number (0 = fullnode, 1 = init, 2 = master, 3 = slaver). * Runs once after the application has started — covering the window before the backup * election first runs — and again on every backup-status change. */ @@ -86,8 +87,14 @@ public void recordNodeRole() { } private int currentRole() { + if (!Args.getInstance().isWitness()) { + // started without witness mode: a fullnode plays no role in any election, even + // when witness keys are configured (the keys only feed the sr_* address series) + return SRMetrics.NODE_ROLE_FULLNODE; + } if (!isBackupConfigured()) { - // no backup election to take part in: this node produces blocks itself + // no backup election to take part in: this node holds the primary witness key + // itself return SRMetrics.NODE_ROLE_MASTER; } if (!isInit) { @@ -96,9 +103,9 @@ private int currentRole() { // report init instead of pretending the election chose a master return SRMetrics.NODE_ROLE_INIT; } - // post-init the status is a real election outcome, so map every state 1:1; the - // pre-init window is already gated by the isInit check above, so INIT here only - // means the election transport is up but no keep-alive timeout has been judged yet + // post-init the status is a real election outcome, so map every state 1:1; INIT + // here covers both the window where the transport is up but no keep-alive timeout + // has been judged yet and the re-election entered after the master lost keep-alive switch (status) { case MASTER: return SRMetrics.NODE_ROLE_MASTER; diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 01ea9f8d681..9ed062371d8 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -15,8 +15,6 @@ import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.Metrics; -import org.tron.common.prometheus.SRMetrics; -import org.tron.consensus.base.Param; import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; @@ -81,11 +79,7 @@ public static void main(String[] args) { Args.getInstance().getChainId()); // the sr snapshot and the initial node_role series read the consensus miners, which // are assembled in ApplicationImpl.startup() via ConsensusService.start(), so they - // are available here; an empty miner list means a fullnode, whose zero sr series is - // published once at startup — gauge values persist until rewritten - if (Param.getInstance().getMiners().isEmpty()) { - SRMetrics.recordSrStatus("", false, 0, 0); - } + // are available here context.getBean(MaintenanceManager.class).recordWitnessMetrics(); context.getBean(BackupManager.class).recordNodeRole(); if (parameter.isSolidityNode()) { diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 071ca8a03dd..4ace2802800 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -63,6 +63,7 @@ public void tearDown() { Args.getInstance().setMetricsPrometheusEnable(true); Args.getInstance().setBackupPort(0); Args.getInstance().setBackupMembers(Collections.emptyList()); + Args.getInstance().witness = false; Param.getInstance().setMiners(new ArrayList<>()); backupManager.setStatus(BackupManager.BackupStatusEnum.MASTER); } @@ -183,14 +184,13 @@ public void testSrRemoveDirect() { } /** - * No witness configured: with an empty miner list, recordWitnessMetrics() writes - * nothing and FullNode publishes the node-level zero sr series once at startup (the - * direct recordSrStatus call below — gauge values persist until rewritten); node_role - * is a single label-less series — this node takes no part in a backup election, so it - * reports master (1) — while node_info keeps its two static labels. + * No witness configured: with an empty miner list a plain fullnode publishes none of + * the sr_* series, and node_role reports 0 because the node started without witness + * mode, while node_info keeps its two static labels. */ @Test public void testSrStatusFullNode() { + Args.getInstance().witness = false; Param.getInstance().setMiners(new ArrayList<>()); maintenanceManager.recordWitnessMetrics(); @@ -198,26 +198,22 @@ public void testSrStatusFullNode() { srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, "")); backupManager.recordNodeRole(); - Assert.assertEquals("node_role is one label-less series", - SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); - - // the startup zero series, exactly what FullNode publishes when miners are empty - SRMetrics.recordSrStatus("", false, 0, 0); - Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, ""), 0.0); - Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_VOTES, ""), 0.0); - Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_RANK, ""), 0.0); + Assert.assertEquals("a node started without witness mode reports role 0", + SRMetrics.NODE_ROLE_FULLNODE, roleGauge(), 0.0); + Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); Assert.assertEquals(1.0, info(), 0.0); } /** - * Witness configured without a backup election (backup members unset): the node always - * produces blocks, so the single role series reports master (1); node_info stays a - * static two-label identity record. + * Witness configured without a backup election (backup members unset): the node + * always holds the primary key itself, so the single role series reports master (2); + * node_info stays a static two-label identity record. */ @Test public void testSrStatusMaster() { + Args.getInstance().witness = true; ByteString witness = registerWitnessWithVotes(1000); registerWitnessWithVotes(50); chainBaseManager.getWitnessScheduleStore() @@ -239,11 +235,12 @@ public void testSrStatusMaster() { } /** - * Backup election slaver: the single role series reports slaver (2) while is_active + * Backup election slaver: the single role series reports slaver (3) while is_active * reflects the active witness set. */ @Test public void testSrStatusSlaver() throws Exception { + Args.getInstance().witness = true; configureBackup(); backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); ByteString activeWitness = registerWitnessWithVotes(10); @@ -267,11 +264,12 @@ public void testSrStatusSlaver() throws Exception { /** * Before the first election round (backup configured, BackupManager.init() not yet run, * which in production only happens after the first fresh block) the node must report - * init (0) instead of falling back to the MASTER field default; the first real status + * init (1) instead of falling back to the MASTER field default; the first real status * after init() rewrites the same single series to the elected role. */ @Test public void testRoleInitBeforeBackupElectionRuns() throws Exception { + Args.getInstance().witness = true; configureBackup(); // a directly constructed manager has not gone through init(): pre-election window @@ -300,6 +298,7 @@ public void testRoleInitBeforeBackupElectionRuns() throws Exception { */ @Test public void testRoleRefreshOnBackupStatusChange() throws Exception { + Args.getInstance().witness = true; configureBackup(); markBackupInitiated(backupManager); @@ -311,12 +310,14 @@ public void testRoleRefreshOnBackupStatusChange() throws Exception { } /** - * After init() has run, status INIT is a real election state — the transport is up but - * no keep-alive timeout has been judged yet — and maps 1:1 to init (0). It must not be - * misreported as slaver (2), the pre-fix behavior. + * After init() has run, status INIT is a real election state — the transport is up + * but no keep-alive timeout has been judged yet, or the master lost keep-alive and a + * re-election is running — and maps 1:1 to init (1). It must not be misreported as + * slaver (3), the pre-fix behavior. */ @Test public void testRoleInitAfterElectionStarted() throws Exception { + Args.getInstance().witness = true; configureBackup(); markBackupInitiated(backupManager); @@ -334,6 +335,7 @@ public void testRoleInitAfterElectionStarted() throws Exception { */ @Test public void testRoleMultiWitnessKeys() throws Exception { + Args.getInstance().witness = true; configureBackup(); String key1 = PublicMethod.getRandomPrivateKey(); String key2 = PublicMethod.getRandomPrivateKey(); diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index 4ad3b7af377..e5763d5ffc2 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -227,15 +227,15 @@ public void testNodeInfoMetric() { @Test public void testNodeRoleMetric() { - // a single label-less series per node; the value encodes the role (0 = init, - // 1 = master, 2 = slaver), here master after recording NODE_ROLE_MASTER + // a single label-less series per node; the value encodes the role (0 = fullnode, + // 1 = init, 2 = master, 3 = slaver), here master after recording NODE_ROLE_MASTER SRMetrics.recordNodeRole(SRMetrics.NODE_ROLE_MASTER); Double value = CollectorRegistry.defaultRegistry.getSampleValue( MetricKeys.Gauge.NODE_ROLE, new String[] {}, new String[] {}); Assert.assertNotNull("tron:node_role sample should exist", value); - Assert.assertEquals(1.0, value, 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, value, 0.0); } @Test From 04571716d5a72e35e735d539b3b85e5b8e5d6cd9 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 22 Oct 2026 15:02:34 +0800 Subject: [PATCH 15/26] docs(metrics): trim verbose comments --- .../tron/common/prometheus/MetricKeys.java | 11 +++------ .../tron/common/prometheus/MetricLabels.java | 6 ++--- .../org/tron/common/prometheus/SRMetrics.java | 17 +++---------- .../consensus/dpos/MaintenanceManager.java | 9 +------ .../org/tron/common/backup/BackupManager.java | 24 +++++-------------- .../main/java/org/tron/program/FullNode.java | 7 ++---- 6 files changed, 17 insertions(+), 57 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 8cdd1c6778c..2746dd732a0 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -14,9 +14,7 @@ public static class Counter { public static final String TXS = "tron:txs"; public static final String MINER = "tron:miner"; public static final String BLOCK_FORK = "tron:block_fork"; - // witness label: bounded cardinality -- SR candidate pool is finite, rotation is - // infrequent (at most once per maintenance interval); kept for at-a-glance SR - // identification in dashboards rather than requiring log cross-referencing. + // witness label: bounded cardinality (finite SR pool) for dashboard readability public static final String SR_SET_CHANGE = "tron:sr_set_change"; public static final String P2P_ERROR = "tron:p2p_error"; public static final String P2P_DISCONNECT = "tron:p2p_disconnect"; @@ -41,14 +39,11 @@ public static class Gauge { public static final String DB_SST_LEVEL = "tron:db_sst_level"; public static final String MANAGER_QUEUE = "tron:manager_queue_size"; public static final String TX_CACHE = "tron:tx_cache"; - // status of the SR configured locally (witness address), refreshed at startup and on - // maintenance blocks, when the vote counts and the active witness set are updated + // per local SR (witness address); refreshed at startup and on every maintenance block public static final String SR_IS_ACTIVE = "tron:sr_is_active"; public static final String SR_VOTES = "tron:sr_votes"; public static final String SR_RANK = "tron:sr_rank"; - // numeric role gauge of the local node in the backup election (0=init, 1=master, - // 2=slaver), one series per local witness address, refreshed at startup and on - // every backup-status change; witness-configured nodes only + // numeric role of this node: 0 = fullnode, 1 = init, 2 = master, 3 = slaver public static final String NODE_ROLE = "tron:node_role"; private Gauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index ba5ee6a81eb..7dda8739373 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -59,8 +59,7 @@ public static class Gauge { public static final String PEERS_VALID = "valid"; - // ADDRESS is the label of the tron:sr_* gauges; tron:node_role carries no label - // (single series, the value encodes the role) + // ADDRESS is the label of the tron:sr_* gauges (node_role carries no label) public static final String ADDRESS = "address"; private Gauge() { @@ -85,8 +84,7 @@ private Histogram() { // Info public static class Info { public static final String VERSION = "version"; - // identifies the genesis block: the label value is the chain id derived from the - // genesis block hash, so the label is named genesis_block_id + // identifies the genesis block: the value is the chain id from the genesis block hash public static final String CHAIN_ID = "genesis_block_id"; private Info() { diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index a6942fdfb04..91f65539fe1 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -10,21 +10,13 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } - // numeric encoding of the tron:node_role gauge value: 0 = fullnode (started without - // witness mode), 1 = init (before the first election round, or during a re-election - // after the master lost keep-alive), 2 = master, 3 = slaver. Read by BackupManager - // when it reports the election state; documented for operators in - // docs/metrics-changelog.md + // node_role: 0 = fullnode, 1 = init (pre-election / re-election), 2 = master, 3 = slaver public static final int NODE_ROLE_FULLNODE = 0; public static final int NODE_ROLE_INIT = 1; public static final int NODE_ROLE_MASTER = 2; public static final int NODE_ROLE_SLAVER = 3; - /** - * Publishes the node-role gauge as a single label-less series, encoding the current - * role as a number (0 = fullnode, 1 = init, 2 = master, 3 = slaver). A role change - * rewrites the value of the same series, so no stale series are left behind. - */ + /** Publishes node_role as one label-less series; role changes rewrite it in place. */ public static void recordNodeRole(int role) { if (!Metrics.enabled()) { return; @@ -32,10 +24,7 @@ public static void recordNodeRole(int role) { Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, role); } - /** - * Publishes the sr_* gauges as one series per local witness address. An empty address - * records the node-level zero series a fullnode exposes when no witness is configured. - */ + /** Publishes one sr_* series per local witness address; none without keys. */ public static void recordSrStatus(String address, boolean active, long votes, long rank) { if (!Metrics.enabled()) { return; diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index 76c8f341fa0..07e579938b3 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -169,14 +169,7 @@ public void doMaintenance() { recordWitnessMetrics(); } - /** - * Records the witness gauges (active-set membership, settled votes, settled rank) for - * every consensus miner the node assembles from its configured witness keys, one - * series per address. A node without a witness keeps the zero series with an empty - * address label, published once at startup by the caller. Runs at the end of every - * maintenance and once at node startup. Never throws: a metrics failure must not - * affect maintenance or block processing. - */ + /** Publishes one sr_* series per consensus miner, per maintenance and at startup; never throws. */ public void recordWitnessMetrics() { if (!Metrics.enabled()) { return; diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 40525f417d3..4c7cbdf25fa 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -73,12 +73,7 @@ public void setStatus(BackupStatusEnum status) { recordNodeRole(); } - /** - * Publishes the {@code tron:node_role} gauge — a single label-less series encoding - * the current role as a number (0 = fullnode, 1 = init, 2 = master, 3 = slaver). - * Runs once after the application has started — covering the window before the backup - * election first runs — and again on every backup-status change. - */ + /** Publishes tron:node_role, once at startup and on every backup-status change. */ public void recordNodeRole() { if (!Metrics.enabled()) { return; @@ -87,25 +82,19 @@ public void recordNodeRole() { } private int currentRole() { + // started without -w: a plain fullnode; witness keys are not even loaded if (!Args.getInstance().isWitness()) { - // started without witness mode: a fullnode plays no role in any election, even - // when witness keys are configured (the keys only feed the sr_* address series) return SRMetrics.NODE_ROLE_FULLNODE; } + // no backup election to take part in: this node holds the primary key itself if (!isBackupConfigured()) { - // no backup election to take part in: this node holds the primary witness key - // itself return SRMetrics.NODE_ROLE_MASTER; } + // status still defaults to MASTER before backup init; not a real election result yet if (!isInit) { - // the backup server only starts after the first fresh block, so between startup - // and the first election round the status default (MASTER) is not a real result; - // report init instead of pretending the election chose a master return SRMetrics.NODE_ROLE_INIT; } - // post-init the status is a real election outcome, so map every state 1:1; INIT - // here covers both the window where the transport is up but no keep-alive timeout - // has been judged yet and the re-election entered after the master lost keep-alive + // post-init INIT covers the pre-keep-alive window and master-loss re-election switch (status) { case MASTER: return SRMetrics.NODE_ROLE_MASTER; @@ -117,8 +106,7 @@ private int currentRole() { } private boolean isBackupConfigured() { - // same condition BackupServer.initServer() uses to start the election transport, - // i.e. the only way init() -- and with it the first real status -- can be reached + // same condition BackupServer.initServer() uses to start the election transport return parameter.getBackupPort() > 0 && !parameter.getBackupMembers().isEmpty(); } diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 9ed062371d8..a4e5ce071fd 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -73,13 +73,10 @@ public static void main(String[] args) { Application appT = ApplicationFactory.create(context); context.registerShutdownHook(); appT.startup(); - // the genesis block id (chainId) is only available after the context refresh - // (Manager.initGenesis) + // the genesis block id (chainId) is only available after context refresh (Manager.initGenesis) Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); - // the sr snapshot and the initial node_role series read the consensus miners, which - // are assembled in ApplicationImpl.startup() via ConsensusService.start(), so they - // are available here + // the witness snapshot reads consensus miners, assembled during appT.startup() context.getBean(MaintenanceManager.class).recordWitnessMetrics(); context.getBean(BackupManager.class).recordNodeRole(); if (parameter.isSolidityNode()) { From 541306afd5d9413cdfc09de4d782156d0765f5cd Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 22 Oct 2026 15:11:39 +0800 Subject: [PATCH 16/26] refactor(metrics): run sr hooks only on the fullnode path --- framework/src/main/java/org/tron/program/FullNode.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index a4e5ce071fd..5333a53f77d 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -76,13 +76,14 @@ public static void main(String[] args) { // the genesis block id (chainId) is only available after context refresh (Manager.initGenesis) Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); - // the witness snapshot reads consensus miners, assembled during appT.startup() - context.getBean(MaintenanceManager.class).recordWitnessMetrics(); - context.getBean(BackupManager.class).recordNodeRole(); if (parameter.isSolidityNode()) { SolidityNode node = context.getBean(SolidityNode.class); node.run(); } + // the witness snapshot reads consensus miners, assembled during appT.startup(); + // solidity nodes never run as SRs, so the SR hooks stay on the fullnode path + context.getBean(MaintenanceManager.class).recordWitnessMetrics(); + context.getBean(BackupManager.class).recordNodeRole(); appT.blockUntilShutdown(); } From 82c00dcc6458d90318efc03983b2d44f7456e6b5 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 22 Oct 2026 18:03:22 +0800 Subject: [PATCH 17/26] refactor(metrics): publish sr gauges from bean lifecycle --- .../consensus/dpos/MaintenanceManager.java | 2 ++ .../org/tron/common/backup/BackupManager.java | 9 +++++++- .../main/java/org/tron/program/FullNode.java | 6 ------ .../program/FullNodeCryptoEngineTest.java | 20 ++---------------- .../java/org/tron/program/FullNodeTest.java | 21 ++----------------- 5 files changed, 14 insertions(+), 44 deletions(-) diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index 07e579938b3..0f16850f959 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -57,6 +57,8 @@ public class MaintenanceManager { public void init() { currentWitness.addAll(consensusDelegate.getActiveWitnesses()); + // miners are assembled by ConsensusService before DposService.start() calls init() + recordWitnessMetrics(); } public void applyBlock(BlockCapsule blockCapsule) { diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 4c7cbdf25fa..4326f3e4bbd 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -14,6 +14,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -74,11 +75,17 @@ public void setStatus(BackupStatusEnum status) { } /** Publishes tron:node_role, once at startup and on every backup-status change. */ + @PostConstruct public void recordNodeRole() { if (!Metrics.enabled()) { return; } - SRMetrics.recordNodeRole(currentRole()); + try { + SRMetrics.recordNodeRole(currentRole()); + } catch (Exception e) { + // a metrics failure must not abort startup or the backup election + logger.warn("Failed to record node role", e); + } } private int currentRole() { diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 5333a53f77d..5f7f050540f 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -9,13 +9,11 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.arch.Arch; -import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.Metrics; -import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.config.args.CLIParameter; @@ -80,10 +78,6 @@ public static void main(String[] args) { SolidityNode node = context.getBean(SolidityNode.class); node.run(); } - // the witness snapshot reads consensus miners, assembled during appT.startup(); - // solidity nodes never run as SRs, so the SR hooks stay on the fullnode path - context.getBean(MaintenanceManager.class).recordWitnessMetrics(); - context.getBean(BackupManager.class).recordNodeRole(); appT.blockUntilShutdown(); } diff --git a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java index f144496559e..398e3595bb6 100644 --- a/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeCryptoEngineTest.java @@ -11,13 +11,11 @@ import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import com.typesafe.config.ConfigUtil; import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.util.ArrayList; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -29,12 +27,9 @@ import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.prometheus.Metrics; -import org.tron.consensus.base.Param; -import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.args.Args; import org.tron.core.config.args.WitnessInitializer; import org.tron.core.exception.TronError; @@ -96,30 +91,19 @@ private void assertStartup(String cryptoConfig) throws Exception { File config = writeConfig(cryptoConfig); File output = new File(temporaryFolder.newFolder(), "chain"); Application application = mock(Application.class); - MaintenanceManager maintenanceManager = mock(MaintenanceManager.class); - BackupManager backupManager = mock(BackupManager.class); - // the application startup is mocked, so ConsensusService.start() never fills the - // consensus miners; FullNode reads them right after startup to publish the fullnode - // zero series, so hand it the empty miner list of a witness-less node - Param.getInstance().setMiners(new ArrayList<>()); // Exercise the real entry point and config loader, while isolating services and chain stores. try (MockedStatic exit = mockStatic(ExitManager.class); MockedStatic logs = mockStatic(LogService.class); MockedStatic metrics = mockStatic(Metrics.class); MockedStatic factory = mockStatic(ApplicationFactory.class); MockedConstruction contexts = - mockConstruction(TronApplicationContext.class, (context, constructionContext) -> { - when(context.getBean(MaintenanceManager.class)).thenReturn(maintenanceManager); - when(context.getBean(BackupManager.class)).thenReturn(backupManager); - })) { + mockConstruction(TronApplicationContext.class)) { factory.when(() -> ApplicationFactory.create(any())).thenReturn(application); FullNode.main(new String[]{"-c", config.toString(), "-d", output.toString()}); assertEquals(1, contexts.constructed().size()); verify(contexts.constructed().get(0)).refresh(); - InOrder inOrder = inOrder(application, maintenanceManager, backupManager); + InOrder inOrder = inOrder(application); inOrder.verify(application).startup(); - inOrder.verify(maintenanceManager).recordWitnessMetrics(); - inOrder.verify(backupManager).recordNodeRole(); inOrder.verify(application).blockUntilShutdown(); } } diff --git a/framework/src/test/java/org/tron/program/FullNodeTest.java b/framework/src/test/java/org/tron/program/FullNodeTest.java index 19882c9e0ab..cda3514d4e5 100644 --- a/framework/src/test/java/org/tron/program/FullNodeTest.java +++ b/framework/src/test/java/org/tron/program/FullNodeTest.java @@ -31,13 +31,10 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.arch.Arch; -import org.tron.common.backup.BackupManager; import org.tron.common.exit.ExitManager; import org.tron.common.log.LogService; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.Metrics; -import org.tron.consensus.base.Param; -import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.config.args.Args; import org.tron.core.config.args.CommandLineArguments; import org.tron.core.exception.TronError; @@ -74,12 +71,6 @@ public void testNodeModeRetainsInitialization() { CommonParameter parameter = new CommonParameter(); parameter.logbackPath = "node-logback.xml"; Application application = Mockito.mock(Application.class); - MaintenanceManager maintenanceManager = Mockito.mock(MaintenanceManager.class); - BackupManager backupManager = Mockito.mock(BackupManager.class); - // the application startup is mocked, so ConsensusService.start() never fills the - // consensus miners; FullNode reads them right after startup to publish the fullnode - // zero series, so hand it the empty miner list of a witness-less node - Param.getInstance().setMiners(new ArrayList<>()); try (MockedStatic exitManager = Mockito.mockStatic(ExitManager.class); MockedStatic arch = Mockito.mockStatic(Arch.class); MockedStatic args = Mockito.mockStatic(Args.class); @@ -88,13 +79,7 @@ public void testNodeModeRetainsInitialization() { MockedStatic metrics = Mockito.mockStatic(Metrics.class); MockedStatic factory = Mockito.mockStatic(ApplicationFactory.class); MockedConstruction contexts = - Mockito.mockConstruction(TronApplicationContext.class, - (context, constructionContext) -> { - Mockito.when(context.getBean(MaintenanceManager.class)) - .thenReturn(maintenanceManager); - Mockito.when(context.getBean(BackupManager.class)) - .thenReturn(backupManager); - }); + Mockito.mockConstruction(TronApplicationContext.class); MockedConstruction ipcClients = Mockito.mockConstruction(IpcClient.class)) { parameters.when(CommonParameter::getInstance).thenReturn(parameter); factory.when(() -> ApplicationFactory.create(Mockito.any())).thenReturn(application); @@ -110,10 +95,8 @@ public void testNodeModeRetainsInitialization() { metrics.verify(Metrics::init); Assert.assertEquals(1, contexts.constructed().size()); Mockito.verify(contexts.constructed().get(0)).refresh(); - InOrder order = Mockito.inOrder(application, maintenanceManager, backupManager); + InOrder order = Mockito.inOrder(application); order.verify(application).startup(); - order.verify(maintenanceManager).recordWitnessMetrics(); - order.verify(backupManager).recordNodeRole(); order.verify(application).blockUntilShutdown(); Assert.assertTrue(ipcClients.constructed().isEmpty()); } From f2e109d467458196205cc60fc3247f2f62f3ef71 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 23 Oct 2026 11:33:06 +0800 Subject: [PATCH 18/26] refactor(metrics): retain witness identity and node role only --- .../tron/common/prometheus/MetricKeys.java | 6 +- .../tron/common/prometheus/MetricLabels.java | 2 +- .../tron/common/prometheus/MetricsGauge.java | 15 +- .../org/tron/common/prometheus/SRMetrics.java | 8 +- .../consensus/dpos/MaintenanceManager.java | 48 ---- docs/metrics-changelog.md | 8 +- .../tron/core/consensus/ConsensusService.java | 26 ++ .../tron/common/prometheus/SRMetricsTest.java | 266 +++++++----------- .../ConsensusServiceMetricsTest.java | 239 ++++++++++++++++ 9 files changed, 383 insertions(+), 235 deletions(-) create mode 100644 framework/src/test/java/org/tron/core/consensus/ConsensusServiceMetricsTest.java diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index d839b417ea8..7667291efc0 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -40,10 +40,8 @@ public static class Gauge { public static final String DB_SST_LEVEL = "tron:db_sst_level"; public static final String MANAGER_QUEUE = "tron:manager_queue_size"; public static final String TX_CACHE = "tron:tx_cache"; - // per local SR (witness address); refreshed at startup and on every maintenance block - public static final String SR_IS_ACTIVE = "tron:sr_is_active"; - public static final String SR_VOTES = "tron:sr_votes"; - public static final String SR_RANK = "tron:sr_rank"; + // per local consensus witness miner; value 1, label is the Base58Check witness address + public static final String WITNESS_ADDRESS = "tron:witness_address"; // numeric role of this node: 0 = fullnode, 1 = init, 2 = master, 3 = slaver public static final String NODE_ROLE = "tron:node_role"; diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index fcd796ef0a9..8d004cf5829 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -62,7 +62,7 @@ public static class Gauge { public static final String PEERS_VALID = "valid"; - // ADDRESS is the label of the tron:sr_* gauges (node_role carries no label) + // ADDRESS is the label of tron:witness_address (node_role carries no label) public static final String ADDRESS = "address"; private Gauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index bffe102eb01..3114c3d3bad 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -19,19 +19,12 @@ class MetricsGauge { init(MetricKeys.Gauge.DB_SIZE_BYTES, "tron db size .", "type", "db", "level"); init(MetricKeys.Gauge.DB_SST_LEVEL, "tron db files .", "type", "db", "level"); init(MetricKeys.Gauge.TX_CACHE, "tron tx cache info.", "type"); - init(MetricKeys.Gauge.SR_IS_ACTIVE, - "1 if the witness with this address is in the active witness set.", - MetricLabels.Gauge.ADDRESS); - init(MetricKeys.Gauge.SR_VOTES, "Vote count of the witness with this address.", - MetricLabels.Gauge.ADDRESS); - init(MetricKeys.Gauge.SR_RANK, - "Vote-count rank of the witness with this address (1-based, 0 when the " - + "witness is unknown).", + init(MetricKeys.Gauge.WITNESS_ADDRESS, + "1 for each consensus witness address this node holds a miner for.", MetricLabels.Gauge.ADDRESS); init(MetricKeys.Gauge.NODE_ROLE, - "Role of this node in the backup election, encoded as a number: 0 = init " - + "(before the first election round), 1 = master (producing blocks), " - + "2 = slaver (standing by)."); + "Role of this node, encoded as a number: 0 = fullnode, 1 = init, " + + "2 = master, 3 = slaver."); } private MetricsGauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 91f65539fe1..311ff679358 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -24,14 +24,12 @@ public static void recordNodeRole(int role) { Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, role); } - /** Publishes one sr_* series per local witness address; none without keys. */ - public static void recordSrStatus(String address, boolean active, long votes, long rank) { + /** Publishes one tron:witness_address series (value 1) for a local witness address. */ + public static void recordWitnessAddress(String address) { if (!Metrics.enabled()) { return; } - Metrics.gaugeSet(MetricKeys.Gauge.SR_IS_ACTIVE, active ? 1 : 0, address); - Metrics.gaugeSet(MetricKeys.Gauge.SR_VOTES, votes, address); - Metrics.gaugeSet(MetricKeys.Gauge.SR_RANK, rank, address); + Metrics.gaugeSet(MetricKeys.Gauge.WITNESS_ADDRESS, 1, address); } public static void recordSrSetChange(List currentWits, List newWits) { diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index 0f16850f959..fd5e4364d0d 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -16,12 +16,8 @@ import org.bouncycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; -import org.tron.common.utils.StringUtil; import org.tron.consensus.ConsensusDelegate; -import org.tron.consensus.base.Param; -import org.tron.consensus.base.Param.Miner; import org.tron.consensus.pbft.PbftManager; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.BlockCapsule; @@ -30,7 +26,6 @@ import org.tron.core.store.DelegationStore; import org.tron.core.store.DynamicPropertiesStore; import org.tron.core.store.VotesStore; -import org.tron.core.store.WitnessStore; @Slf4j(topic = "consensus") @Component @@ -57,8 +52,6 @@ public class MaintenanceManager { public void init() { currentWitness.addAll(consensusDelegate.getActiveWitnesses()); - // miners are assembled by ConsensusService before DposService.start() calls init() - recordWitnessMetrics(); } public void applyBlock(BlockCapsule blockCapsule) { @@ -167,47 +160,6 @@ public void doMaintenance() { delegationStore.setWitnessVote(nextCycle, witness.createDbKey(), witness.getVoteCount()); }); } - - recordWitnessMetrics(); - } - - /** Publishes one sr_* series per consensus miner, per maintenance and at startup; never throws. */ - public void recordWitnessMetrics() { - if (!Metrics.enabled()) { - return; - } - List miners = Param.getInstance().getMiners(); - if (miners == null) { - // consensus never started: nothing to publish (never the case on a running node) - return; - } - for (Miner miner : miners) { - ByteString witnessAddressBs = miner.getWitnessAddress(); - if (witnessAddressBs == null) { - continue; - } - String address = StringUtil.encode58Check(witnessAddressBs.toByteArray()); - try { - boolean active = consensusDelegate.getActiveWitnesses().contains(witnessAddressBs); - WitnessCapsule witness = consensusDelegate.getWitness(witnessAddressBs.toByteArray()); - long votes = witness == null ? 0 : witness.getVoteCount(); - long rank = witnessRank(witnessAddressBs); - SRMetrics.recordSrStatus(address, active, votes, rank); - } catch (Exception e) { - logger.warn("Failed to record witness metrics for {}", address, e); - } - } - } - - private long witnessRank(ByteString witnessAddress) { - List witnesses = consensusDelegate.getAllWitnesses(); - WitnessStore.sortWitnesses(witnesses, false); - for (int i = 0; i < witnesses.size(); i++) { - if (witnesses.get(i).getAddress().equals(witnessAddress)) { - return i + 1L; - } - } - return 0L; } private Map countVote(VotesStore votesStore) { diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 08da47952e7..10b3353bb10 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -10,12 +10,8 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:node_role` (Gauge, no labels) — role of this node, encoded as a number: `0` = `fullnode` (started without witness mode; witness keys, if any, are not loaded on such a node and no sr_* series are published), `1` = `init` (from node startup until the backup election first runs, and again while a re-election is running after the master lost keep-alive), `2` = `master` (holds the primary witness key; also the value when backup is not configured), `3` = `slaver` (standing by for a master). The role is a node-level status, not a per-identity one, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:sr_is_active` (Gauge, label `address`) — 1 when the witness with this address is in the active witness set, 0 otherwise. Refreshed at startup and on maintenance blocks, together with the vote counts. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:sr_votes` (Gauge, label `address`) — vote count of the witness with this address, as of the last maintenance block. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:sr_rank` (Gauge, label `address`) — 1-based vote-count rank of the witness with this address, 0 when the witness is unknown. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - - The `address` label carries the 58check-encoded address of one of the node's consensus witness miners. The miners are assembled once at startup: one miner per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. When several witness private keys are configured, each key address gets its own series with its own values. A node without a witness publishes no `sr_*` series at all. +- `tron:node_role` (Gauge, no labels) — role of this node, encoded as a number: `0` = `fullnode` (started without witness mode; witness keys, if any, are not loaded on such a node and no `tron:witness_address` series is published), `1` = `init` (from node startup until the backup election first runs, and again while a re-election is running after the master lost keep-alive), `2` = `master` (holds the primary witness key; also the value when backup is not configured), `3` = `slaver` (standing by for a master). The role is a node-level status, not a per-identity one, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:witness_address` (Gauge, label `address`) — value `1` for each consensus witness miner this node holds; the label is the Base58Check-encoded witness account address. The set is assembled once at startup from the finalized consensus miners: one series per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. With several witness keys, each address gets its own series. A plain fullnode without a witness miner publishes no `tron:witness_address` series at all. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_already_known` (Counter) — incremented when a block response matches an outstanding adv request whose exact block ID is already known before that response is processed (best-effort signal; concurrent arrivals may be missed). ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java index b3921b1b657..2ed236ee9c6 100644 --- a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java +++ b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java @@ -11,6 +11,9 @@ import org.springframework.stereotype.Component; import org.tron.common.crypto.SignUtils; import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.StringUtil; import org.tron.consensus.Consensus; import org.tron.consensus.base.Param; import org.tron.consensus.base.Param.Miner; @@ -80,9 +83,32 @@ public void start() { param.setBlockHandle(blockHandle); param.setPbftInterface(pbftBaseImpl); consensus.start(param); + publishWitnessIdentities(miners); logger.info("consensus service start success"); } + /** + * Publishes one tron:witness_address series per consensus miner, once at startup. The + * witness account address is used (not the signing-key address, which differs in the + * single-key multisig setup). Never throws: a metrics failure must not abort startup. + */ + private void publishWitnessIdentities(List miners) { + if (!Metrics.enabled()) { + return; + } + for (Miner miner : miners) { + ByteString witnessAddress = miner.getWitnessAddress(); + if (witnessAddress == null) { + continue; + } + try { + SRMetrics.recordWitnessAddress(StringUtil.encode58Check(witnessAddress.toByteArray())); + } catch (Exception e) { + logger.warn("Failed to record witness address metric", e); + } + } + } + public void stop() { logger.info("consensus service closed start."); consensus.stop(); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 4ace2802800..4a743691646 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -13,6 +13,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; import org.tron.common.BaseTest; import org.tron.common.TestConstants; import org.tron.common.backup.BackupManager; @@ -27,7 +28,6 @@ import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.args.Args; import org.tron.core.consensus.ConsensusService; -import org.tron.program.Version; import org.tron.protos.Protocol; import org.tron.protos.Protocol.Vote; @@ -183,81 +183,38 @@ public void testSrRemoveDirect() { Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, stableWit)); } + // ---- node_role: the four states, the pre-election window and status transitions ---- + /** - * No witness configured: with an empty miner list a plain fullnode publishes none of - * the sr_* series, and node_role reports 0 because the node started without witness - * mode, while node_info keeps its two static labels. + * Started without witness mode: a plain fullnode reports role 0. */ @Test - public void testSrStatusFullNode() { + public void testNodeRoleFullNode() { Args.getInstance().witness = false; - Param.getInstance().setMiners(new ArrayList<>()); - - maintenanceManager.recordWitnessMetrics(); - Assert.assertNull("empty miner list must not write an empty-address sr series", - srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, "")); - backupManager.recordNodeRole(); - Assert.assertEquals("a node started without witness mode reports role 0", - SRMetrics.NODE_ROLE_FULLNODE, roleGauge(), 0.0); - - Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), - Args.getInstance().getChainId()); - Assert.assertEquals(1.0, info(), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_FULLNODE, roleGauge(), 0.0); } /** - * Witness configured without a backup election (backup members unset): the node - * always holds the primary key itself, so the single role series reports master (2); - * node_info stays a static two-label identity record. + * Witness configured without a backup election (backup members unset): the node always + * holds the primary key itself, so the single role series reports master (2). */ @Test - public void testSrStatusMaster() { + public void testNodeRoleMaster() { Args.getInstance().witness = true; - ByteString witness = registerWitnessWithVotes(1000); - registerWitnessWithVotes(50); - chainBaseManager.getWitnessScheduleStore() - .saveActiveWitnesses(Collections.singletonList(witness)); - - seedMiners(witness); - backupManager.recordNodeRole(); - maintenanceManager.recordWitnessMetrics(); - - String address = StringUtil.encode58Check(witness.toByteArray()); - Assert.assertEquals(1.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address), 0.0); - Assert.assertEquals(1000.0, srGauge(MetricKeys.Gauge.SR_VOTES, address), 0.0); - Assert.assertEquals(1.0, srGauge(MetricKeys.Gauge.SR_RANK, address), 0.0); Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); - Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), - Args.getInstance().getChainId()); - Assert.assertEquals(1.0, info(), 0.0); } /** - * Backup election slaver: the single role series reports slaver (3) while is_active - * reflects the active witness set. + * Backup election slaver: the node stands by for a master and reports slaver (3). */ @Test - public void testSrStatusSlaver() throws Exception { + public void testNodeRoleSlaver() throws Exception { Args.getInstance().witness = true; configureBackup(); - backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); - ByteString activeWitness = registerWitnessWithVotes(10); - ByteString witness = registerWitnessWithVotes(7); - chainBaseManager.getWitnessScheduleStore() - .saveActiveWitnesses(Collections.singletonList(activeWitness)); - - seedMiners(witness); markBackupInitiated(backupManager); - - backupManager.recordNodeRole(); - maintenanceManager.recordWitnessMetrics(); - - String address = StringUtil.encode58Check(witness.toByteArray()); - Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address), 0.0); - Assert.assertEquals(7.0, srGauge(MetricKeys.Gauge.SR_VOTES, address), 0.0); - Assert.assertTrue(srGauge(MetricKeys.Gauge.SR_RANK, address) > 0); + backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); } @@ -268,7 +225,7 @@ public void testSrStatusSlaver() throws Exception { * after init() rewrites the same single series to the elected role. */ @Test - public void testRoleInitBeforeBackupElectionRuns() throws Exception { + public void testNodeRoleInitBeforeBackupElectionRuns() throws Exception { Args.getInstance().witness = true; configureBackup(); @@ -292,12 +249,25 @@ public void testRoleInitBeforeBackupElectionRuns() throws Exception { } /** - * setStatus() itself republishes the single label-less node_role series without - * touching any store, so a backup-election transition is reflected immediately instead - * of at the next maintenance. + * After init() has run, status INIT is a real election state — the transport is up but no + * keep-alive timeout has been judged yet, or the master lost keep-alive and a re-election + * is running — and maps 1:1 to init (1). It must not be misreported as slaver (3). */ @Test - public void testRoleRefreshOnBackupStatusChange() throws Exception { + public void testNodeRoleInitAfterElectionStarted() throws Exception { + Args.getInstance().witness = true; + configureBackup(); + markBackupInitiated(backupManager); + backupManager.setStatus(BackupManager.BackupStatusEnum.INIT); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + } + + /** + * setStatus() itself republishes the single label-less node_role series, so a + * backup-election transition is reflected immediately. + */ + @Test + public void testNodeRoleRefreshOnBackupStatusChange() throws Exception { Args.getInstance().witness = true; configureBackup(); markBackupInitiated(backupManager); @@ -309,74 +279,77 @@ public void testRoleRefreshOnBackupStatusChange() throws Exception { Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); } + // ---- tron:witness_address: one series per consensus miner, value 1 ---- + /** - * After init() has run, status INIT is a real election state — the transport is up - * but no keep-alive timeout has been judged yet, or the master lost keep-alive and a - * re-election is running — and maps 1:1 to init (1). It must not be misreported as - * slaver (3), the pre-fix behavior. + * A single consensus miner publishes exactly one tron:witness_address series with value 1. */ @Test - public void testRoleInitAfterElectionStarted() throws Exception { - Args.getInstance().witness = true; - configureBackup(); - markBackupInitiated(backupManager); + public void testWitnessAddressValueOne() { + ByteString address = uniqueAddress(); + publishWitnessIdentities( + Collections.singletonList(miner(address, address))); + Assert.assertEquals(1.0, witnessGauge(encode(address)), 0.0); + } - backupManager.setStatus(BackupManager.BackupStatusEnum.INIT); - Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + /** + * Multi-key setup: every consensus miner address gets its own series with value 1. + */ + @Test + public void testWitnessAddressMultipleMiners() { + ByteString address1 = uniqueAddress(); + ByteString address2 = uniqueAddress(); + List miners = new ArrayList<>(); + miners.add(miner(address1, address1)); + miners.add(miner(address2, address2)); + + publishWitnessIdentities(miners); + + Assert.assertEquals(1.0, witnessGauge(encode(address1)), 0.0); + Assert.assertEquals(1.0, witnessGauge(encode(address2)), 0.0); } /** - * Testnet multi-key setup: the consensus layer assembles one miner per configured - * witness private key, so every sr gauge is written as one series per key address while - * node_role stays a single series regardless of the key count. Role recording only - * touches the role series; maintenance-time recording gives each address its own sr - * values (here distinct votes, activity and rank) and leaves the role series alone. A - * status change rewrites the single role series without touching the sr series. + * Single-key multisig: the witness account address may differ from the address derived + * from the signing key. Only the witness account address is published; the signing-key + * address must not appear. */ @Test - public void testRoleMultiWitnessKeys() throws Exception { - Args.getInstance().witness = true; - configureBackup(); - String key1 = PublicMethod.getRandomPrivateKey(); - String key2 = PublicMethod.getRandomPrivateKey(); - ByteString miner1 = addressOf(key1); - ByteString miner2 = addressOf(key2); - seedMiners(miner1, miner2); - String address1 = StringUtil.encode58Check(miner1.toByteArray()); - String address2 = StringUtil.encode58Check(miner2.toByteArray()); - ByteString witness1 = registerWitnessWithVotes(miner1.toByteArray(), 100); - ByteString witness2 = registerWitnessWithVotes(miner2.toByteArray(), 40); - chainBaseManager.getWitnessScheduleStore() - .saveActiveWitnesses(Collections.singletonList(witness1)); - markBackupInitiated(backupManager); + public void testWitnessAddressUsesWitnessAccountNotSigningKey() { + ByteString signingKeyAddress = addressOf(PublicMethod.getRandomPrivateKey()); + ByteString witnessAccountAddress = uniqueAddress(); - backupManager.recordNodeRole(); - Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); - // role recording never writes sr_* series - Assert.assertNull(srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address1)); - Assert.assertNull(srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address2)); - - // one sr series per address, with per-address values: witness1 is active with the - // higher vote count, witness2 stands inactive on the lower rank - maintenanceManager.recordWitnessMetrics(); - Assert.assertEquals(1.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address1), 0.0); - Assert.assertEquals(100.0, srGauge(MetricKeys.Gauge.SR_VOTES, address1), 0.0); - Assert.assertEquals(0.0, srGauge(MetricKeys.Gauge.SR_IS_ACTIVE, address2), 0.0); - Assert.assertEquals(40.0, srGauge(MetricKeys.Gauge.SR_VOTES, address2), 0.0); - // rank is absolute within the full witness store, which also holds genesis - // witnesses; the two local series must at least order by their votes - Assert.assertTrue(srGauge(MetricKeys.Gauge.SR_RANK, address1) > 0); - Assert.assertTrue(srGauge(MetricKeys.Gauge.SR_RANK, address2) > 0); - Assert.assertTrue("higher votes must give the better rank", - srGauge(MetricKeys.Gauge.SR_RANK, address1) - < srGauge(MetricKeys.Gauge.SR_RANK, address2)); - - // sr recording never touches the role series; a status change flips the single - // role series and leaves the sr series alone - backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); - Assert.assertEquals(100.0, srGauge(MetricKeys.Gauge.SR_VOTES, address1), 0.0); - Assert.assertEquals(40.0, srGauge(MetricKeys.Gauge.SR_VOTES, address2), 0.0); + publishWitnessIdentities( + Collections.singletonList(miner(signingKeyAddress, witnessAccountAddress))); + + Assert.assertEquals(1.0, witnessGauge(encode(witnessAccountAddress)), 0.0); + Assert.assertNull(witnessGauge(encode(signingKeyAddress))); + } + + /** + * No miner configured: a plain fullnode publishes no tron:witness_address series at all, + * not even one with an empty address label. + */ + @Test + public void testNoWitnessAddressSeriesForEmptyMiners() { + publishWitnessIdentities(new ArrayList<>()); + Assert.assertNull(witnessGauge("")); + } + + /** + * Metrics disabled: no tron:witness_address series is published. + */ + @Test + public void testWitnessAddressNotPublishedWhenMetricsDisabled() { + ByteString address = uniqueAddress(); + Args.getInstance().setMetricsPrometheusEnable(false); + try { + publishWitnessIdentities( + Collections.singletonList(miner(address, address))); + Assert.assertNull(witnessGauge(encode(address))); + } finally { + Args.getInstance().setMetricsPrometheusEnable(true); + } } /** @@ -400,64 +373,37 @@ private void markBackupInitiated(BackupManager manager) throws Exception { } /** - * Builds the consensus-miner identity the node assembles from a configured private - * key: the address derived from that key. + * Drives the private startup hook directly; production publishes once from start(), + * the test exercises the same loop without starting consensus. */ - private ByteString addressOf(String privateKeyHex) { - return ByteString.copyFrom( - SignUtils.fromPrivate(ByteArray.fromHexString(privateKeyHex)).getAddress()); + private void publishWitnessIdentities(List miners) { + ReflectionTestUtils.invokeMethod(consensusService, "publishWitnessIdentities", miners); } /** - * Seeds the consensus miners the metrics path reads its identities from, one miner - * per address. The private key bytes are never read by the metrics path; tearDown - * clears the miner list again. + * Builds the consensus-miner identity the node assembles: the signing-key address and the + * witness account address are passed explicitly, exactly as ConsensusService does. */ - private void seedMiners(ByteString... addresses) { - Param param = Param.getInstance(); - List miners = new ArrayList<>(); - for (ByteString address : addresses) { - miners.add(param.new Miner(new byte[0], address, address)); - } - param.setMiners(miners); + private Param.Miner miner(ByteString privateKeyAddress, ByteString witnessAddress) { + return Param.getInstance().new Miner(new byte[0], privateKeyAddress, witnessAddress); } - private ByteString registerWitnessWithVotes(long votes) { - ByteString address = uniqueAddress(); - chainBaseManager.getWitnessStore().put(address.toByteArray(), - new WitnessCapsule(address, votes, "")); - chainBaseManager.getAccountStore().put(address.toByteArray(), - new AccountCapsule(Protocol.Account.newBuilder().setAddress(address).build())); - return address; + private ByteString addressOf(String privateKeyHex) { + return ByteString.copyFrom( + SignUtils.fromPrivate(ByteArray.fromHexString(privateKeyHex)).getAddress()); } - /** - * Registers a witness for a locally configured miner's address so the multi-key test - * can assert distinguishable per-address sr values. - */ - private ByteString registerWitnessWithVotes(byte[] addressBytes, long votes) { - ByteString address = ByteString.copyFrom(addressBytes); - chainBaseManager.getWitnessStore().put(address.toByteArray(), - new WitnessCapsule(address, votes, "")); - chainBaseManager.getAccountStore().put(address.toByteArray(), - new AccountCapsule(Protocol.Account.newBuilder().setAddress(address).build())); - return address; + private String encode(ByteString address) { + return StringUtil.encode58Check(address.toByteArray()); } - private Double srGauge(String name, String address) { + private Double witnessGauge(String address) { return CollectorRegistry.defaultRegistry.getSampleValue( - name, + MetricKeys.Gauge.WITNESS_ADDRESS, new String[] {MetricLabels.Gauge.ADDRESS}, new String[] {address}); } - private Double info() { - return CollectorRegistry.defaultRegistry.getSampleValue( - "tron:node_info", - new String[] {MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID}, - new String[] {Version.getVersion(), Args.getInstance().getChainId()}); - } - private Double roleGauge() { return CollectorRegistry.defaultRegistry.getSampleValue( MetricKeys.Gauge.NODE_ROLE, diff --git a/framework/src/test/java/org/tron/core/consensus/ConsensusServiceMetricsTest.java b/framework/src/test/java/org/tron/core/consensus/ConsensusServiceMetricsTest.java new file mode 100644 index 00000000000..078346dc40a --- /dev/null +++ b/framework/src/test/java/org/tron/core/consensus/ConsensusServiceMetricsTest.java @@ -0,0 +1,239 @@ +package org.tron.core.consensus; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockSettings; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; +import org.tron.common.args.GenesisBlock; +import org.tron.common.crypto.SignUtils; +import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.LocalWitnesses; +import org.tron.common.utils.StringUtil; +import org.tron.consensus.Consensus; +import org.tron.consensus.base.BlockHandle; +import org.tron.consensus.base.Param; +import org.tron.consensus.base.Param.Miner; +import org.tron.consensus.base.PbftInterface; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.store.WitnessStore; + +/** + * Startup integration tests for the witness-identity publication added to + * {@link ConsensusService#start()}. Pure JUnit4 + Mockito: no Spring, no real database, no + * {@code BaseTest}. SRMetrics is static-mocked so nothing touches the global Prometheus registry. + */ +public class ConsensusServiceMetricsTest { + + // Two valid, distinct ECDSA private keys (scalars 1 and 2). + private static final String KEY_ONE = + "0000000000000000000000000000000000000000000000000000000000000001"; + private static final String KEY_TWO = + "0000000000000000000000000000000000000000000000000000000000000002"; + + private ConsensusService service; + private Consensus consensus; + private WitnessStore witnessStore; + private BlockHandleImpl blockHandle; + private PbftBaseImpl pbftBaseImpl; + + // Singleton state saved so each test leaves the process as it found it. + private LocalWitnesses savedWitnesses; + private boolean savedMetricsEnabled; + private boolean savedEnable; + private boolean savedNeedSyncCheck; + private int savedMinParticipationRate; + private int savedBlockProduceTimeoutPercent; + private GenesisBlock savedGenesisBlock; + private List savedMiners; + private BlockHandle savedBlockHandle; + private int savedAgreeNodeCount; + private PbftInterface savedPbftInterface; + + @Before + public void setUp() throws Exception { + savedWitnesses = Args.getLocalWitnesses(); + savedMetricsEnabled = CommonParameter.getInstance().isMetricsPrometheusEnable(); + + Param param = Param.getInstance(); + savedEnable = param.isEnable(); + savedNeedSyncCheck = param.isNeedSyncCheck(); + savedMinParticipationRate = param.getMinParticipationRate(); + savedBlockProduceTimeoutPercent = param.getBlockProduceTimeoutPercent(); + savedGenesisBlock = param.getGenesisBlock(); + savedMiners = param.getMiners(); + savedBlockHandle = param.getBlockHandle(); + savedAgreeNodeCount = param.getAgreeNodeCount(); + savedPbftInterface = param.getPbftInterface(); + + consensus = mock(Consensus.class); + witnessStore = mock(WitnessStore.class); + blockHandle = mock(BlockHandleImpl.class); + pbftBaseImpl = mock(PbftBaseImpl.class); + + service = new ConsensusService(); + inject(service, "consensus", consensus); + inject(service, "witnessStore", witnessStore); + inject(service, "blockHandle", blockHandle); + inject(service, "pbftBaseImpl", pbftBaseImpl); + } + + @After + public void tearDown() { + Args.setLocalWitnesses(savedWitnesses); + CommonParameter.getInstance().setMetricsPrometheusEnable(savedMetricsEnabled); + + Param param = Param.getInstance(); + param.setEnable(savedEnable); + param.setNeedSyncCheck(savedNeedSyncCheck); + param.setMinParticipationRate(savedMinParticipationRate); + param.setBlockProduceTimeoutPercent(savedBlockProduceTimeoutPercent); + param.setGenesisBlock(savedGenesisBlock); + param.setMiners(savedMiners); + param.setBlockHandle(savedBlockHandle); + param.setAgreeNodeCount(savedAgreeNodeCount); + param.setPbftInterface(savedPbftInterface); + } + + @Test + public void singleKeyPublishesWitnessAccountNotSigningKey() throws Exception { + byte[] signingKey = derivedAddress(KEY_ONE); + byte[] witnessAccount = derivedAddress(KEY_TWO); + LocalWitnesses witnesses = witnesses(KEY_ONE); + witnesses.initWitnessAccountAddress(witnessAccount); + Args.setLocalWitnesses(witnesses); + CommonParameter.getInstance().setMetricsPrometheusEnable(true); + when(witnessStore.get(witnessAccount)).thenReturn(mock(WitnessCapsule.class)); + + try (MockedStatic sr = mockStatic(SRMetrics.class)) { + service.start(); + + sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(witnessAccount))); + sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(signingKey)), + never()); + } + verify(consensus).start(Mockito.any(Param.class)); + verify(witnessStore).get(witnessAccount); + verifyNoMoreInteractions(witnessStore); + } + + @Test + public void multiKeyPublishesEveryWitnessAddress() throws Exception { + Args.setLocalWitnesses(witnesses(KEY_ONE, KEY_TWO)); + CommonParameter.getInstance().setMetricsPrometheusEnable(true); + byte[] addrOne = derivedAddress(KEY_ONE); + byte[] addrTwo = derivedAddress(KEY_TWO); + when(witnessStore.get(addrOne)).thenReturn(mock(WitnessCapsule.class)); + when(witnessStore.get(addrTwo)).thenReturn(mock(WitnessCapsule.class)); + + try (MockedStatic sr = mockStatic(SRMetrics.class)) { + service.start(); + + sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(addrOne))); + sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(addrTwo))); + } + verify(consensus).start(Mockito.any(Param.class)); + verify(witnessStore).get(addrOne); + verify(witnessStore).get(addrTwo); + verifyNoMoreInteractions(witnessStore); + } + + @Test + public void noKeysPublishesNothingAndQueriesNoWitness() throws Exception { + Args.setLocalWitnesses(new LocalWitnesses()); + CommonParameter.getInstance().setMetricsPrometheusEnable(true); + + try (MockedStatic sr = mockStatic(SRMetrics.class)) { + service.start(); + sr.verifyNoInteractions(); + } + verify(consensus).start(Mockito.any(Param.class)); + verifyNoInteractions(witnessStore); + } + + @Test + public void disabledMetricsDoesNotPublish() throws Exception { + byte[] witnessAccount = derivedAddress(KEY_TWO); + LocalWitnesses witnesses = witnesses(KEY_ONE); + witnesses.initWitnessAccountAddress(witnessAccount); + Args.setLocalWitnesses(witnesses); + CommonParameter.getInstance().setMetricsPrometheusEnable(false); + when(witnessStore.get(witnessAccount)).thenReturn(mock(WitnessCapsule.class)); + + try (MockedStatic sr = mockStatic(SRMetrics.class)) { + service.start(); + sr.verifyNoInteractions(); + } + verify(consensus).start(Mockito.any(Param.class)); + verify(witnessStore).get(witnessAccount); + verifyNoMoreInteractions(witnessStore); + } + + @Test + public void metricsFailureDoesNotAbortStart() throws Exception { + Args.setLocalWitnesses(witnesses(KEY_ONE, KEY_TWO)); + CommonParameter.getInstance().setMetricsPrometheusEnable(true); + byte[] addrOne = derivedAddress(KEY_ONE); + byte[] addrTwo = derivedAddress(KEY_TWO); + when(witnessStore.get(addrOne)).thenReturn(mock(WitnessCapsule.class)); + when(witnessStore.get(addrTwo)).thenReturn(mock(WitnessCapsule.class)); + + MockSettings throwing = Mockito.withSettings().defaultAnswer(invocation -> { + throw new RuntimeException("metrics unavailable"); + }); + try (MockedStatic sr = Mockito.mockStatic(SRMetrics.class, throwing)) { + service.start(); + + // The per-miner try/catch must keep the loop going after the first failure. + sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(addrOne))); + sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(addrTwo))); + } + verify(consensus).start(Mockito.any(Param.class)); + verify(witnessStore).get(addrOne); + verify(witnessStore).get(addrTwo); + verifyNoMoreInteractions(witnessStore); + } + + @Test + public void nullWitnessAddressIsSkipped() throws Exception { + CommonParameter.getInstance().setMetricsPrometheusEnable(true); + Miner miner = Param.getInstance().new Miner(null, null, null); + + try (MockedStatic sr = mockStatic(SRMetrics.class)) { + ReflectionTestUtils.invokeMethod(service, "publishWitnessIdentities", Arrays.asList(miner)); + sr.verifyNoInteractions(); + } + } + + private static LocalWitnesses witnesses(String... keys) { + LocalWitnesses witnesses = new LocalWitnesses(); + witnesses.setPrivateKeys(Arrays.asList(keys)); + return witnesses; + } + + private static byte[] derivedAddress(String privateKey) { + return SignUtils.fromPrivate(ByteArray.fromHexString(privateKey)).getAddress(); + } + + private static void inject(Object target, String name, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } +} From 3f55a940748f801ba24e4cc034d21d16c5531925 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 23 Oct 2026 11:44:59 +0800 Subject: [PATCH 19/26] refactor(metrics): publish witness address once from maintenance init --- .../consensus/dpos/MaintenanceManager.java | 32 +++ .../tron/core/consensus/ConsensusService.java | 26 -- .../tron/common/prometheus/SRMetricsTest.java | 20 +- .../ConsensusServiceMetricsTest.java | 239 ------------------ 4 files changed, 42 insertions(+), 275 deletions(-) delete mode 100644 framework/src/test/java/org/tron/core/consensus/ConsensusServiceMetricsTest.java diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index fd5e4364d0d..6dca590fc68 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -16,8 +16,12 @@ import org.bouncycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.StringUtil; import org.tron.consensus.ConsensusDelegate; +import org.tron.consensus.base.Param; +import org.tron.consensus.base.Param.Miner; import org.tron.consensus.pbft.PbftManager; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.BlockCapsule; @@ -52,6 +56,8 @@ public class MaintenanceManager { public void init() { currentWitness.addAll(consensusDelegate.getActiveWitnesses()); + // miners are assembled by ConsensusService before DposService.start() calls init() + recordWitnessMetrics(); } public void applyBlock(BlockCapsule blockCapsule) { @@ -162,6 +168,32 @@ public void doMaintenance() { } } + /** + * Publishes one tron:witness_address series per consensus miner. The witness account address is + * immutable, so this runs once from {@link #init()} and is never refreshed. + */ + public void recordWitnessMetrics() { + if (!Metrics.enabled()) { + return; + } + List miners = Param.getInstance().getMiners(); + if (miners == null) { + // miners are not assembled yet + return; + } + for (Miner miner : miners) { + if (miner.getWitnessAddress() == null) { + continue; + } + try { + String address = StringUtil.encode58Check(miner.getWitnessAddress().toByteArray()); + SRMetrics.recordWitnessAddress(address); + } catch (Exception e) { + logger.warn("Failed to publish witness address metrics", e); + } + } + } + private Map countVote(VotesStore votesStore) { final Map countWitness = Maps.newHashMap(); Iterator> dbIterator = votesStore.iterator(); diff --git a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java index 2ed236ee9c6..b3921b1b657 100644 --- a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java +++ b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java @@ -11,9 +11,6 @@ import org.springframework.stereotype.Component; import org.tron.common.crypto.SignUtils; import org.tron.common.parameter.CommonParameter; -import org.tron.common.prometheus.Metrics; -import org.tron.common.prometheus.SRMetrics; -import org.tron.common.utils.StringUtil; import org.tron.consensus.Consensus; import org.tron.consensus.base.Param; import org.tron.consensus.base.Param.Miner; @@ -83,32 +80,9 @@ public void start() { param.setBlockHandle(blockHandle); param.setPbftInterface(pbftBaseImpl); consensus.start(param); - publishWitnessIdentities(miners); logger.info("consensus service start success"); } - /** - * Publishes one tron:witness_address series per consensus miner, once at startup. The - * witness account address is used (not the signing-key address, which differs in the - * single-key multisig setup). Never throws: a metrics failure must not abort startup. - */ - private void publishWitnessIdentities(List miners) { - if (!Metrics.enabled()) { - return; - } - for (Miner miner : miners) { - ByteString witnessAddress = miner.getWitnessAddress(); - if (witnessAddress == null) { - continue; - } - try { - SRMetrics.recordWitnessAddress(StringUtil.encode58Check(witnessAddress.toByteArray())); - } catch (Exception e) { - logger.warn("Failed to record witness address metric", e); - } - } - } - public void stop() { logger.info("consensus service closed start."); consensus.stop(); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 4a743691646..4ced605c229 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -13,7 +13,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.springframework.test.util.ReflectionTestUtils; import org.tron.common.BaseTest; import org.tron.common.TestConstants; import org.tron.common.backup.BackupManager; @@ -287,7 +286,7 @@ public void testNodeRoleRefreshOnBackupStatusChange() throws Exception { @Test public void testWitnessAddressValueOne() { ByteString address = uniqueAddress(); - publishWitnessIdentities( + recordWitnessMetrics( Collections.singletonList(miner(address, address))); Assert.assertEquals(1.0, witnessGauge(encode(address)), 0.0); } @@ -303,7 +302,7 @@ public void testWitnessAddressMultipleMiners() { miners.add(miner(address1, address1)); miners.add(miner(address2, address2)); - publishWitnessIdentities(miners); + recordWitnessMetrics(miners); Assert.assertEquals(1.0, witnessGauge(encode(address1)), 0.0); Assert.assertEquals(1.0, witnessGauge(encode(address2)), 0.0); @@ -319,7 +318,7 @@ public void testWitnessAddressUsesWitnessAccountNotSigningKey() { ByteString signingKeyAddress = addressOf(PublicMethod.getRandomPrivateKey()); ByteString witnessAccountAddress = uniqueAddress(); - publishWitnessIdentities( + recordWitnessMetrics( Collections.singletonList(miner(signingKeyAddress, witnessAccountAddress))); Assert.assertEquals(1.0, witnessGauge(encode(witnessAccountAddress)), 0.0); @@ -332,7 +331,7 @@ public void testWitnessAddressUsesWitnessAccountNotSigningKey() { */ @Test public void testNoWitnessAddressSeriesForEmptyMiners() { - publishWitnessIdentities(new ArrayList<>()); + recordWitnessMetrics(new ArrayList<>()); Assert.assertNull(witnessGauge("")); } @@ -344,7 +343,7 @@ public void testWitnessAddressNotPublishedWhenMetricsDisabled() { ByteString address = uniqueAddress(); Args.getInstance().setMetricsPrometheusEnable(false); try { - publishWitnessIdentities( + recordWitnessMetrics( Collections.singletonList(miner(address, address))); Assert.assertNull(witnessGauge(encode(address))); } finally { @@ -373,11 +372,12 @@ private void markBackupInitiated(BackupManager manager) throws Exception { } /** - * Drives the private startup hook directly; production publishes once from start(), - * the test exercises the same loop without starting consensus. + * Drives the startup publication the way production does: the miners are assembled first and + * the maintenance manager publishes once from init(). */ - private void publishWitnessIdentities(List miners) { - ReflectionTestUtils.invokeMethod(consensusService, "publishWitnessIdentities", miners); + private void recordWitnessMetrics(List miners) { + Param.getInstance().setMiners(miners); + maintenanceManager.recordWitnessMetrics(); } /** diff --git a/framework/src/test/java/org/tron/core/consensus/ConsensusServiceMetricsTest.java b/framework/src/test/java/org/tron/core/consensus/ConsensusServiceMetricsTest.java deleted file mode 100644 index 078346dc40a..00000000000 --- a/framework/src/test/java/org/tron/core/consensus/ConsensusServiceMetricsTest.java +++ /dev/null @@ -1,239 +0,0 @@ -package org.tron.core.consensus; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.List; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.MockSettings; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.springframework.test.util.ReflectionTestUtils; -import org.tron.common.args.GenesisBlock; -import org.tron.common.crypto.SignUtils; -import org.tron.common.parameter.CommonParameter; -import org.tron.common.prometheus.SRMetrics; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.LocalWitnesses; -import org.tron.common.utils.StringUtil; -import org.tron.consensus.Consensus; -import org.tron.consensus.base.BlockHandle; -import org.tron.consensus.base.Param; -import org.tron.consensus.base.Param.Miner; -import org.tron.consensus.base.PbftInterface; -import org.tron.core.capsule.WitnessCapsule; -import org.tron.core.config.args.Args; -import org.tron.core.store.WitnessStore; - -/** - * Startup integration tests for the witness-identity publication added to - * {@link ConsensusService#start()}. Pure JUnit4 + Mockito: no Spring, no real database, no - * {@code BaseTest}. SRMetrics is static-mocked so nothing touches the global Prometheus registry. - */ -public class ConsensusServiceMetricsTest { - - // Two valid, distinct ECDSA private keys (scalars 1 and 2). - private static final String KEY_ONE = - "0000000000000000000000000000000000000000000000000000000000000001"; - private static final String KEY_TWO = - "0000000000000000000000000000000000000000000000000000000000000002"; - - private ConsensusService service; - private Consensus consensus; - private WitnessStore witnessStore; - private BlockHandleImpl blockHandle; - private PbftBaseImpl pbftBaseImpl; - - // Singleton state saved so each test leaves the process as it found it. - private LocalWitnesses savedWitnesses; - private boolean savedMetricsEnabled; - private boolean savedEnable; - private boolean savedNeedSyncCheck; - private int savedMinParticipationRate; - private int savedBlockProduceTimeoutPercent; - private GenesisBlock savedGenesisBlock; - private List savedMiners; - private BlockHandle savedBlockHandle; - private int savedAgreeNodeCount; - private PbftInterface savedPbftInterface; - - @Before - public void setUp() throws Exception { - savedWitnesses = Args.getLocalWitnesses(); - savedMetricsEnabled = CommonParameter.getInstance().isMetricsPrometheusEnable(); - - Param param = Param.getInstance(); - savedEnable = param.isEnable(); - savedNeedSyncCheck = param.isNeedSyncCheck(); - savedMinParticipationRate = param.getMinParticipationRate(); - savedBlockProduceTimeoutPercent = param.getBlockProduceTimeoutPercent(); - savedGenesisBlock = param.getGenesisBlock(); - savedMiners = param.getMiners(); - savedBlockHandle = param.getBlockHandle(); - savedAgreeNodeCount = param.getAgreeNodeCount(); - savedPbftInterface = param.getPbftInterface(); - - consensus = mock(Consensus.class); - witnessStore = mock(WitnessStore.class); - blockHandle = mock(BlockHandleImpl.class); - pbftBaseImpl = mock(PbftBaseImpl.class); - - service = new ConsensusService(); - inject(service, "consensus", consensus); - inject(service, "witnessStore", witnessStore); - inject(service, "blockHandle", blockHandle); - inject(service, "pbftBaseImpl", pbftBaseImpl); - } - - @After - public void tearDown() { - Args.setLocalWitnesses(savedWitnesses); - CommonParameter.getInstance().setMetricsPrometheusEnable(savedMetricsEnabled); - - Param param = Param.getInstance(); - param.setEnable(savedEnable); - param.setNeedSyncCheck(savedNeedSyncCheck); - param.setMinParticipationRate(savedMinParticipationRate); - param.setBlockProduceTimeoutPercent(savedBlockProduceTimeoutPercent); - param.setGenesisBlock(savedGenesisBlock); - param.setMiners(savedMiners); - param.setBlockHandle(savedBlockHandle); - param.setAgreeNodeCount(savedAgreeNodeCount); - param.setPbftInterface(savedPbftInterface); - } - - @Test - public void singleKeyPublishesWitnessAccountNotSigningKey() throws Exception { - byte[] signingKey = derivedAddress(KEY_ONE); - byte[] witnessAccount = derivedAddress(KEY_TWO); - LocalWitnesses witnesses = witnesses(KEY_ONE); - witnesses.initWitnessAccountAddress(witnessAccount); - Args.setLocalWitnesses(witnesses); - CommonParameter.getInstance().setMetricsPrometheusEnable(true); - when(witnessStore.get(witnessAccount)).thenReturn(mock(WitnessCapsule.class)); - - try (MockedStatic sr = mockStatic(SRMetrics.class)) { - service.start(); - - sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(witnessAccount))); - sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(signingKey)), - never()); - } - verify(consensus).start(Mockito.any(Param.class)); - verify(witnessStore).get(witnessAccount); - verifyNoMoreInteractions(witnessStore); - } - - @Test - public void multiKeyPublishesEveryWitnessAddress() throws Exception { - Args.setLocalWitnesses(witnesses(KEY_ONE, KEY_TWO)); - CommonParameter.getInstance().setMetricsPrometheusEnable(true); - byte[] addrOne = derivedAddress(KEY_ONE); - byte[] addrTwo = derivedAddress(KEY_TWO); - when(witnessStore.get(addrOne)).thenReturn(mock(WitnessCapsule.class)); - when(witnessStore.get(addrTwo)).thenReturn(mock(WitnessCapsule.class)); - - try (MockedStatic sr = mockStatic(SRMetrics.class)) { - service.start(); - - sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(addrOne))); - sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(addrTwo))); - } - verify(consensus).start(Mockito.any(Param.class)); - verify(witnessStore).get(addrOne); - verify(witnessStore).get(addrTwo); - verifyNoMoreInteractions(witnessStore); - } - - @Test - public void noKeysPublishesNothingAndQueriesNoWitness() throws Exception { - Args.setLocalWitnesses(new LocalWitnesses()); - CommonParameter.getInstance().setMetricsPrometheusEnable(true); - - try (MockedStatic sr = mockStatic(SRMetrics.class)) { - service.start(); - sr.verifyNoInteractions(); - } - verify(consensus).start(Mockito.any(Param.class)); - verifyNoInteractions(witnessStore); - } - - @Test - public void disabledMetricsDoesNotPublish() throws Exception { - byte[] witnessAccount = derivedAddress(KEY_TWO); - LocalWitnesses witnesses = witnesses(KEY_ONE); - witnesses.initWitnessAccountAddress(witnessAccount); - Args.setLocalWitnesses(witnesses); - CommonParameter.getInstance().setMetricsPrometheusEnable(false); - when(witnessStore.get(witnessAccount)).thenReturn(mock(WitnessCapsule.class)); - - try (MockedStatic sr = mockStatic(SRMetrics.class)) { - service.start(); - sr.verifyNoInteractions(); - } - verify(consensus).start(Mockito.any(Param.class)); - verify(witnessStore).get(witnessAccount); - verifyNoMoreInteractions(witnessStore); - } - - @Test - public void metricsFailureDoesNotAbortStart() throws Exception { - Args.setLocalWitnesses(witnesses(KEY_ONE, KEY_TWO)); - CommonParameter.getInstance().setMetricsPrometheusEnable(true); - byte[] addrOne = derivedAddress(KEY_ONE); - byte[] addrTwo = derivedAddress(KEY_TWO); - when(witnessStore.get(addrOne)).thenReturn(mock(WitnessCapsule.class)); - when(witnessStore.get(addrTwo)).thenReturn(mock(WitnessCapsule.class)); - - MockSettings throwing = Mockito.withSettings().defaultAnswer(invocation -> { - throw new RuntimeException("metrics unavailable"); - }); - try (MockedStatic sr = Mockito.mockStatic(SRMetrics.class, throwing)) { - service.start(); - - // The per-miner try/catch must keep the loop going after the first failure. - sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(addrOne))); - sr.verify(() -> SRMetrics.recordWitnessAddress(StringUtil.encode58Check(addrTwo))); - } - verify(consensus).start(Mockito.any(Param.class)); - verify(witnessStore).get(addrOne); - verify(witnessStore).get(addrTwo); - verifyNoMoreInteractions(witnessStore); - } - - @Test - public void nullWitnessAddressIsSkipped() throws Exception { - CommonParameter.getInstance().setMetricsPrometheusEnable(true); - Miner miner = Param.getInstance().new Miner(null, null, null); - - try (MockedStatic sr = mockStatic(SRMetrics.class)) { - ReflectionTestUtils.invokeMethod(service, "publishWitnessIdentities", Arrays.asList(miner)); - sr.verifyNoInteractions(); - } - } - - private static LocalWitnesses witnesses(String... keys) { - LocalWitnesses witnesses = new LocalWitnesses(); - witnesses.setPrivateKeys(Arrays.asList(keys)); - return witnesses; - } - - private static byte[] derivedAddress(String privateKey) { - return SignUtils.fromPrivate(ByteArray.fromHexString(privateKey)).getAddress(); - } - - private static void inject(Object target, String name, Object value) throws Exception { - Field field = target.getClass().getDeclaredField(name); - field.setAccessible(true); - field.set(target, value); - } -} From 4f42f076e30b71e84ef80bc1a48b9ea02f2748c8 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 23 Oct 2026 12:03:09 +0800 Subject: [PATCH 20/26] refactor(metrics): publish witness address from consensus startup --- .../consensus/dpos/MaintenanceManager.java | 32 ------------------- .../tron/core/consensus/ConsensusService.java | 22 +++++++++++++ .../tron/common/prometheus/SRMetricsTest.java | 7 ++-- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index 6dca590fc68..fd5e4364d0d 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -16,12 +16,8 @@ import org.bouncycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; -import org.tron.common.utils.StringUtil; import org.tron.consensus.ConsensusDelegate; -import org.tron.consensus.base.Param; -import org.tron.consensus.base.Param.Miner; import org.tron.consensus.pbft.PbftManager; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.BlockCapsule; @@ -56,8 +52,6 @@ public class MaintenanceManager { public void init() { currentWitness.addAll(consensusDelegate.getActiveWitnesses()); - // miners are assembled by ConsensusService before DposService.start() calls init() - recordWitnessMetrics(); } public void applyBlock(BlockCapsule blockCapsule) { @@ -168,32 +162,6 @@ public void doMaintenance() { } } - /** - * Publishes one tron:witness_address series per consensus miner. The witness account address is - * immutable, so this runs once from {@link #init()} and is never refreshed. - */ - public void recordWitnessMetrics() { - if (!Metrics.enabled()) { - return; - } - List miners = Param.getInstance().getMiners(); - if (miners == null) { - // miners are not assembled yet - return; - } - for (Miner miner : miners) { - if (miner.getWitnessAddress() == null) { - continue; - } - try { - String address = StringUtil.encode58Check(miner.getWitnessAddress().toByteArray()); - SRMetrics.recordWitnessAddress(address); - } catch (Exception e) { - logger.warn("Failed to publish witness address metrics", e); - } - } - } - private Map countVote(VotesStore votesStore) { final Map countWitness = Maps.newHashMap(); Iterator> dbIterator = votesStore.iterator(); diff --git a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java index b3921b1b657..775a37655c1 100644 --- a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java +++ b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java @@ -11,6 +11,9 @@ import org.springframework.stereotype.Component; import org.tron.common.crypto.SignUtils; import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.Metrics; +import org.tron.common.prometheus.SRMetrics; +import org.tron.common.utils.StringUtil; import org.tron.consensus.Consensus; import org.tron.consensus.base.Param; import org.tron.consensus.base.Param.Miner; @@ -77,12 +80,31 @@ public void start() { } param.setMiners(miners); + recordWitnessAddresses(miners); param.setBlockHandle(blockHandle); param.setPbftInterface(pbftBaseImpl); consensus.start(param); logger.info("consensus service start success"); } + private void recordWitnessAddresses(List miners) { + if (!Metrics.enabled() || miners == null) { + return; + } + for (Miner miner : miners) { + ByteString witnessAddress = miner.getWitnessAddress(); + if (witnessAddress == null) { + continue; + } + try { + String address = StringUtil.encode58Check(witnessAddress.toByteArray()); + SRMetrics.recordWitnessAddress(address); + } catch (Exception e) { + logger.warn("Failed to record witness address metrics.", e); + } + } + } + public void stop() { logger.info("consensus service closed start."); consensus.stop(); diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 4ced605c229..13e388d11ef 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -13,6 +13,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; import org.tron.common.BaseTest; import org.tron.common.TestConstants; import org.tron.common.backup.BackupManager; @@ -372,12 +373,12 @@ private void markBackupInitiated(BackupManager manager) throws Exception { } /** - * Drives the startup publication the way production does: the miners are assembled first and - * the maintenance manager publishes once from init(). + * Drives the startup publication the way production does: the miners are assembled by + * ConsensusService, which publishes the witness identities once during startup. */ private void recordWitnessMetrics(List miners) { Param.getInstance().setMiners(miners); - maintenanceManager.recordWitnessMetrics(); + ReflectionTestUtils.invokeMethod(consensusService, "recordWitnessAddresses", miners); } /** From fa332d3c2b2d6f864e8f5579dc2ade628532004c Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 23 Oct 2026 16:00:04 +0800 Subject: [PATCH 21/26] fix(metrics): separate node identity from backup role --- .../tron/common/prometheus/MetricKeys.java | 4 +- .../tron/common/prometheus/MetricLabels.java | 2 +- .../tron/common/prometheus/MetricsGauge.java | 6 +- .../org/tron/common/prometheus/SRMetrics.java | 21 +- docs/metrics-changelog.md | 3 +- .../org/tron/common/backup/BackupManager.java | 9 +- .../tron/core/consensus/ConsensusService.java | 19 ++ .../tron/common/backup/BackupManagerTest.java | 63 +++++ .../tron/common/prometheus/SRMetricsTest.java | 262 +++++++++++++++--- .../prometheus/PrometheusApiServiceTest.java | 37 ++- 10 files changed, 363 insertions(+), 63 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 7667291efc0..53d0a4f2714 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -42,8 +42,10 @@ public static class Gauge { public static final String TX_CACHE = "tron:tx_cache"; // per local consensus witness miner; value 1, label is the Base58Check witness address public static final String WITNESS_ADDRESS = "tron:witness_address"; - // numeric role of this node: 0 = fullnode, 1 = init, 2 = master, 3 = slaver + // numeric backup-election role of this node: 0 = init, 1 = master, 2 = slaver public static final String NODE_ROLE = "tron:node_role"; + // numeric identity of this node: 0 = fullnode, 1 = witness + public static final String NODE_TYPE = "tron:node_type"; private Gauge() { throw new IllegalStateException("Gauge"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index 8d004cf5829..858e06c5b02 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -62,7 +62,7 @@ public static class Gauge { public static final String PEERS_VALID = "valid"; - // ADDRESS is the label of tron:witness_address (node_role carries no label) + // ADDRESS is the label of tron:witness_address (node_role and node_type carry no label) public static final String ADDRESS = "address"; private Gauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index 3114c3d3bad..88196d70eb3 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -22,9 +22,11 @@ class MetricsGauge { init(MetricKeys.Gauge.WITNESS_ADDRESS, "1 for each consensus witness address this node holds a miner for.", MetricLabels.Gauge.ADDRESS); + init(MetricKeys.Gauge.NODE_TYPE, + "Type of this node, encoded as a number: 0 = fullnode, 1 = witness."); init(MetricKeys.Gauge.NODE_ROLE, - "Role of this node, encoded as a number: 0 = fullnode, 1 = init, " - + "2 = master, 3 = slaver."); + "Backup-election role of this node, encoded as a number: 0 = init, " + + "1 = master, 2 = slaver."); } private MetricsGauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 311ff679358..3d968f0f40a 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -10,11 +10,22 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } - // node_role: 0 = fullnode, 1 = init (pre-election / re-election), 2 = master, 3 = slaver - public static final int NODE_ROLE_FULLNODE = 0; - public static final int NODE_ROLE_INIT = 1; - public static final int NODE_ROLE_MASTER = 2; - public static final int NODE_ROLE_SLAVER = 3; + // node_type: 0 = fullnode, 1 = witness + public static final int NODE_TYPE_FULLNODE = 0; + public static final int NODE_TYPE_WITNESS = 1; + + // node_role: 0 = init (pre-election / re-election), 1 = master, 2 = slaver + public static final int NODE_ROLE_INIT = 0; + public static final int NODE_ROLE_MASTER = 1; + public static final int NODE_ROLE_SLAVER = 2; + + /** Publishes node_type as one label-less series; 0 = fullnode, 1 = witness. */ + public static void recordNodeType(int nodeType) { + if (!Metrics.enabled()) { + return; + } + Metrics.gaugeSet(MetricKeys.Gauge.NODE_TYPE, nodeType); + } /** Publishes node_role as one label-less series; role changes rewrite it in place. */ public static void recordNodeRole(int role) { diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 10b3353bb10..77f3e377847 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -10,7 +10,8 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:node_role` (Gauge, no labels) — role of this node, encoded as a number: `0` = `fullnode` (started without witness mode; witness keys, if any, are not loaded on such a node and no `tron:witness_address` series is published), `1` = `init` (from node startup until the backup election first runs, and again while a re-election is running after the master lost keep-alive), `2` = `master` (holds the primary witness key; also the value when backup is not configured), `3` = `slaver` (standing by for a master). The role is a node-level status, not a per-identity one, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_type` (Gauge, no labels) — identity of this node, encoded as a number: `0` = `fullnode` (started without witness mode; witness keys are not loaded and no `tron:witness_address` series is published), `1` = `witness` (started with `-w`). It is a process identity, published once at startup and independent of the backup-election role. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_role` (Gauge, no labels) — backup-election role of this node, encoded as a number: `0` = `init` (backup election configured but not yet run, and again while a re-election is running after the master lost keep-alive), `1` = `master` (the backup primary; also the standalone default for both node types when backup is not configured), `2` = `slaver` (standing by for a master). The role is a node-level status, independent of `tron:node_type`, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:witness_address` (Gauge, label `address`) — value `1` for each consensus witness miner this node holds; the label is the Base58Check-encoded witness account address. The set is assembled once at startup from the finalized consensus miners: one series per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. With several witness keys, each address gets its own series. A plain fullnode without a witness miner publishes no `tron:witness_address` series at all. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 4326f3e4bbd..75b299687e8 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -28,7 +28,6 @@ import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.Metrics; import org.tron.common.prometheus.SRMetrics; -import org.tron.core.config.args.Args; import org.tron.p2p.utils.NetUtil; @Slf4j(topic = "backup") @@ -74,7 +73,7 @@ public void setStatus(BackupStatusEnum status) { recordNodeRole(); } - /** Publishes tron:node_role, once at startup and on every backup-status change. */ + /** Publishes tron:node_role once at startup, and again on every backup-status change. */ @PostConstruct public void recordNodeRole() { if (!Metrics.enabled()) { @@ -89,11 +88,7 @@ public void recordNodeRole() { } private int currentRole() { - // started without -w: a plain fullnode; witness keys are not even loaded - if (!Args.getInstance().isWitness()) { - return SRMetrics.NODE_ROLE_FULLNODE; - } - // no backup election to take part in: this node holds the primary key itself + // no backup configured: default to master, for either identity if (!isBackupConfigured()) { return SRMetrics.NODE_ROLE_MASTER; } diff --git a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java index 775a37655c1..28f1195ae64 100644 --- a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java +++ b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java @@ -81,6 +81,7 @@ public void start() { param.setMiners(miners); recordWitnessAddresses(miners); + recordNodeType(); param.setBlockHandle(blockHandle); param.setPbftInterface(pbftBaseImpl); consensus.start(param); @@ -105,6 +106,24 @@ private void recordWitnessAddresses(List miners) { } } + /** + * Publishes tron:node_type once at consensus startup; 0 = fullnode, 1 = witness. It is + * published independently of the witness-address publication, and even when no miner is + * configured, so the identity series always exists. Catches its own failure so a metrics + * problem can neither skip the witness publication nor abort consensus startup. + */ + private void recordNodeType() { + if (!Metrics.enabled()) { + return; + } + try { + SRMetrics.recordNodeType(Args.getInstance().isWitness() + ? SRMetrics.NODE_TYPE_WITNESS : SRMetrics.NODE_TYPE_FULLNODE); + } catch (Exception e) { + logger.warn("Failed to record node type metrics.", e); + } + } + public void stop() { logger.info("consensus service closed start."); consensus.stop(); diff --git a/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java b/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java index cd2a18ac594..b6326c72929 100644 --- a/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java +++ b/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java @@ -1,6 +1,7 @@ package org.tron.common.backup; import io.netty.channel.Channel; +import io.prometheus.client.CollectorRegistry; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -20,12 +21,16 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import org.tron.common.TestConstants; import org.tron.common.backup.BackupManager.BackupStatusEnum; import org.tron.common.backup.message.KeepAliveMessage; import org.tron.common.backup.socket.BackupServer; +import org.tron.common.backup.socket.MessageHandler; import org.tron.common.backup.socket.UdpEvent; import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.SRMetrics; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.ReflectUtils; import org.tron.consensus.base.Param; @@ -187,6 +192,64 @@ public void testBackupServerLifecycleDuringKeepAliveInterval() throws Exception BackupTestUtils.assertExecutorsTerminated(manager, backupServer); } + /** + * A backup group whose only member is this node's own address is a "single-master" group: + * no peer ever answers a keep-alive, so the node must leave the pre-election INIT window and + * self-promote to MASTER once the keep-alive timeout elapses. + * + *

Drives the real {@link BackupManager#init()} and its real scheduled keep-alive task. The + * timeout condition is made deterministic by ageing the last keep-alive time, so the first + * (fixed 1s) scheduler tick is guaranteed to observe an expired keep-alive; recovery is then + * awaited by polling, so there is no fixed sleep. The published {@code tron:node_role} must + * track the transition (0 = init, 1 = master) and the manager executor must terminate. + */ + @Test(timeout = 30000) + public void singleMasterGroupRecoversFromInitToMaster() throws Exception { + CommonParameter parameter = CommonParameter.getInstance(); + parameter.setBackupPriority(8); + // the group's only member is ourselves, so init() drops it and no peer can answer + parameter.setBackupMembers( + Collections.singletonList(InetAddress.getLocalHost().getHostAddress())); + + boolean previousMetrics = Args.getInstance().isMetricsPrometheusEnable(); + Args.getInstance().setMetricsPrometheusEnable(true); + BackupManager single = new BackupManager(); + // absorb any keep-alive send in case the local IP lookup behaves differently + single.setMessageHandler(Mockito.mock(MessageHandler.class)); + try { + // backup configured but not yet init'ed: the pre-election window reports init (0) + Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, single.getStatus()); + single.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, nodeRoleGauge(), 0.0); + + single.init(); + Assert.assertEquals(BackupManager.BackupStatusEnum.INIT, single.getStatus()); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, nodeRoleGauge(), 0.0); + + // no keep-alive has ever arrived, so the next keep-alive tick must self-promote. + // setStatus() flips the volatile status before it republishes the gauge, so wait on + // the published series (the externally visible result), not on the status field alone. + ReflectUtils.setFieldValue(single, "lastKeepAliveTime", 0L); + BackupTestUtils.awaitCondition("lone master to recover from INIT", + () -> single.getStatus() == BackupManager.BackupStatusEnum.MASTER + && Double.valueOf(SRMetrics.NODE_ROLE_MASTER).equals(nodeRoleGauge())); + Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, single.getStatus()); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, nodeRoleGauge(), 0.0); + } finally { + single.stop(); + Args.getInstance().setMetricsPrometheusEnable(previousMetrics); + } + + ScheduledExecutorService executorService = + (ScheduledExecutorService) ReflectUtils.getFieldObject(single, "executorService"); + Assert.assertTrue("single-master executor must terminate", executorService.isTerminated()); + } + + private Double nodeRoleGauge() { + return CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_ROLE, new String[] {}, new String[] {}); + } + // ===== domain-handling tests for init() ===== @Test(timeout = 5000) diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 13e388d11ef..c842f24b3b2 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -2,6 +2,8 @@ import com.google.protobuf.ByteString; import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.common.TextFormat; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -13,6 +15,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.springframework.test.util.ReflectionTestUtils; import org.tron.common.BaseTest; import org.tron.common.TestConstants; @@ -183,83 +187,221 @@ public void testSrRemoveDirect() { Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, stableWit)); } - // ---- node_role: the four states, the pre-election window and status transitions ---- + // ---- tron:node_type: process identity, published at startup and independent of the role ---- /** - * Started without witness mode: a plain fullnode reports role 0. + * Identity is published once at startup by ConsensusService: 0 for a node started without + * witness mode, 1 for a node started with -w. */ @Test - public void testNodeRoleFullNode() { + public void testNodeTypeBothIdentities() { + for (boolean witness : new boolean[] {false, true}) { + Args.getInstance().witness = witness; + publishNodeType(); + int expected = witness ? SRMetrics.NODE_TYPE_WITNESS : SRMetrics.NODE_TYPE_FULLNODE; + Assert.assertEquals(expected, nodeTypeGauge(), 0.0); + } + } + + /** + * Metrics disabled: recordNodeType() short-circuits, so the previously published value stands + * even when the identity would have changed. + */ + @Test + public void testNodeTypeNotPublishedWhenMetricsDisabled() { Args.getInstance().witness = false; - backupManager.recordNodeRole(); - Assert.assertEquals(SRMetrics.NODE_ROLE_FULLNODE, roleGauge(), 0.0); + publishNodeType(); + Assert.assertEquals(SRMetrics.NODE_TYPE_FULLNODE, nodeTypeGauge(), 0.0); + + Args.getInstance().setMetricsPrometheusEnable(false); + try { + Args.getInstance().witness = true; + publishNodeType(); + Assert.assertEquals(SRMetrics.NODE_TYPE_FULLNODE, nodeTypeGauge(), 0.0); + } finally { + Args.getInstance().setMetricsPrometheusEnable(true); + } } /** - * Witness configured without a backup election (backup members unset): the node always - * holds the primary key itself, so the single role series reports master (2). + * Metrics disabled: recordNodeRole() short-circuits, so the previously published role stands + * even when the backup configuration would resolve to a different one. */ @Test - public void testNodeRoleMaster() { - Args.getInstance().witness = true; + public void testNodeRoleNotPublishedWhenMetricsDisabled() { + Args.getInstance().witness = false; + // no backup configured: baseline master (1) backupManager.recordNodeRole(); Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + + // a configuration that would resolve to init (0) if it were published + Args.getInstance().setBackupPort(PublicMethod.chooseRandomPort()); + Args.getInstance().setBackupMembers(Collections.singletonList("127.0.0.2")); + Args.getInstance().setMetricsPrometheusEnable(false); + try { + backupManager.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + } finally { + Args.getInstance().setMetricsPrometheusEnable(true); + Args.getInstance().setBackupPort(0); + Args.getInstance().setBackupMembers(Collections.emptyList()); + } } /** - * Backup election slaver: the node stands by for a master and reports slaver (3). + * A failure while publishing node_type must not propagate: ConsensusService catches it, so it + * can neither skip the witness-address publication running right beside it nor abort startup. */ @Test - public void testNodeRoleSlaver() throws Exception { - Args.getInstance().witness = true; - configureBackup(); - markBackupInitiated(backupManager); - backupManager.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); + public void testNodeTypeFailureDoesNotBlockWitnessPublication() { + Args.getInstance().witness = false; + ByteString address = uniqueAddress(); + List miners = Collections.singletonList(miner(address, address)); + + try (MockedStatic mocked = Mockito.mockStatic(SRMetrics.class)) { + mocked.when(() -> SRMetrics.recordNodeType(Mockito.anyInt())) + .thenThrow(new RuntimeException("node_type failure")); + mocked.when(() -> SRMetrics.recordWitnessAddress(Mockito.anyString())) + .thenCallRealMethod(); + + // must not propagate the node_type failure + publishNodeType(); + + // the witness-address publication still runs independently on the same startup path + ReflectionTestUtils.invokeMethod(consensusService, "recordWitnessAddresses", miners); + } + + Assert.assertEquals(1.0, witnessGauge(encode(address)), 0.0); } + // ---- tron:node_role: the three states, the pre-election window and status transitions ---- + /** - * Before the first election round (backup configured, BackupManager.init() not yet run, - * which in production only happens after the first fresh block) the node must report - * init (1) instead of falling back to the MASTER field default; the first real status - * after init() rewrites the same single series to the elected role. + * No backup configured: both identities report master (1) as the standalone default; the + * backup status defaults to MASTER before any election runs. */ @Test - public void testNodeRoleInitBeforeBackupElectionRuns() throws Exception { + public void testNodeRoleMasterWhenNoBackupBothIdentities() { + Args.getInstance().witness = false; + backupManager.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + Args.getInstance().witness = true; - configureBackup(); + backupManager.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + } - // a directly constructed manager has not gone through init(): pre-election window - BackupManager fresh = new BackupManager(); - try { - fresh.recordNodeRole(); - Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + /** + * A partial backup configuration — a usable port without members, or members without a usable + * port — is not an election setup (the same condition BackupServer.initServer() uses), so the + * node reports master (1) for both identities and no transport is started. + */ + @Test + public void testNodeRoleMasterWhenBackupPartiallyConfigured() { + for (boolean witness : new boolean[] {false, true}) { + for (boolean portUsable : new boolean[] {false, true}) { + Args.getInstance().witness = witness; + Args.getInstance().setBackupPort(portUsable ? PublicMethod.chooseRandomPort() : 0); + Args.getInstance().setBackupMembers(portUsable + ? Collections.emptyList() : Collections.singletonList("127.0.0.2")); + + // one half of the pair is always unset, so this is never a usable election config + BackupManager fresh = new BackupManager(); + try { + fresh.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, roleGauge(), 0.0); + Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, fresh.getStatus()); + } finally { + fresh.stop(); + } + } + } + } - // a status change while init() has not run stays inside the window - fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + /** + * Backup configured but the election has not run yet: even though the status field still + * defaults to MASTER, the node reports init (0), for both identities; a status change before + * init() stays inside the window, and after init() the status maps to the elected role. + */ + @Test + public void testNodeRoleInitBeforeBackupElectionBothIdentities() throws Exception { + configureBackup(); - // once the election has run, the status maps to the elected role - markBackupInitiated(fresh); - fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); - Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); - } finally { - fresh.stop(); + for (boolean witness : new boolean[] {false, true}) { + BackupManager fresh = new BackupManager(); + try { + Args.getInstance().witness = witness; + + // a directly constructed manager has not gone through init(): default status is MASTER + Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, fresh.getStatus()); + fresh.recordNodeRole(); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + + // a status change before init() stays inside the window + fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + + // once the election has run, the status maps to the elected role + markBackupInitiated(fresh); + fresh.setStatus(BackupManager.BackupStatusEnum.SLAVER); + Assert.assertEquals(SRMetrics.NODE_ROLE_SLAVER, roleGauge(), 0.0); + } finally { + fresh.stop(); + } } } /** - * After init() has run, status INIT is a real election state — the transport is up but no - * keep-alive timeout has been judged yet, or the master lost keep-alive and a re-election - * is running — and maps 1:1 to init (1). It must not be misreported as slaver (3). + * Identity and role are fully independent: for both identities the startup publication is + * fullnode/witness + init, and the role then cycles INIT -> MASTER -> SLAVER -> MASTER while + * node_type never changes. Every publication rewrites the single label-less series in place, + * so no duplicate registration or stale role series accumulates. */ @Test - public void testNodeRoleInitAfterElectionStarted() throws Exception { - Args.getInstance().witness = true; + public void testNodeTypeIndependentOfRoleBothIdentities() throws Exception { configureBackup(); - markBackupInitiated(backupManager); - backupManager.setStatus(BackupManager.BackupStatusEnum.INIT); - Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + + BackupManager.BackupStatusEnum[] statuses = { + BackupManager.BackupStatusEnum.INIT, + BackupManager.BackupStatusEnum.MASTER, + BackupManager.BackupStatusEnum.SLAVER, + BackupManager.BackupStatusEnum.MASTER}; + int[] roles = { + SRMetrics.NODE_ROLE_INIT, + SRMetrics.NODE_ROLE_MASTER, + SRMetrics.NODE_ROLE_SLAVER, + SRMetrics.NODE_ROLE_MASTER}; + + for (boolean witness : new boolean[] {false, true}) { + BackupManager fresh = new BackupManager(); + try { + Args.getInstance().witness = witness; + int expectedType = witness ? SRMetrics.NODE_TYPE_WITNESS : SRMetrics.NODE_TYPE_FULLNODE; + + // startup: node_type comes from ConsensusService, node_role from BackupManager; + // configured-but-not-init is INIT, not the MASTER default + publishNodeType(); + fresh.recordNodeRole(); + Assert.assertEquals(expectedType, nodeTypeGauge(), 0.0); + Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); + + // flip the startup identity: setStatus() must only republish the role, never recompute + // node_type, so the originally published identity has to keep standing + Args.getInstance().witness = !witness; + markBackupInitiated(fresh); + for (int i = 0; i < statuses.length; i++) { + fresh.setStatus(statuses[i]); + Assert.assertEquals(roles[i], roleGauge(), 0.0); + Assert.assertEquals("identity must not change with the role", expectedType, + nodeTypeGauge(), 0.0); + } + + Assert.assertEquals(1, sampleCount(MetricKeys.Gauge.NODE_ROLE)); + Assert.assertEquals(1, sampleCount(MetricKeys.Gauge.NODE_TYPE)); + } finally { + fresh.stop(); + } + } } /** @@ -372,6 +514,14 @@ private void markBackupInitiated(BackupManager manager) throws Exception { isInit.setBoolean(manager, true); } + /** + * Drives the startup identity publication the way production does: ConsensusService.start() + * calls its private recordNodeType() hook right after assembling the miners. + */ + private void publishNodeType() { + ReflectionTestUtils.invokeMethod(consensusService, "recordNodeType"); + } + /** * Drives the startup publication the way production does: the miners are assembled by * ConsensusService, which publishes the witness identities once during startup. @@ -412,6 +562,30 @@ private Double roleGauge() { new String[] {}); } + private Double nodeTypeGauge() { + return CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_TYPE, + new String[] {}, + new String[] {}); + } + + /** Counts the exported sample lines of a label-less metric, to detect duplicate registration. */ + private int sampleCount(String metricName) { + StringWriter writer = new StringWriter(); + try { + TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples()); + } catch (Exception e) { + throw new IllegalStateException(e); + } + int count = 0; + for (String line : writer.toString().split("\n")) { + if (line.startsWith(metricName + " ") || line.startsWith(metricName + "{")) { + count++; + } + } + return count; + } + private ByteString registerWitness() { ByteString address = uniqueAddress(); chainBaseManager.getWitnessStore().put(address.toByteArray(), new WitnessCapsule(address)); diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index e5763d5ffc2..72578d5c0ed 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -3,6 +3,8 @@ import com.google.common.collect.Maps; import com.google.protobuf.ByteString; import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.common.TextFormat; +import java.io.StringWriter; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -227,8 +229,8 @@ public void testNodeInfoMetric() { @Test public void testNodeRoleMetric() { - // a single label-less series per node; the value encodes the role (0 = fullnode, - // 1 = init, 2 = master, 3 = slaver), here master after recording NODE_ROLE_MASTER + // a single label-less series per node; the value encodes the role (0 = init, + // 1 = master, 2 = slaver), here master after recording NODE_ROLE_MASTER SRMetrics.recordNodeRole(SRMetrics.NODE_ROLE_MASTER); Double value = CollectorRegistry.defaultRegistry.getSampleValue( MetricKeys.Gauge.NODE_ROLE, @@ -238,6 +240,37 @@ public void testNodeRoleMetric() { Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, value, 0.0); } + @Test + public void testNodeTypeMetric() { + // label-less identity series: 0 = fullnode, 1 = witness + SRMetrics.recordNodeType(SRMetrics.NODE_TYPE_WITNESS); + Double value = CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Gauge.NODE_TYPE, + new String[] {}, + new String[] {}); + Assert.assertNotNull("tron:node_type sample should exist", value); + Assert.assertEquals(SRMetrics.NODE_TYPE_WITNESS, value, 0.0); + } + + @Test + public void testNodeGaugesTextExport() throws Exception { + SRMetrics.recordNodeType(SRMetrics.NODE_TYPE_FULLNODE); + SRMetrics.recordNodeRole(SRMetrics.NODE_ROLE_INIT); + + StringWriter writer = new StringWriter(); + TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples()); + String text = writer.toString(); + + Assert.assertTrue("scrape output should expose tron:node_type", + text.contains("tron:node_type ")); + Assert.assertTrue("scrape output should expose tron:node_role", + text.contains("tron:node_role ")); + Assert.assertTrue("tron:node_type should export the recorded value", + text.contains("tron:node_type 0.0")); + Assert.assertTrue("tron:node_role should export the recorded value", + text.contains("tron:node_role 0.0")); + } + @Test public void testNodeInfoUnknownKey() { // unknown key exercises the null-guard branch in MetricsInfo.set From 345d81ad59cf847b2f68cefe44270c4b8c949e85 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 23 Oct 2026 17:20:12 +0800 Subject: [PATCH 22/26] docs(metrics): restore unrelated comments --- .../src/main/java/org/tron/common/prometheus/MetricKeys.java | 4 +++- .../main/java/org/tron/common/prometheus/MetricLabels.java | 3 ++- framework/src/main/java/org/tron/program/FullNode.java | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 53d0a4f2714..fe11ae17fa4 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -14,7 +14,9 @@ public static class Counter { public static final String TXS = "tron:txs"; public static final String MINER = "tron:miner"; public static final String BLOCK_FORK = "tron:block_fork"; - // witness label: bounded cardinality (finite SR pool) for dashboard readability + // witness label: bounded cardinality -- SR candidate pool is finite, rotation is + // infrequent (at most once per maintenance interval); kept for at-a-glance SR + // identification in dashboards rather than requiring log cross-referencing. public static final String SR_SET_CHANGE = "tron:sr_set_change"; public static final String P2P_ERROR = "tron:p2p_error"; public static final String P2P_DISCONNECT = "tron:p2p_disconnect"; diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index 858e06c5b02..5fae9b54672 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -87,7 +87,8 @@ private Histogram() { // Info public static class Info { public static final String VERSION = "version"; - // identifies the genesis block: the value is the chain id from the genesis block hash + // identifies the genesis block: the label value is the chain id derived from the + // genesis block hash, so the label is named genesis_block_id public static final String CHAIN_ID = "genesis_block_id"; private Info() { diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 5f7f050540f..94d49fc43fd 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -71,7 +71,8 @@ public static void main(String[] args) { Application appT = ApplicationFactory.create(context); context.registerShutdownHook(); appT.startup(); - // the genesis block id (chainId) is only available after context refresh (Manager.initGenesis) + // the genesis block id (chainId) is only available after the context refresh + // (Manager.initGenesis) Metrics.info(MetricKeys.Info.NODE_INFO, Version.getVersion(), Args.getInstance().getChainId()); if (parameter.isSolidityNode()) { From 795eb43336b978620c18d04e95ace6dd24dbf71d Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 23 Oct 2026 17:44:28 +0800 Subject: [PATCH 23/26] refactor(metrics): remove redundant witness address null check --- .../main/java/org/tron/core/consensus/ConsensusService.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java index 28f1195ae64..48e6e2f4bd1 100644 --- a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java +++ b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java @@ -94,9 +94,6 @@ private void recordWitnessAddresses(List miners) { } for (Miner miner : miners) { ByteString witnessAddress = miner.getWitnessAddress(); - if (witnessAddress == null) { - continue; - } try { String address = StringUtil.encode58Check(witnessAddress.toByteArray()); SRMetrics.recordWitnessAddress(address); From 8b53bbf230dedda72cdde8cccea18915900d05f0 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 23 Oct 2026 18:00:26 +0800 Subject: [PATCH 24/26] test(metrics): cover node type publication during consensus startup --- .../tron/common/prometheus/SRMetricsTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index c842f24b3b2..292d1932475 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -25,6 +25,7 @@ import org.tron.common.utils.ByteArray; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.StringUtil; +import org.tron.consensus.Consensus; import org.tron.consensus.base.Param; import org.tron.consensus.dpos.MaintenanceManager; import org.tron.core.capsule.AccountCapsule; @@ -274,6 +275,33 @@ public void testNodeTypeFailureDoesNotBlockWitnessPublication() { Assert.assertEquals(1.0, witnessGauge(encode(address)), 0.0); } + /** + * The identity metric must be published by {@link ConsensusService#start()} itself, not by a + * caller: with the election engine stubbed out, a real start() has to write tron:node_type from + * Args.isWitness(), even for a node with zero miners. Pre-setting the opposite value pins the + * call site, so dropping the publication from start() fails here. + */ + @Test + public void testStartPublishesNodeTypeForZeroMiners() { + Consensus consensusMock = Mockito.mock(Consensus.class); + ConsensusService service = new ConsensusService(); + ReflectionTestUtils.setField(service, "consensus", consensusMock); + + for (boolean witness : new boolean[] {false, true}) { + Args.getInstance().witness = witness; + // opposite of the expected value, so a start() that skipped the publication cannot pass + SRMetrics.recordNodeType(witness ? SRMetrics.NODE_TYPE_FULLNODE + : SRMetrics.NODE_TYPE_WITNESS); + + service.start(); + + int expected = witness ? SRMetrics.NODE_TYPE_WITNESS : SRMetrics.NODE_TYPE_FULLNODE; + Assert.assertEquals(expected, nodeTypeGauge(), 0.0); + } + + Mockito.verify(consensusMock, Mockito.times(2)).start(Mockito.any(Param.class)); + } + // ---- tron:node_role: the three states, the pre-election window and status transitions ---- /** From 26a7004a70a7c0dc229672b1d4e41007202507e4 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 23 Oct 2026 19:49:34 +0800 Subject: [PATCH 25/26] refactor(metrics): rename node type gauge to is witness --- .../tron/common/prometheus/MetricKeys.java | 4 +- .../tron/common/prometheus/MetricLabels.java | 2 +- .../tron/common/prometheus/MetricsGauge.java | 5 +- .../org/tron/common/prometheus/SRMetrics.java | 12 +-- docs/metrics-changelog.md | 4 +- .../tron/core/consensus/ConsensusService.java | 14 +-- .../tron/common/prometheus/SRMetricsTest.java | 94 +++++++++---------- .../prometheus/PrometheusApiServiceTest.java | 22 ++--- 8 files changed, 79 insertions(+), 78 deletions(-) diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index fe11ae17fa4..be9dff5951c 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -46,8 +46,8 @@ public static class Gauge { public static final String WITNESS_ADDRESS = "tron:witness_address"; // numeric backup-election role of this node: 0 = init, 1 = master, 2 = slaver public static final String NODE_ROLE = "tron:node_role"; - // numeric identity of this node: 0 = fullnode, 1 = witness - public static final String NODE_TYPE = "tron:node_type"; + // witness flag of this node: 0 = not witness, 1 = witness + public static final String IS_WITNESS = "tron:is_witness"; private Gauge() { throw new IllegalStateException("Gauge"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index 5fae9b54672..f576bf7936b 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -62,7 +62,7 @@ public static class Gauge { public static final String PEERS_VALID = "valid"; - // ADDRESS is the label of tron:witness_address (node_role and node_type carry no label) + // ADDRESS is the label of tron:witness_address (node_role and is_witness carry no label) public static final String ADDRESS = "address"; private Gauge() { diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index 88196d70eb3..276d3cd89c2 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -22,8 +22,9 @@ class MetricsGauge { init(MetricKeys.Gauge.WITNESS_ADDRESS, "1 for each consensus witness address this node holds a miner for.", MetricLabels.Gauge.ADDRESS); - init(MetricKeys.Gauge.NODE_TYPE, - "Type of this node, encoded as a number: 0 = fullnode, 1 = witness."); + init(MetricKeys.Gauge.IS_WITNESS, + "Whether this node runs in witness mode, encoded as a number: 0 = not witness, " + + "1 = witness."); init(MetricKeys.Gauge.NODE_ROLE, "Backup-election role of this node, encoded as a number: 0 = init, " + "1 = master, 2 = slaver."); diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java index 3d968f0f40a..82516b78bd9 100644 --- a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -10,21 +10,21 @@ private SRMetrics() { throw new IllegalStateException("SRMetrics"); } - // node_type: 0 = fullnode, 1 = witness - public static final int NODE_TYPE_FULLNODE = 0; - public static final int NODE_TYPE_WITNESS = 1; + // is_witness: 0 = not witness, 1 = witness + public static final int NOT_WITNESS = 0; + public static final int WITNESS = 1; // node_role: 0 = init (pre-election / re-election), 1 = master, 2 = slaver public static final int NODE_ROLE_INIT = 0; public static final int NODE_ROLE_MASTER = 1; public static final int NODE_ROLE_SLAVER = 2; - /** Publishes node_type as one label-less series; 0 = fullnode, 1 = witness. */ - public static void recordNodeType(int nodeType) { + /** Publishes is_witness as one label-less series; 0 = not witness, 1 = witness. */ + public static void recordIsWitness(int isWitness) { if (!Metrics.enabled()) { return; } - Metrics.gaugeSet(MetricKeys.Gauge.NODE_TYPE, nodeType); + Metrics.gaugeSet(MetricKeys.Gauge.IS_WITNESS, isWitness); } /** Publishes node_role as one label-less series; role changes rewrite it in place. */ diff --git a/docs/metrics-changelog.md b/docs/metrics-changelog.md index 77f3e377847..971f6979e1c 100644 --- a/docs/metrics-changelog.md +++ b/docs/metrics-changelog.md @@ -10,8 +10,8 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron #### Core - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:node_type` (Gauge, no labels) — identity of this node, encoded as a number: `0` = `fullnode` (started without witness mode; witness keys are not loaded and no `tron:witness_address` series is published), `1` = `witness` (started with `-w`). It is a process identity, published once at startup and independent of the backup-election role. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) -- `tron:node_role` (Gauge, no labels) — backup-election role of this node, encoded as a number: `0` = `init` (backup election configured but not yet run, and again while a re-election is running after the master lost keep-alive), `1` = `master` (the backup primary; also the standalone default for both node types when backup is not configured), `2` = `slaver` (standing by for a master). The role is a node-level status, independent of `tron:node_type`, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:is_witness` (Gauge, no labels) — whether this node runs in witness mode, encoded as a number: `0` = `not witness` (started without witness mode; witness keys are not loaded and no `tron:witness_address` series is published), `1` = `witness` (started with `-w`). It is a process identity, published once at startup and independent of the backup-election role. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) +- `tron:node_role` (Gauge, no labels) — backup-election role of this node, encoded as a number: `0` = `init` (backup election configured but not yet run, and again while a re-election is running after the master lost keep-alive), `1` = `master` (the backup primary; also the standalone default for both node types when backup is not configured), `2` = `slaver` (standing by for a master). The role is a node-level status, independent of `tron:is_witness`, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:witness_address` (Gauge, label `address`) — value `1` for each consensus witness miner this node holds; the label is the Base58Check-encoded witness account address. The set is assembled once at startup from the finalized consensus miners: one series per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. With several witness keys, each address gets its own series. A plain fullnode without a witness miner publishes no `tron:witness_address` series at all. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923)) diff --git a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java index 48e6e2f4bd1..9a0332e112f 100644 --- a/framework/src/main/java/org/tron/core/consensus/ConsensusService.java +++ b/framework/src/main/java/org/tron/core/consensus/ConsensusService.java @@ -81,7 +81,7 @@ public void start() { param.setMiners(miners); recordWitnessAddresses(miners); - recordNodeType(); + recordIsWitness(); param.setBlockHandle(blockHandle); param.setPbftInterface(pbftBaseImpl); consensus.start(param); @@ -104,20 +104,20 @@ private void recordWitnessAddresses(List miners) { } /** - * Publishes tron:node_type once at consensus startup; 0 = fullnode, 1 = witness. It is + * Publishes tron:is_witness once at consensus startup; 0 = not witness, 1 = witness. It is * published independently of the witness-address publication, and even when no miner is - * configured, so the identity series always exists. Catches its own failure so a metrics + * configured, so the flag series always exists. Catches its own failure so a metrics * problem can neither skip the witness publication nor abort consensus startup. */ - private void recordNodeType() { + private void recordIsWitness() { if (!Metrics.enabled()) { return; } try { - SRMetrics.recordNodeType(Args.getInstance().isWitness() - ? SRMetrics.NODE_TYPE_WITNESS : SRMetrics.NODE_TYPE_FULLNODE); + SRMetrics.recordIsWitness(Args.getInstance().isWitness() + ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS); } catch (Exception e) { - logger.warn("Failed to record node type metrics.", e); + logger.warn("Failed to record is-witness metrics.", e); } } diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java index 292d1932475..968b0e5363d 100644 --- a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -188,37 +188,37 @@ public void testSrRemoveDirect() { Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, stableWit)); } - // ---- tron:node_type: process identity, published at startup and independent of the role ---- + // ---- tron:is_witness: process flag, published at startup and independent of the role ---- /** - * Identity is published once at startup by ConsensusService: 0 for a node started without + * Flag is published once at startup by ConsensusService: 0 for a node started without * witness mode, 1 for a node started with -w. */ @Test - public void testNodeTypeBothIdentities() { + public void testIsWitnessBothValues() { for (boolean witness : new boolean[] {false, true}) { Args.getInstance().witness = witness; - publishNodeType(); - int expected = witness ? SRMetrics.NODE_TYPE_WITNESS : SRMetrics.NODE_TYPE_FULLNODE; - Assert.assertEquals(expected, nodeTypeGauge(), 0.0); + publishIsWitness(); + int expected = witness ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS; + Assert.assertEquals(expected, isWitnessGauge(), 0.0); } } /** - * Metrics disabled: recordNodeType() short-circuits, so the previously published value stands - * even when the identity would have changed. + * Metrics disabled: recordIsWitness() short-circuits, so the previously published value stands + * even when the flag would have changed. */ @Test - public void testNodeTypeNotPublishedWhenMetricsDisabled() { + public void testIsWitnessNotPublishedWhenMetricsDisabled() { Args.getInstance().witness = false; - publishNodeType(); - Assert.assertEquals(SRMetrics.NODE_TYPE_FULLNODE, nodeTypeGauge(), 0.0); + publishIsWitness(); + Assert.assertEquals(SRMetrics.NOT_WITNESS, isWitnessGauge(), 0.0); Args.getInstance().setMetricsPrometheusEnable(false); try { Args.getInstance().witness = true; - publishNodeType(); - Assert.assertEquals(SRMetrics.NODE_TYPE_FULLNODE, nodeTypeGauge(), 0.0); + publishIsWitness(); + Assert.assertEquals(SRMetrics.NOT_WITNESS, isWitnessGauge(), 0.0); } finally { Args.getInstance().setMetricsPrometheusEnable(true); } @@ -250,23 +250,23 @@ public void testNodeRoleNotPublishedWhenMetricsDisabled() { } /** - * A failure while publishing node_type must not propagate: ConsensusService catches it, so it + * A failure while publishing is_witness must not propagate: ConsensusService catches it, so it * can neither skip the witness-address publication running right beside it nor abort startup. */ @Test - public void testNodeTypeFailureDoesNotBlockWitnessPublication() { + public void testIsWitnessFailureDoesNotBlockWitnessPublication() { Args.getInstance().witness = false; ByteString address = uniqueAddress(); List miners = Collections.singletonList(miner(address, address)); try (MockedStatic mocked = Mockito.mockStatic(SRMetrics.class)) { - mocked.when(() -> SRMetrics.recordNodeType(Mockito.anyInt())) - .thenThrow(new RuntimeException("node_type failure")); + mocked.when(() -> SRMetrics.recordIsWitness(Mockito.anyInt())) + .thenThrow(new RuntimeException("is_witness failure")); mocked.when(() -> SRMetrics.recordWitnessAddress(Mockito.anyString())) .thenCallRealMethod(); - // must not propagate the node_type failure - publishNodeType(); + // must not propagate the is_witness failure + publishIsWitness(); // the witness-address publication still runs independently on the same startup path ReflectionTestUtils.invokeMethod(consensusService, "recordWitnessAddresses", miners); @@ -276,13 +276,13 @@ public void testNodeTypeFailureDoesNotBlockWitnessPublication() { } /** - * The identity metric must be published by {@link ConsensusService#start()} itself, not by a - * caller: with the election engine stubbed out, a real start() has to write tron:node_type from - * Args.isWitness(), even for a node with zero miners. Pre-setting the opposite value pins the - * call site, so dropping the publication from start() fails here. + * The flag metric must be published by {@link ConsensusService#start()} itself, not by a + * caller: with the election engine stubbed out, a real start() has to write tron:is_witness + * from Args.isWitness(), even for a node with zero miners. Pre-setting the opposite value pins + * the call site, so dropping the publication from start() fails here. */ @Test - public void testStartPublishesNodeTypeForZeroMiners() { + public void testStartPublishesIsWitnessForZeroMiners() { Consensus consensusMock = Mockito.mock(Consensus.class); ConsensusService service = new ConsensusService(); ReflectionTestUtils.setField(service, "consensus", consensusMock); @@ -290,13 +290,13 @@ public void testStartPublishesNodeTypeForZeroMiners() { for (boolean witness : new boolean[] {false, true}) { Args.getInstance().witness = witness; // opposite of the expected value, so a start() that skipped the publication cannot pass - SRMetrics.recordNodeType(witness ? SRMetrics.NODE_TYPE_FULLNODE - : SRMetrics.NODE_TYPE_WITNESS); + SRMetrics.recordIsWitness(witness ? SRMetrics.NOT_WITNESS + : SRMetrics.WITNESS); service.start(); - int expected = witness ? SRMetrics.NODE_TYPE_WITNESS : SRMetrics.NODE_TYPE_FULLNODE; - Assert.assertEquals(expected, nodeTypeGauge(), 0.0); + int expected = witness ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS; + Assert.assertEquals(expected, isWitnessGauge(), 0.0); } Mockito.verify(consensusMock, Mockito.times(2)).start(Mockito.any(Param.class)); @@ -380,13 +380,13 @@ public void testNodeRoleInitBeforeBackupElectionBothIdentities() throws Exceptio } /** - * Identity and role are fully independent: for both identities the startup publication is - * fullnode/witness + init, and the role then cycles INIT -> MASTER -> SLAVER -> MASTER while - * node_type never changes. Every publication rewrites the single label-less series in place, + * Flag and role are fully independent: for both flag values the startup publication is + * not-witness/witness + init, and the role then cycles INIT -> MASTER -> SLAVER -> MASTER while + * is_witness never changes. Every publication rewrites the single label-less series in place, * so no duplicate registration or stale role series accumulates. */ @Test - public void testNodeTypeIndependentOfRoleBothIdentities() throws Exception { + public void testIsWitnessIndependentOfRoleBothValues() throws Exception { configureBackup(); BackupManager.BackupStatusEnum[] statuses = { @@ -404,28 +404,28 @@ public void testNodeTypeIndependentOfRoleBothIdentities() throws Exception { BackupManager fresh = new BackupManager(); try { Args.getInstance().witness = witness; - int expectedType = witness ? SRMetrics.NODE_TYPE_WITNESS : SRMetrics.NODE_TYPE_FULLNODE; + int expectedIsWitness = witness ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS; - // startup: node_type comes from ConsensusService, node_role from BackupManager; + // startup: is_witness comes from ConsensusService, node_role from BackupManager; // configured-but-not-init is INIT, not the MASTER default - publishNodeType(); + publishIsWitness(); fresh.recordNodeRole(); - Assert.assertEquals(expectedType, nodeTypeGauge(), 0.0); + Assert.assertEquals(expectedIsWitness, isWitnessGauge(), 0.0); Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, roleGauge(), 0.0); - // flip the startup identity: setStatus() must only republish the role, never recompute - // node_type, so the originally published identity has to keep standing + // flip the startup flag: setStatus() must only republish the role, never recompute + // is_witness, so the originally published flag has to keep standing Args.getInstance().witness = !witness; markBackupInitiated(fresh); for (int i = 0; i < statuses.length; i++) { fresh.setStatus(statuses[i]); Assert.assertEquals(roles[i], roleGauge(), 0.0); - Assert.assertEquals("identity must not change with the role", expectedType, - nodeTypeGauge(), 0.0); + Assert.assertEquals("is_witness must not change with the role", expectedIsWitness, + isWitnessGauge(), 0.0); } Assert.assertEquals(1, sampleCount(MetricKeys.Gauge.NODE_ROLE)); - Assert.assertEquals(1, sampleCount(MetricKeys.Gauge.NODE_TYPE)); + Assert.assertEquals(1, sampleCount(MetricKeys.Gauge.IS_WITNESS)); } finally { fresh.stop(); } @@ -543,11 +543,11 @@ private void markBackupInitiated(BackupManager manager) throws Exception { } /** - * Drives the startup identity publication the way production does: ConsensusService.start() - * calls its private recordNodeType() hook right after assembling the miners. + * Drives the startup flag publication the way production does: ConsensusService.start() + * calls its private recordIsWitness() hook right after assembling the miners. */ - private void publishNodeType() { - ReflectionTestUtils.invokeMethod(consensusService, "recordNodeType"); + private void publishIsWitness() { + ReflectionTestUtils.invokeMethod(consensusService, "recordIsWitness"); } /** @@ -590,9 +590,9 @@ private Double roleGauge() { new String[] {}); } - private Double nodeTypeGauge() { + private Double isWitnessGauge() { return CollectorRegistry.defaultRegistry.getSampleValue( - MetricKeys.Gauge.NODE_TYPE, + MetricKeys.Gauge.IS_WITNESS, new String[] {}, new String[] {}); } diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index 72578d5c0ed..7dae98638ab 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -241,32 +241,32 @@ public void testNodeRoleMetric() { } @Test - public void testNodeTypeMetric() { - // label-less identity series: 0 = fullnode, 1 = witness - SRMetrics.recordNodeType(SRMetrics.NODE_TYPE_WITNESS); + public void testIsWitnessMetric() { + // label-less flag series: 0 = not witness, 1 = witness + SRMetrics.recordIsWitness(SRMetrics.WITNESS); Double value = CollectorRegistry.defaultRegistry.getSampleValue( - MetricKeys.Gauge.NODE_TYPE, + MetricKeys.Gauge.IS_WITNESS, new String[] {}, new String[] {}); - Assert.assertNotNull("tron:node_type sample should exist", value); - Assert.assertEquals(SRMetrics.NODE_TYPE_WITNESS, value, 0.0); + Assert.assertNotNull("tron:is_witness sample should exist", value); + Assert.assertEquals(SRMetrics.WITNESS, value, 0.0); } @Test public void testNodeGaugesTextExport() throws Exception { - SRMetrics.recordNodeType(SRMetrics.NODE_TYPE_FULLNODE); + SRMetrics.recordIsWitness(SRMetrics.NOT_WITNESS); SRMetrics.recordNodeRole(SRMetrics.NODE_ROLE_INIT); StringWriter writer = new StringWriter(); TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples()); String text = writer.toString(); - Assert.assertTrue("scrape output should expose tron:node_type", - text.contains("tron:node_type ")); + Assert.assertTrue("scrape output should expose tron:is_witness", + text.contains("tron:is_witness ")); Assert.assertTrue("scrape output should expose tron:node_role", text.contains("tron:node_role ")); - Assert.assertTrue("tron:node_type should export the recorded value", - text.contains("tron:node_type 0.0")); + Assert.assertTrue("tron:is_witness should export the recorded value", + text.contains("tron:is_witness 0.0")); Assert.assertTrue("tron:node_role should export the recorded value", text.contains("tron:node_role 0.0")); } From c1a58aa96aeb65b3792002d8189b626368d94ea6 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 26 Oct 2026 11:47:26 +0800 Subject: [PATCH 26/26] refactor(metrics): synchronize node role publication --- .../main/java/org/tron/common/backup/BackupManager.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 75b299687e8..2352258cec4 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -75,16 +75,11 @@ public void setStatus(BackupStatusEnum status) { /** Publishes tron:node_role once at startup, and again on every backup-status change. */ @PostConstruct - public void recordNodeRole() { + public synchronized void recordNodeRole() { if (!Metrics.enabled()) { return; } - try { - SRMetrics.recordNodeRole(currentRole()); - } catch (Exception e) { - // a metrics failure must not abort startup or the backup election - logger.warn("Failed to record node role", e); - } + SRMetrics.recordNodeRole(currentRole()); } private int currentRole() {