Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
a21b2a7
feat(metrics): extend node_info and add node status gauges
warku123 Oct 19, 2026
a9cc0ab
fix(metrics): use master/slaver for node role labels
warku123 Oct 19, 2026
9bd4f6b
fix(metrics): refresh node role on backup status change
warku123 Oct 19, 2026
53f4e39
refactor(metrics): tidy naming in node status recording
warku123 Oct 19, 2026
cd2ec20
refactor(metrics): refresh only witness gauges at maintenance blocks
warku123 Oct 19, 2026
c526e68
refactor(metrics): record witness gauges in the maintenance flow
warku123 Oct 19, 2026
2cb285f
test(framework): stub metrics beans in startup fixtures
warku123 Oct 20, 2026
a0fa17f
refactor(metrics): expose witness role as one-hot gauge
warku123 Oct 20, 2026
65159f7
refactor(metrics): encode node role as single numeric gauge
warku123 Oct 20, 2026
c992dda
feat(metrics): publish node_role per configured witness address
warku123 Oct 21, 2026
98719c6
feat(metrics): bind sr gauges to witness addresses
warku123 Oct 21, 2026
97eeaeb
fix(metrics): derive per-key witness addresses for multi-key nodes
warku123 Oct 21, 2026
04271d5
refactor(metrics): single-series node_role and miner-sourced sr addre…
warku123 Oct 22, 2026
16f15a2
refactor(metrics): four-state node_role and fullnode semantics
warku123 Oct 22, 2026
0457171
docs(metrics): trim verbose comments
warku123 Oct 22, 2026
541306a
refactor(metrics): run sr hooks only on the fullnode path
warku123 Oct 22, 2026
82c00dc
refactor(metrics): publish sr gauges from bean lifecycle
warku123 Oct 22, 2026
385dc89
Merge remote-tracking branch 'upstream/release_v4.8.3' into feature/s…
warku123 Oct 22, 2026
f2e109d
refactor(metrics): retain witness identity and node role only
warku123 Oct 23, 2026
3f55a94
refactor(metrics): publish witness address once from maintenance init
warku123 Oct 23, 2026
4f42f07
refactor(metrics): publish witness address from consensus startup
warku123 Oct 23, 2026
fa332d3
fix(metrics): separate node identity from backup role
warku123 Oct 23, 2026
345d81a
docs(metrics): restore unrelated comments
warku123 Oct 23, 2026
795eb43
refactor(metrics): remove redundant witness address null check
warku123 Oct 23, 2026
8b53bbf
test(metrics): cover node type publication during consensus startup
warku123 Oct 23, 2026
26a7004
refactor(metrics): rename node type gauge to is witness
warku123 Oct 23, 2026
efd3062
Merge remote-tracking branch 'upstream/release_v4.8.3' into feature/s…
warku123 Oct 26, 2026
c1a58aa
refactor(metrics): synchronize node role publication
warku123 Oct 26, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ public static class Gauge {
public static final String DB_SST_LEVEL = "tron:db_sst_level";
public static final String MANAGER_QUEUE = "tron:manager_queue_size";
public static final String TX_CACHE = "tron:tx_cache";
// per local consensus witness miner; value 1, label is the Base58Check witness address
public static final String WITNESS_ADDRESS = "tron:witness_address";
// numeric backup-election role of this node: 0 = init, 1 = master, 2 = slaver
public static final String NODE_ROLE = "tron:node_role";
// witness flag of this node: 0 = not witness, 1 = witness
public static final String IS_WITNESS = "tron:is_witness";

private Gauge() {
throw new IllegalStateException("Gauge");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ public static class Gauge {

public static final String PEERS_VALID = "valid";

// ADDRESS is the label of tron:witness_address (node_role and is_witness carry no label)
public static final String ADDRESS = "address";

private Gauge() {
throw new IllegalStateException("Gauge");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,15 @@ class MetricsGauge {
init(MetricKeys.Gauge.DB_SIZE_BYTES, "tron db size .", "type", "db", "level");
init(MetricKeys.Gauge.DB_SST_LEVEL, "tron db files .", "type", "db", "level");
init(MetricKeys.Gauge.TX_CACHE, "tron tx cache info.", "type");
init(MetricKeys.Gauge.WITNESS_ADDRESS,
"1 for each consensus witness address this node holds a miner for.",
MetricLabels.Gauge.ADDRESS);
init(MetricKeys.Gauge.IS_WITNESS,
"Whether this node runs in witness mode, encoded as a number: 0 = not witness, "
+ "1 = witness.");
init(MetricKeys.Gauge.NODE_ROLE,
Comment thread
SeriousCoding789 marked this conversation as resolved.
"Backup-election role of this node, encoded as a number: 0 = init, "
+ "1 = master, 2 = slaver.");
}

private MetricsGauge() {
Expand Down
33 changes: 33 additions & 0 deletions common/src/main/java/org/tron/common/prometheus/SRMetrics.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,39 @@ private SRMetrics() {
throw new IllegalStateException("SRMetrics");
}

// is_witness: 0 = not witness, 1 = witness
public static final int NOT_WITNESS = 0;
public static final int WITNESS = 1;

// node_role: 0 = init (pre-election / re-election), 1 = master, 2 = slaver
public static final int NODE_ROLE_INIT = 0;
public static final int NODE_ROLE_MASTER = 1;
public static final int NODE_ROLE_SLAVER = 2;

/** Publishes is_witness as one label-less series; 0 = not witness, 1 = witness. */
public static void recordIsWitness(int isWitness) {
if (!Metrics.enabled()) {
return;
}
Metrics.gaugeSet(MetricKeys.Gauge.IS_WITNESS, isWitness);
}

/** Publishes node_role as one label-less series; role changes rewrite it in place. */
public static void recordNodeRole(int role) {
if (!Metrics.enabled()) {
return;
}
Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, role);
}

/** Publishes one tron:witness_address series (value 1) for a local witness address. */
public static void recordWitnessAddress(String address) {
if (!Metrics.enabled()) {
return;
}
Metrics.gaugeSet(MetricKeys.Gauge.WITNESS_ADDRESS, 1, address);
}

public static void recordSrSetChange(List<ByteString> currentWits, List<ByteString> newWits) {
if (!Metrics.enabled()) {
return;
Expand Down
3 changes: 3 additions & 0 deletions docs/metrics-changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron
#### Core

- `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
- `tron:is_witness` (Gauge, no labels) — whether this node runs in witness mode, encoded as a number: `0` = `not witness` (started without witness mode; witness keys are not loaded and no `tron:witness_address` series is published), `1` = `witness` (started with `-w`). It is a process identity, published once at startup and independent of the backup-election role. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
- `tron:node_role` (Gauge, no labels) — backup-election role of this node, encoded as a number: `0` = `init` (backup election configured but not yet run, and again while a re-election is running after the master lost keep-alive), `1` = `master` (the backup primary; also the standalone default for both node types when backup is not configured), `2` = `slaver` (standing by for a master). The role is a node-level status, independent of `tron:is_witness`, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
- `tron:witness_address` (Gauge, label `address`) — value `1` for each consensus witness miner this node holds; the label is the Base58Check-encoded witness account address. The set is assembled once at startup from the finalized consensus miners: one series per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. With several witness keys, each address gets its own series. A plain fullnode without a witness miner publishes no `tron:witness_address` series at all. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
- `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
- `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
- `tron:block_already_known` (Counter) — incremented when a block response matches an outstanding adv request whose exact block ID is already known before that response is processed (best-effort signal; concurrent arrivals may be missed). ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
Expand Down
38 changes: 38 additions & 0 deletions framework/src/main/java/org/tron/common/backup/BackupManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -25,6 +26,8 @@
import org.tron.common.backup.socket.UdpEvent;
import org.tron.common.es.ExecutorServiceManager;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.prometheus.Metrics;
import org.tron.common.prometheus.SRMetrics;
import org.tron.p2p.utils.NetUtil;

@Slf4j(topic = "backup")
Expand Down Expand Up @@ -67,6 +70,41 @@ public class BackupManager implements EventHandler {
public void setStatus(BackupStatusEnum status) {
logger.info("Change backup status to {}", status);
this.status = status;
recordNodeRole();
}

/** Publishes tron:node_role once at startup, and again on every backup-status change. */
@PostConstruct
public synchronized void recordNodeRole() {
if (!Metrics.enabled()) {
return;
}
SRMetrics.recordNodeRole(currentRole());
}

private int currentRole() {
// no backup configured: default to master, for either identity
if (!isBackupConfigured()) {
return SRMetrics.NODE_ROLE_MASTER;
}
// status still defaults to MASTER before backup init; not a real election result yet
if (!isInit) {
Comment thread
SeriousCoding789 marked this conversation as resolved.
return SRMetrics.NODE_ROLE_INIT;
}
// post-init INIT covers the pre-keep-alive window and master-loss re-election
switch (status) {
case MASTER:
return SRMetrics.NODE_ROLE_MASTER;
case SLAVER:
return SRMetrics.NODE_ROLE_SLAVER;
default:
return SRMetrics.NODE_ROLE_INIT;
}
}

private boolean isBackupConfigured() {
// same condition BackupServer.initServer() uses to start the election transport
return parameter.getBackupPort() > 0 && !parameter.getBackupMembers().isEmpty();
}

public void init() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
import org.springframework.stereotype.Component;
import org.tron.common.crypto.SignUtils;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.prometheus.Metrics;
import org.tron.common.prometheus.SRMetrics;
import org.tron.common.utils.StringUtil;
import org.tron.consensus.Consensus;
import org.tron.consensus.base.Param;
import org.tron.consensus.base.Param.Miner;
Expand Down Expand Up @@ -77,12 +80,47 @@ public void start() {
}

param.setMiners(miners);
recordWitnessAddresses(miners);
recordIsWitness();
param.setBlockHandle(blockHandle);
param.setPbftInterface(pbftBaseImpl);
consensus.start(param);
logger.info("consensus service start success");
}

private void recordWitnessAddresses(List<Miner> miners) {
if (!Metrics.enabled() || miners == null) {
return;
}
for (Miner miner : miners) {
ByteString witnessAddress = miner.getWitnessAddress();
try {
String address = StringUtil.encode58Check(witnessAddress.toByteArray());
SRMetrics.recordWitnessAddress(address);
} catch (Exception e) {
logger.warn("Failed to record witness address metrics.", e);
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can put try/catch outside of for loop.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The try/catch is intentionally kept inside the loop so that a failure for one witness address does not prevent the remaining addresses from being published. Moving it outside would stop processing at the first failure, so I’d prefer to keep the per-miner isolation here.

}
}

/**
* Publishes tron:is_witness once at consensus startup; 0 = not witness, 1 = witness. It is
* published independently of the witness-address publication, and even when no miner is
* configured, so the flag series always exists. Catches its own failure so a metrics
* problem can neither skip the witness publication nor abort consensus startup.
*/
private void recordIsWitness() {
if (!Metrics.enabled()) {
return;
}
try {
SRMetrics.recordIsWitness(Args.getInstance().isWitness()
? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS);
} catch (Exception e) {
logger.warn("Failed to record is-witness metrics.", e);
}
}

public void stop() {
logger.info("consensus service closed start.");
consensus.stop();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.tron.common.backup;

import io.netty.channel.Channel;
import io.prometheus.client.CollectorRegistry;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
Expand All @@ -20,14 +21,19 @@
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import org.tron.common.TestConstants;
import org.tron.common.backup.BackupManager.BackupStatusEnum;
import org.tron.common.backup.message.KeepAliveMessage;
import org.tron.common.backup.socket.BackupServer;
import org.tron.common.backup.socket.MessageHandler;
import org.tron.common.backup.socket.UdpEvent;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.prometheus.MetricKeys;
import org.tron.common.prometheus.SRMetrics;
import org.tron.common.utils.PublicMethod;
import org.tron.common.utils.ReflectUtils;
import org.tron.consensus.base.Param;
import org.tron.core.config.args.Args;
import org.tron.core.config.args.InetUtil;

Expand Down Expand Up @@ -76,6 +82,9 @@ public void tearDown() throws Exception {
});
InetUtil.dnsLookup = previousDnsLookup;
Args.clearParam();
// the consensus miner list is a process-wide singleton: reset it so this class
// cannot leak state into other tests of the same JVM
Param.getInstance().setMiners(new ArrayList<>());
BackupTestUtils.throwIfAnyError(errors);
}

Expand Down Expand Up @@ -183,6 +192,64 @@ public void testBackupServerLifecycleDuringKeepAliveInterval() throws Exception
BackupTestUtils.assertExecutorsTerminated(manager, backupServer);
}

/**
* A backup group whose only member is this node's own address is a "single-master" group:
* no peer ever answers a keep-alive, so the node must leave the pre-election INIT window and
* self-promote to MASTER once the keep-alive timeout elapses.
*
* <p>Drives the real {@link BackupManager#init()} and its real scheduled keep-alive task. The
* timeout condition is made deterministic by ageing the last keep-alive time, so the first
* (fixed 1s) scheduler tick is guaranteed to observe an expired keep-alive; recovery is then
* awaited by polling, so there is no fixed sleep. The published {@code tron:node_role} must
* track the transition (0 = init, 1 = master) and the manager executor must terminate.
*/
@Test(timeout = 30000)
public void singleMasterGroupRecoversFromInitToMaster() throws Exception {
CommonParameter parameter = CommonParameter.getInstance();
parameter.setBackupPriority(8);
// the group's only member is ourselves, so init() drops it and no peer can answer
parameter.setBackupMembers(
Collections.singletonList(InetAddress.getLocalHost().getHostAddress()));

boolean previousMetrics = Args.getInstance().isMetricsPrometheusEnable();
Args.getInstance().setMetricsPrometheusEnable(true);
BackupManager single = new BackupManager();
// absorb any keep-alive send in case the local IP lookup behaves differently
single.setMessageHandler(Mockito.mock(MessageHandler.class));
try {
// backup configured but not yet init'ed: the pre-election window reports init (0)
Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, single.getStatus());
single.recordNodeRole();
Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, nodeRoleGauge(), 0.0);

single.init();
Assert.assertEquals(BackupManager.BackupStatusEnum.INIT, single.getStatus());
Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, nodeRoleGauge(), 0.0);

// no keep-alive has ever arrived, so the next keep-alive tick must self-promote.
// setStatus() flips the volatile status before it republishes the gauge, so wait on
// the published series (the externally visible result), not on the status field alone.
ReflectUtils.setFieldValue(single, "lastKeepAliveTime", 0L);
BackupTestUtils.awaitCondition("lone master to recover from INIT",
() -> single.getStatus() == BackupManager.BackupStatusEnum.MASTER
&& Double.valueOf(SRMetrics.NODE_ROLE_MASTER).equals(nodeRoleGauge()));
Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, single.getStatus());
Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, nodeRoleGauge(), 0.0);
} finally {
single.stop();
Args.getInstance().setMetricsPrometheusEnable(previousMetrics);
}

ScheduledExecutorService executorService =
(ScheduledExecutorService) ReflectUtils.getFieldObject(single, "executorService");
Assert.assertTrue("single-master executor must terminate", executorService.isTerminated());
}

private Double nodeRoleGauge() {
return CollectorRegistry.defaultRegistry.getSampleValue(
MetricKeys.Gauge.NODE_ROLE, new String[] {}, new String[] {});
}

// ===== domain-handling tests for init() =====

@Test(timeout = 5000)
Expand Down
Loading
Loading