From fa2412e3dfa2e23c7936c78dca60a631e7fb585a Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Mon, 31 Aug 2026 17:58:10 +0300 Subject: [PATCH 01/17] raw --- .../SnapshotStatusReproducerTest.java | 167 ++++++++++++++++++ .../snapshot/SnapshotStatusTask.java | 45 ++++- .../snapshot/SnapshotRestoreProcess.java | 2 +- 3 files changed, 204 insertions(+), 10 deletions(-) create mode 100644 modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java new file mode 100644 index 0000000000000..1462cdeb9b0bf --- /dev/null +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java @@ -0,0 +1,167 @@ +package org.apache.ignite.internal.commandline.snapshot; + +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.stream.IntStream; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyResult; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.util.GridCommandHandlerAbstractTest; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.testframework.GridTestUtils.assertContains; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** + * + */ +public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { + /** Snapshot check latch. */ + private static CountDownLatch snapshotCheckLatch; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setPluginProviders(new BlockingCheckSnapshotPluginProvider()) + .setIncludeEventTypes(EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + autoConfirmation = false; + + cleanPersistenceDir(); + + snapshotCheckLatch = new CountDownLatch(1); + + startGrids(3); + + grid(0).cluster().state(ClusterState.ACTIVE); + + IgniteCache cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME); + + IntStream.range(0, 2048).forEach(i -> cache.put(i, i)); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(true); + + cleanPersistenceDir(); + } + + /** + * + */ + @Test + public void test() throws Exception { + injectTestSystemOut(); + + IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + + String snapshotName = "test_snapshot"; + + snapshotMgr.createSnapshot(snapshotName).get(getTestTimeout()); + + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + CountDownLatch restoreStarterdLatch = new CountDownLatch(1); + + grid(0).events().localListen( + e -> { + restoreStarterdLatch.countDown(); + + return false; + }, + EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED + ); + + IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot(snapshotName, null, null, 0, true); + + try { + restoreStarterdLatch.await(getTestTimeout(), MILLISECONDS); + assertFalse("Snapshot future has finished", restoreFut.isDone()); + + int code = execute("--snapshot", "restore", snapshotName, "--status"); + + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + + assertFalse("Snapshot future has finished", restoreFut.isDone()); + + code = execute("--snapshot", "status"); + + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + assertContains(log, testOut.toString(), "Restore snapshot operation is in progress."); + } + finally { + snapshotCheckLatch.countDown(); + + // Wait for future to finish in order to avoid excessive message about task cancellation. + restoreFut.get(); + } + } + + /** */ + private static class BlockingCheckSnapshotPluginProvider extends AbstractTestPluginProvider { + /** {@inheritDoc} */ + @Override public String name() { + return "BlockingCheckSnapshotPluginProvider"; + } + + /** {@inheritDoc} */ + @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.equals(cls)) + return (T)new BlockingCheckSnapshotManager(((IgniteEx)ctx.grid()).context()); + + return null; + } + } + + /** Blocks restore proccess. */ + protected static class BlockingCheckSnapshotManager extends IgniteSnapshotManager { + /** */ + public BlockingCheckSnapshotManager(GridKernalContext ctx) { + super(ctx); + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture checkSnapshot( + String name, + @Nullable String snpPath, + @Nullable Collection grps, + boolean includeCustomHandlers, + int incIdx, boolean check + ) { + return super.checkSnapshot(name, snpPath, grps, includeCustomHandlers, incIdx, check) + .chain(fut -> { + try { + if(check) + ;// snapshotCheckLatch.await(5, MINUTES); + } + catch (Throwable e) { + log.error(X.getFullStackTrace(e)); + } + + return fut.result(); + }); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 01476d309e71f..f60fd461d2ce5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -84,14 +84,14 @@ public class SnapshotStatusTask extends VisorMultiNodeTask s.requestId.equals(s0.requestId)); + res = F.view(res, s -> s.reqId.equals(s0.reqId)); // Merge nodes progress. Map> progress = new HashMap<>(); res.forEach(s -> progress.putAll(s.progress)); - return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.requestId, s0.startTime, progress); + return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.reqId, s0.startTime, progress); } /** */ @@ -164,6 +164,30 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { ); } + mreg = ignite.context().metric().registry(SNAPSHOT_); + + long startTime = mreg.findMetric("startTime").value(); + + if (startTime > mreg.findMetric("endTime").value()) { + return new SnapshotStatus( + SnapshotOperation.RESTORE, + mreg.findMetric("snapshotName").getAsString(), + mreg.findMetric("incrementIndex").value(), + mreg.findMetric("requestId").getAsString(), + mreg.findMetric("startTime").value(), + F.asMap( + ignite.localNode().id(), + new T5<>( + (long)mreg.findMetric("processedPartitions").value(), + (long)mreg.findMetric("totalPartitions").value(), + (long)mreg.findMetric("processedWalSegments").value(), + (long)mreg.findMetric("totalWalSegments").value(), + mreg.findMetric("processedWalEntries").value() + ) + ) + ); + } + return null; } } @@ -183,7 +207,7 @@ public static class SnapshotStatus implements Serializable { private final int incIdx; /** Request ID. */ - private final String requestId; + private final String reqId; /** Start time. */ private final long startTime; @@ -196,14 +220,14 @@ public SnapshotStatus( SnapshotOperation op, String name, int incIdx, - String requestId, + String reqId, long startTime, Map> progress ) { this.op = op; this.name = name; this.incIdx = incIdx; - this.requestId = requestId; + this.reqId = reqId; this.startTime = startTime; this.progress = Collections.unmodifiableMap(progress); } @@ -225,7 +249,7 @@ public int incrementIndex() { /** @return Request ID. */ public String requestId() { - return requestId; + return reqId; } /** @return Start time. */ @@ -241,10 +265,13 @@ public Map> progress() { /** Snapshot operation type. */ public enum SnapshotOperation { - /** Create snapshot. */ + /** Snapshot creation. */ CREATE, - /** Restore snapshot. */ - RESTORE + /** Snapshot restoration. */ + RESTORE, + + /** Snapshot checking before restoration. */ + CHECK_RESTORE } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 77f56f4d4cc7f..50c589407d154 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -351,7 +351,7 @@ public IgniteFutureImpl start( .stream() .findFirst(); - if (!firstMeta.isPresent()) { + if (firstMeta.isEmpty()) { finishProcess( fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot metadata read") From 19dde26c9ccd7decac6c0c775235736d75402ebf Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Tue, 1 Sep 2026 01:50:27 +0300 Subject: [PATCH 02/17] in-progress --- .../SnapshotStatusReproducerTest.java | 11 +- .../management/api/ComputeCommand.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 36 ++- .../snapshot/SnapshotStatusTask.java | 263 ++++++++++++++---- .../snapshot/SnapshotCheckProcess.java | 7 + .../snapshot/SnapshotRestoreStatusTask.java | 2 +- .../feature/SupportedFeatureRegistry.java | 3 + .../resources/META-INF/classnames.properties | 2 + .../TestIgniteReleaseFeatures_2_19_0.java | 3 + 9 files changed, 261 insertions(+), 68 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java index 1462cdeb9b0bf..1de6947a6720b 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java @@ -24,11 +24,8 @@ import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; import static org.apache.ignite.testframework.GridTestUtils.assertContains; -import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; -/** - * - */ +/** */ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { /** Snapshot check latch. */ private static CountDownLatch snapshotCheckLatch; @@ -68,9 +65,7 @@ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest cleanPersistenceDir(); } - /** - * - */ + /** */ @Test public void test() throws Exception { injectTestSystemOut(); @@ -154,7 +149,7 @@ public BlockingCheckSnapshotManager(GridKernalContext ctx) { .chain(fut -> { try { if(check) - ;// snapshotCheckLatch.await(5, MINUTES); + snapshotCheckLatch.await(5, MINUTES); } catch (Throwable e) { log.error(X.getFullStackTrace(e)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java index 10cdcf343c57e..6551e4ac48883 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java @@ -28,7 +28,7 @@ * Command that executed with some compute task. */ public interface ComputeCommand extends Command { - /** @return Task class. */ + /** @return Default task class. */ public Class> taskClass(); /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 18f4fc5b690e6..66212036030e7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -52,7 +52,7 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand taskClass() { + @Override public Class taskClass() { return SnapshotStatusTask.class; } @@ -64,15 +64,18 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; GridStringBuilder s = new GridStringBuilder(); if (isCreating) s.a("Create snapshot operation is in progress.").nl(); - else + else if(isRestoring) s.a("Restore snapshot operation is in progress.").nl(); + else + s.a("Check snapshot operation is in progress.").nl(); s.a("Snapshot name: ").a(status.name()).nl(); s.a("Incremental: ").a(isIncremental).nl(); @@ -94,10 +97,12 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand> rows = status.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) .map(e -> desc.buildRow(e.getKey(), e.getValue())) @@ -246,4 +251,25 @@ private static class RestoreIncrementalSnapshotTaskProgressDesc extends Snapshot return result; } } + + /** */ + private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { + /** */ + CheckSnapshotTaskProgressDesc(boolean incremental) { + super(F.asList("Node ID", "Processed, bytes", "Total, bytes", "Percent")); + } + + /** {@inheritDoc} */ + @Override public List buildRow(UUID nodeId, T5 progress) { + long processed = progress.get1(); + long total = progress.get2(); + + if (total <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown"); + + String percent = (int)(processed * 100 / total) + "%"; + + return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent); + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index f60fd461d2ce5..116e0d04b17d2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -18,25 +18,35 @@ package org.apache.ignite.internal.management.snapshot; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.management.api.NoArg; +import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.util.lang.GridFunc; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.metric.MetricRegistry; +import org.apache.ignite.resources.LoggerResource; +import org.apache.ignite.spi.metric.BooleanMetric; import org.apache.ignite.spi.metric.IntMetric; import org.apache.ignite.spi.metric.LongMetric; import org.jetbrains.annotations.Nullable; @@ -54,9 +64,68 @@ public class SnapshotStatusTask extends VisorMultiNodeTask job(NoArg arg) { - return new SnapshotStatusJob(arg, debug); + if (checkStatusSupported == null) + resolveCheckStatusSupported(); + + assert checkStatusSupported != null; + + return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); + } + + /** */ + private void resolveCheckStatusSupported() { + var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); + + if (!ignite.context().rollingUpgrade().features().isActive(feature)) { + if (log.isInfoEnabled()) { + log.info("The snapshot-check-aware status feature isn't enabled. The status is available only for " + + "snapshot creation and restoration."); + } + + checkStatusSupported = false; + + return; + } + + for (var n : ignite.cluster().nodes()) { + if (!(n instanceof IgniteClusterNode)) { + if (log.isInfoEnabled()) { + log.info(String.format( + "Cannot extract features of node %s. The status is available only for snapshot creation and restoration.", + n.id() + )); + } + + checkStatusSupported = false; + + return; + } + + if (!((IgniteClusterNode)n).features().contains(feature)) { + if (log.isInfoEnabled()) { + log.info(String.format( + "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + + "for snapshot creation and restoration.", + n.id() + )); + } + + checkStatusSupported = false; + + return; + } + } + + checkStatusSupported = true; } /** {@inheritDoc} */ @@ -65,7 +134,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask results) { + @Override protected @Nullable SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -75,23 +144,41 @@ public class SnapshotStatusTask extends VisorMultiNodeTask res = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); + Collection res0 = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); // There is no snapshot operation. - if (res.isEmpty()) + if (res0.isEmpty()) return null; - SnapshotStatus s0 = F.first(res); + SnapshotStatus firstRes = F.first(res0); // Filter out differing requests due to concurrent updates on nodes. - res = F.view(res, s -> s.reqId.equals(s0.reqId)); + Collection sameRqRes = F.view(res0, s -> s.reqId.equals(firstRes.reqId)); + + if (firstRes instanceof SnapshotStatusTask.SnapshotStatusV2) { + var statusV2 = (SnapshotStatusTask.SnapshotStatusV2)firstRes; + + assert !F.isEmpty(statusV2.allCheckStatuses); + + Map> mergedAllCheckStatuses = U.newHashMap(sameRqRes.size()); + + sameRqRes.forEach(s -> { + assert s instanceof SnapshotStatusTask.SnapshotStatusV2; + + mergedAllCheckStatuses.putAll(((SnapshotStatusTask.SnapshotStatusV2)s).allCheckStatuses); + }); + + statusV2.allCheckStatuses = mergedAllCheckStatuses; + + return statusV2; + } // Merge nodes progress. - Map> progress = new HashMap<>(); + Map> mergedProgress = U.newHashMap(sameRqRes.size()); - res.forEach(s -> progress.putAll(s.progress)); + sameRqRes.forEach(s -> mergedProgress.putAll(s.progress)); - return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.reqId, s0.startTime, progress); + return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); } /** */ @@ -103,12 +190,12 @@ private static class SnapshotStatusJob extends SnapshotJob( mreg.findMetric("CurrentSnapshotProcessedSize").value(), mreg.findMetric("CurrentSnapshotTotalSize").value(), - -1L, -1L, -1L); + -1L, + -1L, + -1L + ); } return new SnapshotStatus( @@ -164,41 +254,17 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { ); } - mreg = ignite.context().metric().registry(SNAPSHOT_); - - long startTime = mreg.findMetric("startTime").value(); - - if (startTime > mreg.findMetric("endTime").value()) { - return new SnapshotStatus( - SnapshotOperation.RESTORE, - mreg.findMetric("snapshotName").getAsString(), - mreg.findMetric("incrementIndex").value(), - mreg.findMetric("requestId").getAsString(), - mreg.findMetric("startTime").value(), - F.asMap( - ignite.localNode().id(), - new T5<>( - (long)mreg.findMetric("processedPartitions").value(), - (long)mreg.findMetric("totalPartitions").value(), - (long)mreg.findMetric("processedWalSegments").value(), - (long)mreg.findMetric("totalWalSegments").value(), - mreg.findMetric("processedWalEntries").value() - ) - ) - ); - } - return null; } } /** Snapshot operation status. */ - public static class SnapshotStatus implements Serializable { + static class SnapshotStatus implements Serializable { /** */ private static final long serialVersionUID = 0L; - /** Operation type. */ - private final SnapshotOperation op; + /** Operation type. {@code Null} for other operation types. */ + private final @Nullable SnapshotOperation op; /** Snapshot name. */ private final String name; @@ -216,8 +282,8 @@ public static class SnapshotStatus implements Serializable { private final Map> progress; /** */ - public SnapshotStatus( - SnapshotOperation op, + private SnapshotStatus( + @Nullable SnapshotOperation op, String name, int incIdx, String reqId, @@ -232,46 +298,137 @@ public SnapshotStatus( this.progress = Collections.unmodifiableMap(progress); } - /** @return Operation type. */ - public SnapshotOperation operation() { + /** @return Operation type. {@code Null} for other operation types. */ + @Nullable SnapshotOperation operation() { return op; } /** @return Snapshot name. */ - public String name() { + String name() { return name; } /** @return Incremental snapshot index. */ - public int incrementIndex() { + int incrementIndex() { return incIdx; } /** @return Request ID. */ - public String requestId() { + String requestId() { return reqId; } /** @return Start time. */ - public long startTime() { + long startTime() { return startTime; } /** @return Progress of operation on nodes. */ - public Map> progress() { + Map> progress() { return progress; } } /** Snapshot operation type. */ - public enum SnapshotOperation { + enum SnapshotOperation { /** Snapshot creation. */ CREATE, /** Snapshot restoration. */ - RESTORE, + RESTORE + } + + /** */ + private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatusV2 run(@Nullable NoArg arg) throws IgniteException { + var res1 = super.run(arg); + + if (res1 != null) + return new SnapshotStatusV2(res1); + + List checkStatuses = null; + + for (var snpCheckMReg : ignite.context().metric()) { + if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) + continue; + + if (checkStatuses == null) + checkStatuses = new ArrayList<>(); - /** Snapshot checking before restoration. */ - CHECK_RESTORE + int incIdx = snpCheckMReg.findMetric("incrementIndex") == null + ? 0 + : ((IntMetric)snpCheckMReg.findMetric("incrementIndex")).value(); + + T5 metrics; + + if (incIdx > 0) { + metrics = new T5<>( + (long)snpCheckMReg.findMetric("processedWalSegments").value(), + (long)snpCheckMReg.findMetric("totalWalSegments").value(), + -1L, + -1L, + -1L + ); + } + else { + metrics = new T5<>( + snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, + (long)snpCheckMReg.findMetric("processedPartitions").value(), + (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() + ); + } + + checkStatuses.add(new SnapshotStatus( + null, + MetricUtils.fromFullName(snpCheckMReg.name()).get2(), + incIdx, + snpCheckMReg.findMetric("requestId").getAsString(), + ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), + GridFunc.asMap(ignite.localNode().id(), metrics) + )); + } + + return checkStatuses == null ? null : new SnapshotStatusV2(Collections.singletonMap(ignite.localNode().id(), checkStatuses)); + } + } + + /** Supports snapsho status. */ + private static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { + /** */ + private static final long serialVersionUID = 0L; + + /** Statuses of snapshot check operations per nodeID. */ + private @Nullable Map> allCheckStatuses; + + /** */ + private SnapshotStatusV2(SnapshotStatus s1) { + super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); + } + + /** */ + private SnapshotStatusV2(Map> allCheckStatuses) { + // Single, V1 status holds first found check status. + super( + null, + F.first(allCheckStatuses.values()).get(0).name, + F.first(allCheckStatuses.values()).get(0).incIdx, + F.first(allCheckStatuses.values()).get(0).reqId, + F.first(allCheckStatuses.values()).get(0).startTime, + F.first(allCheckStatuses.values()).get(0).progress + ); + + this.allCheckStatuses = allCheckStatuses; + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 2f19878617a15..f069919248f91 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -732,6 +732,13 @@ private void registerMetrics(SnapshotCheckContext ctx) { mreg.register("startTime", U::currentTimeMillis, "The system time of the start of the cluster snapshot check operation on current node."); + mreg.register( + "requestId", + () -> ctx.req.requestId().toString(), + String.class, + "The request ID of the last running cluster snapshot restore operation on this node." + ); + if (ctx.req.incrementalIndex() > 0) { mreg.register("incrementIndex", ctx.req::incrementalIndex, "The index of incremental snapshot of the snapshot check operation."); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java index 14a395779c9d4..6df4081be8c29 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java @@ -40,7 +40,7 @@ */ @GridInternal @Deprecated -class SnapshotRestoreStatusTask extends ComputeTaskAdapter { +public class SnapshotRestoreStatusTask extends ComputeTaskAdapter { /** Serial version uid. */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java index 7b3e55b85d3c4..fc91f1b838065 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java @@ -93,4 +93,7 @@ public class SupportedFeatureRegistry { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); + + /** */ + public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(1); } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 65f0244da47b7..11594924fe6fd 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -658,7 +658,9 @@ org.apache.ignite.internal.management.snapshot.SnapshotRestoreTask$SnapshotStart org.apache.ignite.internal.management.snapshot.SnapshotStatusTask org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatus +org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusV2 org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJob +org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJobV2 org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetAllCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetCommandArg diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java index 3cb497ee64701..7fb49a2744440 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java @@ -21,4 +21,7 @@ public class TestIgniteReleaseFeatures_2_19_0 { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); + + /** */ + public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(1); } From f18d2fc3fa170b5489f79b5afb90a78bf7cc366e Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 15:40:03 +0300 Subject: [PATCH 03/17] raw --- .../SnapshotStatusReproducerTest.java | 144 +++++++++--------- .../management/api/ComputeCommand.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 46 +++++- .../snapshot/SnapshotStatusTask.java | 2 +- .../snapshot/SnapshotCheckProcess.java | 2 + .../util/distributed/SingleNodeMessage.java | 6 + 6 files changed, 116 insertions(+), 86 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java index 1de6947a6720b..115441ea33b37 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java @@ -1,40 +1,38 @@ package org.apache.ignite.internal.commandline.snapshot; -import java.util.Collection; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.stream.IntStream; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyResult; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; import org.apache.ignite.internal.util.future.IgniteFutureImpl; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.plugin.AbstractTestPluginProvider; -import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.util.GridCommandHandlerAbstractTest; import org.jetbrains.annotations.Nullable; import org.junit.Test; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.testframework.GridTestUtils.assertContains; /** */ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { - /** Snapshot check latch. */ - private static CountDownLatch snapshotCheckLatch; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName) - .setPluginProviders(new BlockingCheckSnapshotPluginProvider()) - .setIncludeEventTypes(EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED); + return super.getConfiguration(igniteInstanceName).setCommunicationSpi(new TestCommunicationSpi()); } /** {@inheritDoc} */ @@ -45,8 +43,6 @@ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest cleanPersistenceDir(); - snapshotCheckLatch = new CountDownLatch(1); - startGrids(3); grid(0).cluster().state(ClusterState.ACTIVE); @@ -78,85 +74,81 @@ public void test() throws Exception { grid(0).destroyCache(DEFAULT_CACHE_NAME); - CountDownLatch restoreStarterdLatch = new CountDownLatch(1); + awaitPartitionMapExchange(); - grid(0).events().localListen( - e -> { - restoreStarterdLatch.countDown(); + CountDownLatch checkSingleResultsReceivedLatch = new CountDownLatch(2); + CountDownLatch restoreSingleResultsReceivedLatch = new CountDownLatch(2); + CountDownLatch proceedCheckLatch = new CountDownLatch(1); - return false; - }, - EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED - ); + for(var ig : Arrays.asList(grid(1), grid(2))) { + ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { + if(!(msg instanceof GridIoMessage ioMsg)) + return; - IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot(snapshotName, null, null, 0, true); + if(!(ioMsg.message() instanceof SingleNodeMessage sm)) + return; - try { - restoreStarterdLatch.await(getTestTimeout(), MILLISECONDS); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) { + restoreSingleResultsReceivedLatch.countDown(); + } else if (sm.type() == CHECK_SNAPSHOT_METAS.ordinal()) { + checkSingleResultsReceivedLatch.countDown(); - int code = execute("--snapshot", "restore", snapshotName, "--status"); + try { + assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + } - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot(snapshotName, null, null, 0, true); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); - code = execute("--snapshot", "status"); + // Make sure no restoration started or finished. + assertTrue(restoreSingleResultsReceivedLatch.getCount() > 0); + assertFalse("Snapshot future has finished", restoreFut.isDone()); - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - assertContains(log, testOut.toString(), "Restore snapshot operation is in progress."); - } - finally { - snapshotCheckLatch.countDown(); + int code = execute("--snapshot", "status"); - // Wait for future to finish in order to avoid excessive message about task cancellation. - restoreFut.get(); - } + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + assertContains(log, testOut.toString(), "Restore snapshot operation is in progress."); + + assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + // Wait for future to finish in order to avoid excessive message about task cancellation. + restoreFut.get(); } /** */ - private static class BlockingCheckSnapshotPluginProvider extends AbstractTestPluginProvider { - /** {@inheritDoc} */ - @Override public String name() { - return "BlockingCheckSnapshotPluginProvider"; - } + private class TestCommunicationSpi extends TcpCommunicationSpi { + /** */ + private volatile @Nullable Consumer msgCsmr; /** {@inheritDoc} */ - @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { - if (IgniteSnapshotManager.class.equals(cls)) - return (T)new BlockingCheckSnapshotManager(((IgniteEx)ctx.grid()).context()); + @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; - return null; - } - } + if (msgCsmr != null) + msgCsmr.accept(msg); - /** Blocks restore proccess. */ - protected static class BlockingCheckSnapshotManager extends IgniteSnapshotManager { - /** */ - public BlockingCheckSnapshotManager(GridKernalContext ctx) { - super(ctx); + super.sendMessage(node, msg); } /** {@inheritDoc} */ - @Override public IgniteInternalFuture checkSnapshot( - String name, - @Nullable String snpPath, - @Nullable Collection grps, - boolean includeCustomHandlers, - int incIdx, boolean check - ) { - return super.checkSnapshot(name, snpPath, grps, includeCustomHandlers, incIdx, check) - .chain(fut -> { - try { - if(check) - snapshotCheckLatch.await(5, MINUTES); - } - catch (Throwable e) { - log.error(X.getFullStackTrace(e)); - } + @Override public void sendMessage( + ClusterNode node, + Message msg, + IgniteInClosure ackC + ) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; + + if (msgCsmr != null) + msgCsmr.accept(msg); - return fut.result(); - }); + super.sendMessage(node, msg, ackC); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java index 6551e4ac48883..10cdcf343c57e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java @@ -28,7 +28,7 @@ * Command that executed with some compute task. */ public interface ComputeCommand extends Command { - /** @return Default task class. */ + /** @return Task class. */ public Class> taskClass(); /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 66212036030e7..9ff10c7121926 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -72,7 +72,7 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand buildRow(UUID nodeId, T5 progress) { - long processed = progress.get1(); - long total = progress.get2(); + if (inc) { + long processed = progress.get1(); + long total = progress.get2(); - if (total <= 0) - return F.asList(nodeId, "unknown", "unknown", "unknown"); + if (total <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown"); - String percent = (int)(processed * 100 / total) + "%"; + String percent = (int)(processed * 100 / total) + "%"; - return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent); + return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent); + } + + long partitionsToCheck = progress.get3(); + long partsToCheck = progress.get5(); + + if (partitionsToCheck <= 0 || partsToCheck <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown"); + + // Checked partitions in current part ratio * shapshot parts ratio. + double totalRatio = ((double)progress.get2() / partitionsToCheck) * ((double)progress.get4() / partsToCheck); + + return F.asList( + nodeId, + progress.get1() == 0L ? "false" : "true", // Full check flag; + U.humanReadableByteCount(progress.get2()), // Checked partitions in current snapshot part; + U.humanReadableByteCount(partitionsToCheck), // Total partitions in current snapshot part; + U.humanReadableByteCount(progress.get4()), // Checked shapshot parts; + U.humanReadableByteCount(partsToCheck), // Total snapshot parts to check. + ((int)(totalRatio * 100.0d)) + '%' + ); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 116e0d04b17d2..bab1af847cb23 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -383,7 +383,7 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { metrics = new T5<>( snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, (long)snpCheckMReg.findMetric("processedPartitions").value(), - (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("totalPartitions").value(), (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() ); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index f069919248f91..6ee811b8858c8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -724,6 +724,8 @@ private boolean baseline(UUID nodeId) { /** */ private void registerMetrics(SnapshotCheckContext ctx) { + log.error("TEST | registerMetrics"); + MetricRegistryImpl mreg = kctx.metric().registry(MetricUtils.metricName(SNAPSHOT_CHECK_METRIC, ctx.req.snapshotName())); assert !mreg.iterator().hasNext(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java index fc6d5943bbee1..5c9d9cb276ef8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.Order; import org.apache.ignite.internal.util.ErrorMessage; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; +import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageFactory; import org.jetbrains.annotations.Nullable; @@ -99,4 +100,9 @@ public boolean hasError() { @Nullable public Throwable error() { return ErrorMessage.error(errMsg); } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SingleNodeMessage.class, this); + } } From dfe63eec11070e844bd2384e106e27af49de04b8 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 15:40:03 +0300 Subject: [PATCH 04/17] raw --- .../SnapshotStatusReproducerTest.java | 158 +++++++------- .../management/api/ComputeCommand.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 136 ++++++++---- .../snapshot/SnapshotStatusTask.java | 206 ++++++++++++------ .../util/distributed/SingleNodeMessage.java | 6 + 5 files changed, 323 insertions(+), 185 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java index 1de6947a6720b..f70a56da143aa 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java @@ -1,40 +1,39 @@ package org.apache.ignite.internal.commandline.snapshot; -import java.util.Collection; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.IntStream; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyResult; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; import org.apache.ignite.internal.util.future.IgniteFutureImpl; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.plugin.AbstractTestPluginProvider; -import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.util.GridCommandHandlerAbstractTest; import org.jetbrains.annotations.Nullable; import org.junit.Test; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.testframework.GridTestUtils.assertContains; /** */ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { - /** Snapshot check latch. */ - private static CountDownLatch snapshotCheckLatch; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName) - .setPluginProviders(new BlockingCheckSnapshotPluginProvider()) - .setIncludeEventTypes(EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED); + return super.getConfiguration(igniteInstanceName).setCommunicationSpi(new TestCommunicationSpi()); } /** {@inheritDoc} */ @@ -45,8 +44,6 @@ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest cleanPersistenceDir(); - snapshotCheckLatch = new CountDownLatch(1); - startGrids(3); grid(0).cluster().state(ClusterState.ACTIVE); @@ -72,91 +69,94 @@ public void test() throws Exception { IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); - String snapshotName = "test_snapshot"; - - snapshotMgr.createSnapshot(snapshotName).get(getTestTimeout()); + snapshotMgr.createSnapshot("test_snapshot").get(getTestTimeout()); grid(0).destroyCache(DEFAULT_CACHE_NAME); - CountDownLatch restoreStarterdLatch = new CountDownLatch(1); + awaitPartitionMapExchange(); + + var checkSingleResultsReceivedLatch = new CountDownLatch(2); + var restoreSingleResultsReceivedLatch = new AtomicInteger(2); + var proceedCheckLatch = new CountDownLatch(1); + + for (var ig : Arrays.asList(grid(1), grid(2))) { + ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { + if (!(msg instanceof GridIoMessage ioMsg)) + return; + + if (!(ioMsg.message() instanceof SingleNodeMessage sm)) + return; - grid(0).events().localListen( - e -> { - restoreStarterdLatch.countDown(); + if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) + restoreSingleResultsReceivedLatch.decrementAndGet(); + else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { + checkSingleResultsReceivedLatch.countDown(); - return false; - }, - EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED - ); + try { + assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + } - IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot(snapshotName, null, null, 0, true); + IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - try { - restoreStarterdLatch.await(getTestTimeout(), MILLISECONDS); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); - int code = execute("--snapshot", "restore", snapshotName, "--status"); + // Make sure no restoration started or finished. + assertTrue(restoreSingleResultsReceivedLatch.get() > 0); + assertFalse("Snapshot future has finished", restoreFut.isDone()); - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + int code = execute("--snapshot", "status"); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + // Ensures that there is a status despite unstarted restore process. + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - code = execute("--snapshot", "status"); + var out = testOut.toString(); - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - assertContains(log, testOut.toString(), "Restore snapshot operation is in progress."); - } - finally { - snapshotCheckLatch.countDown(); + assertContains(log, out, "Check snapshot operation is in progress"); + assertContains(log, out, "Snapshot name: test_snapshot"); + assertContains(log, out, "Incremental: false"); + assertContains(log, out, "Estimated operation progress:"); - // Wait for future to finish in order to avoid excessive message about task cancellation. - restoreFut.get(); - } + proceedCheckLatch.countDown(); + + // Wait for future to finish in order to avoid excessive message about task cancellation. + restoreFut.get(); + + assertTrue(restoreSingleResultsReceivedLatch.get() == 0); } /** */ - private static class BlockingCheckSnapshotPluginProvider extends AbstractTestPluginProvider { - /** {@inheritDoc} */ - @Override public String name() { - return "BlockingCheckSnapshotPluginProvider"; - } + private static class TestCommunicationSpi extends TcpCommunicationSpi { + /** */ + private volatile @Nullable Consumer msgCsmr; /** {@inheritDoc} */ - @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { - if (IgniteSnapshotManager.class.equals(cls)) - return (T)new BlockingCheckSnapshotManager(((IgniteEx)ctx.grid()).context()); + @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; - return null; - } - } + if (msgCsmr != null) + msgCsmr.accept(msg); - /** Blocks restore proccess. */ - protected static class BlockingCheckSnapshotManager extends IgniteSnapshotManager { - /** */ - public BlockingCheckSnapshotManager(GridKernalContext ctx) { - super(ctx); + super.sendMessage(node, msg); } /** {@inheritDoc} */ - @Override public IgniteInternalFuture checkSnapshot( - String name, - @Nullable String snpPath, - @Nullable Collection grps, - boolean includeCustomHandlers, - int incIdx, boolean check - ) { - return super.checkSnapshot(name, snpPath, grps, includeCustomHandlers, incIdx, check) - .chain(fut -> { - try { - if(check) - snapshotCheckLatch.await(5, MINUTES); - } - catch (Throwable e) { - log.error(X.getFullStackTrace(e)); - } + @Override public void sendMessage( + ClusterNode node, + Message msg, + IgniteInClosure ackC + ) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; + + if (msgCsmr != null) + msgCsmr.accept(msg); - return fut.result(); - }); + super.sendMessage(node, msg, ackC); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java index 6551e4ac48883..10cdcf343c57e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java @@ -28,7 +28,7 @@ * Command that executed with some compute task. */ public interface ComputeCommand extends Command { - /** @return Default task class. */ + /** @return Task class. */ public Class> taskClass(); /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 66212036030e7..1fd77f760c4fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.ignite.internal.management.SystemViewCommand; @@ -64,53 +65,72 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; + boolean creating = SnapshotStatusTask.SnapshotOperation.CREATE == status.operation(); + boolean restoring = SnapshotStatusTask.SnapshotOperation.RESTORE == status.operation(); + boolean inc = status.incrementIndex() > 0; - GridStringBuilder s = new GridStringBuilder(); + assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(creating || restoring); - if (isCreating) - s.a("Create snapshot operation is in progress.").nl(); - else if(isRestoring) - s.a("Restore snapshot operation is in progress.").nl(); - else - s.a("Check snapshot operation is in progress.").nl(); + // The check oeration can be run in parallel for different snapshots. + List multipleOpsView = creating || restoring + ? Collections.singletonList(status) + : ((SnapshotStatusTask.SnapshotStatusV2)status).allCheckStatuses; - s.a("Snapshot name: ").a(status.name()).nl(); - s.a("Incremental: ").a(isIncremental).nl(); + // Flag of additional line delimiter. + AtomicBoolean oneOp = new AtomicBoolean(true); - if (isIncremental) - s.a("Increment index: ").a(status.incrementIndex()).nl(); + multipleOpsView.forEach(s0 -> { + GridStringBuilder s = new GridStringBuilder(); - s.a("Operation request ID: ").a(status.requestId()).nl(); - s.a("Started at: ").a(DateFormat.getDateTimeInstance().format(new Date(status.startTime()))).nl(); - s.a("Duration: ").a(X.timeSpan2DHMSM(System.currentTimeMillis() - status.startTime())).nl() - .nl(); - s.a("Estimated operation progress:").nl(); + if (!oneOp.get()) + printer.accept(U.nl()); - printer.accept(s.toString()); + if (creating) { + assert multipleOpsView.size() == 1; - SnapshotTaskProgressDesc desc; + s.a("Create snapshot operation is in progress.").nl(); + } + else if (restoring) { + assert multipleOpsView.size() == 1; + + s.a("Restore snapshot operation is in progress.").nl(); + } + else + s.a("Check snapshot operation" + (multipleOpsView.size() < 2 ? " is " : "s are ") + "in progress.").nl(); - if (isCreating && isIncremental) - desc = new CreateIncrementalSnapshotTaskProgressDesc(); - else if (isCreating) - desc = new CreateFullSnapshotTaskProgressDesc(); - else if (isRestoring && isIncremental) - desc = new RestoreIncrementalSnapshotTaskProgressDesc(); - else if (isRestoring) - desc = new RestoreFullSnapshotTaskProgressDesc(); - else - desc = new CheckSnapshotTaskProgressDesc(isIncremental); + s.a("Snapshot name: ").a(s0.name()).nl(); + s.a("Incremental: ").a(inc).nl(); - List> rows = status.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) - .map(e -> desc.buildRow(e.getKey(), e.getValue())) - .collect(Collectors.toList()); + if (inc) + s.a("Increment index: ").a(s0.incrementIndex()).nl(); - SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); + s.a("Operation request ID: ").a(s0.requestId()).nl(); + s.a("Started at: ").a(DateFormat.getDateTimeInstance().format(new Date(s0.startTime()))).nl(); + s.a("Duration: ").a(X.timeSpan2DHMSM(System.currentTimeMillis() - s0.startTime())).nl() + .nl(); + s.a("Estimated operation progress:").nl(); - printer.accept(U.nl()); + printer.accept(s.toString()); + + SnapshotTaskProgressDesc desc; + + if (creating) + desc = inc ? new CreateIncrementalSnapshotTaskProgressDesc() : new CreateFullSnapshotTaskProgressDesc(); + else if (restoring) + desc = inc ? new RestoreIncrementalSnapshotTaskProgressDesc() : new RestoreFullSnapshotTaskProgressDesc(); + else + desc = new CheckSnapshotTaskProgressDesc(inc); + + List> rows = s0.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) + .map(e -> desc.buildRow(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + + SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); + + printer.accept(U.nl()); + + oneOp.set(false); + }); } /** Describes progress of a snapshot task. */ @@ -254,22 +274,52 @@ private static class RestoreIncrementalSnapshotTaskProgressDesc extends Snapshot /** */ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { + /** */ + private final boolean inc; + /** */ CheckSnapshotTaskProgressDesc(boolean incremental) { - super(F.asList("Node ID", "Processed, bytes", "Total, bytes", "Percent")); + super(incremental + ? F.asList("Node ID", "processedWalSegments", "totalWalSegments", "percent") + : F.asList("Node ID", "fullCheck", "processedPartitions", "totalPartitions", + "processedSnapshotParts", "snapshotPartsToProcess", "percent") + ); + + inc = incremental; } /** {@inheritDoc} */ @Override public List buildRow(UUID nodeId, T5 progress) { - long processed = progress.get1(); - long total = progress.get2(); + if (inc) { + long processed = progress.get1(); + long total = progress.get2(); - if (total <= 0) - return F.asList(nodeId, "unknown", "unknown", "unknown"); + if (total <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown"); - String percent = (int)(processed * 100 / total) + "%"; + String percent = (int)(processed * 100 / total) + "%"; - return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent); + return F.asList(nodeId, processed, total, percent); + } + + long partitionsToCheck = progress.get3(); + long partsToCheck = progress.get5(); + + if (partitionsToCheck <= 0 || partsToCheck <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown"); + + // Ration of checked partitions in current snapshot part * total parts ratio. + double totalRatio = ((double)progress.get2() / partitionsToCheck) * ((double)progress.get4() / partsToCheck); + + return F.asList( + nodeId, + progress.get1() == 0L ? "false" : "true", // Full check flag; + progress.get2(), // Checked partitions in current snapshot part; + partitionsToCheck, // Total partitions in current snapshot part; + progress.get4(), // Checked shapshot parts; + partsToCheck, // Total snapshot parts to check. + ((int)(totalRatio * 100.0d)) + '%' + ); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 116e0d04b17d2..02775544c1b9e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -20,7 +20,6 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; @@ -61,17 +60,25 @@ */ @GridInternal public class SnapshotStatusTask extends VisorMultiNodeTask { - /** */ + /** + * + */ private static final long serialVersionUID = 0L; - /** */ + /** + * + */ @LoggerResource private transient IgniteLogger log; - /** */ + /** + * + */ private transient @Nullable Boolean checkStatusSupported; - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected VisorJob job(NoArg arg) { if (checkStatusSupported == null) resolveCheckStatusSupported(); @@ -81,7 +88,9 @@ public class SnapshotStatusTask extends VisorMultiNodeTask jobNodes(VisorTaskArgument arg) { return nodeIds(ignite.cluster().forServers().nodes()); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected @Nullable SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -155,22 +168,29 @@ private void resolveCheckStatusSupported() { // Filter out differing requests due to concurrent updates on nodes. Collection sameRqRes = F.view(res0, s -> s.reqId.equals(firstRes.reqId)); - if (firstRes instanceof SnapshotStatusTask.SnapshotStatusV2) { - var statusV2 = (SnapshotStatusTask.SnapshotStatusV2)firstRes; + if (firstRes instanceof SnapshotStatusV2 firstResV2) { + assert !F.isEmpty(firstResV2.allCheckStatuses); - assert !F.isEmpty(statusV2.allCheckStatuses); - - Map> mergedAllCheckStatuses = U.newHashMap(sameRqRes.size()); + // Check status: snpName, per node collection. + Map statusesMap = U.newHashMap(sameRqRes.size()); sameRqRes.forEach(s -> { assert s instanceof SnapshotStatusTask.SnapshotStatusV2; - mergedAllCheckStatuses.putAll(((SnapshotStatusTask.SnapshotStatusV2)s).allCheckStatuses); - }); + for (SnapshotStatus s0 : ((SnapshotStatusTask.SnapshotStatusV2)s).allCheckStatuses) { + var prev = statusesMap.putIfAbsent(s0.name, s0); + + if (prev == null) + continue; + + // Merge nodes progress. + prev.progress().putAll(s0.progress()); + } - statusV2.allCheckStatuses = mergedAllCheckStatuses; + firstResV2.allCheckStatuses = new ArrayList<>(statusesMap.values()); + }); - return statusV2; + return firstResV2; } // Merge nodes progress. @@ -181,20 +201,26 @@ private void resolveCheckStatusSupported() { return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); } - /** */ + /** + * + */ private static class SnapshotStatusJob extends SnapshotJob { - /** */ + /** + * + */ private static final long serialVersionUID = 0L; /** - * @param arg Job argument. + * @param arg Job argument. * @param debug Flag indicating whether debug information should be printed into node log. */ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { if (!CU.isPersistenceEnabled(ignite.context().config())) return null; @@ -258,30 +284,48 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { } } - /** Snapshot operation status. */ - static class SnapshotStatus implements Serializable { - /** */ + /** + * Snapshot operation status. + */ + public static class SnapshotStatus implements Serializable { + /** + * + */ private static final long serialVersionUID = 0L; - /** Operation type. {@code Null} for other operation types. */ + /** + * Operation type. {@code Null} for other operation types. + */ private final @Nullable SnapshotOperation op; - /** Snapshot name. */ + /** + * Snapshot name. + */ private final String name; - /** Incremental snapshot index. */ + /** + * Incremental snapshot index. + */ private final int incIdx; - /** Request ID. */ + /** + * Request ID. + */ private final String reqId; - /** Start time. */ + /** + * Start time. + */ private final long startTime; - /** Progress of operation on nodes. */ + /** + * Progress of operation on nodes. + */ private final Map> progress; - /** */ + /** + * + */ private SnapshotStatus( @Nullable SnapshotOperation op, String name, @@ -295,60 +339,86 @@ private SnapshotStatus( this.incIdx = incIdx; this.reqId = reqId; this.startTime = startTime; - this.progress = Collections.unmodifiableMap(progress); + this.progress = progress; } - /** @return Operation type. {@code Null} for other operation types. */ + /** + * @return Operation type. {@code Null} for other operation types. + */ @Nullable SnapshotOperation operation() { return op; } - /** @return Snapshot name. */ + /** + * @return Snapshot name. + */ String name() { return name; } - /** @return Incremental snapshot index. */ + /** + * @return Incremental snapshot index. + */ int incrementIndex() { return incIdx; } - /** @return Request ID. */ + /** + * @return Request ID. + */ String requestId() { return reqId; } - /** @return Start time. */ + /** + * @return Start time. + */ long startTime() { return startTime; } - /** @return Progress of operation on nodes. */ + /** + * @return Progress of operation on nodes. + */ Map> progress() { return progress; } } - /** Snapshot operation type. */ + /** + * Snapshot operation type. + */ enum SnapshotOperation { - /** Snapshot creation. */ + /** + * Snapshot creation. + */ CREATE, - /** Snapshot restoration. */ + /** + * Snapshot restoration. + */ RESTORE } - /** */ + /** + * + */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** */ + /** + * + */ private static final long serialVersionUID = 0L; - /** */ + /** + * + */ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected @Nullable SnapshotStatusV2 run(@Nullable NoArg arg) throws IgniteException { var res1 = super.run(arg); @@ -383,49 +453,61 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { metrics = new T5<>( snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, (long)snpCheckMReg.findMetric("processedPartitions").value(), - (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("totalPartitions").value(), (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() ); } - checkStatuses.add(new SnapshotStatus( + var status = new SnapshotStatus( null, MetricUtils.fromFullName(snpCheckMReg.name()).get2(), incIdx, snpCheckMReg.findMetric("requestId").getAsString(), ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), GridFunc.asMap(ignite.localNode().id(), metrics) - )); + ); + + checkStatuses.add(status); } - return checkStatuses == null ? null : new SnapshotStatusV2(Collections.singletonMap(ignite.localNode().id(), checkStatuses)); + return checkStatuses == null ? null : new SnapshotStatusV2(checkStatuses); } } - /** Supports snapsho status. */ - private static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** */ + /** + * Supports snapsho status. + */ + public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { + /** + * + */ private static final long serialVersionUID = 0L; - /** Statuses of snapshot check operations per nodeID. */ - private @Nullable Map> allCheckStatuses; + /** + * Nodes' statuses of all snapshot check operations + */ + @Nullable List allCheckStatuses; - /** */ + /** + * + */ private SnapshotStatusV2(SnapshotStatus s1) { super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); } - /** */ - private SnapshotStatusV2(Map> allCheckStatuses) { + /** + * + */ + private SnapshotStatusV2(List allCheckStatuses) { // Single, V1 status holds first found check status. super( null, - F.first(allCheckStatuses.values()).get(0).name, - F.first(allCheckStatuses.values()).get(0).incIdx, - F.first(allCheckStatuses.values()).get(0).reqId, - F.first(allCheckStatuses.values()).get(0).startTime, - F.first(allCheckStatuses.values()).get(0).progress + allCheckStatuses.get(0).name, + allCheckStatuses.get(0).incIdx, + allCheckStatuses.get(0).reqId, + allCheckStatuses.get(0).startTime, + allCheckStatuses.get(0).progress ); this.allCheckStatuses = allCheckStatuses; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java index fc6d5943bbee1..5c9d9cb276ef8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.Order; import org.apache.ignite.internal.util.ErrorMessage; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; +import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageFactory; import org.jetbrains.annotations.Nullable; @@ -99,4 +100,9 @@ public boolean hasError() { @Nullable public Throwable error() { return ErrorMessage.error(errMsg); } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SingleNodeMessage.class, this); + } } From 4007c1567107d93e03ee72e40c010de973f331e6 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 18:42:32 +0300 Subject: [PATCH 05/17] test fixes --- .../SnapshotStatusReproducerTest.java | 162 ------------------ .../ignite/util/GridCommandHandlerTest.java | 118 ++++++++++++- .../snapshot/SnapshotCheckProcess.java | 2 - 3 files changed, 117 insertions(+), 165 deletions(-) delete mode 100644 modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java deleted file mode 100644 index f70a56da143aa..0000000000000 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java +++ /dev/null @@ -1,162 +0,0 @@ -package org.apache.ignite.internal.commandline.snapshot; - -import java.util.Arrays; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.stream.IntStream; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteException; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.cluster.ClusterState; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.managers.communication.GridIoMessage; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; -import org.apache.ignite.internal.util.distributed.SingleNodeMessage; -import org.apache.ignite.internal.util.future.IgniteFutureImpl; -import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.spi.IgniteSpiException; -import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; -import org.apache.ignite.util.GridCommandHandlerAbstractTest; -import org.jetbrains.annotations.Nullable; -import org.junit.Test; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; -import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; -import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; -import static org.apache.ignite.testframework.GridTestUtils.assertContains; - -/** */ -public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName).setCommunicationSpi(new TestCommunicationSpi()); - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - autoConfirmation = false; - - cleanPersistenceDir(); - - startGrids(3); - - grid(0).cluster().state(ClusterState.ACTIVE); - - IgniteCache cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME); - - IntStream.range(0, 2048).forEach(i -> cache.put(i, i)); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - stopAllGrids(true); - - cleanPersistenceDir(); - } - - /** */ - @Test - public void test() throws Exception { - injectTestSystemOut(); - - IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); - - snapshotMgr.createSnapshot("test_snapshot").get(getTestTimeout()); - - grid(0).destroyCache(DEFAULT_CACHE_NAME); - - awaitPartitionMapExchange(); - - var checkSingleResultsReceivedLatch = new CountDownLatch(2); - var restoreSingleResultsReceivedLatch = new AtomicInteger(2); - var proceedCheckLatch = new CountDownLatch(1); - - for (var ig : Arrays.asList(grid(1), grid(2))) { - ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { - if (!(msg instanceof GridIoMessage ioMsg)) - return; - - if (!(ioMsg.message() instanceof SingleNodeMessage sm)) - return; - - if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) - restoreSingleResultsReceivedLatch.decrementAndGet(); - else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { - checkSingleResultsReceivedLatch.countDown(); - - try { - assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - }; - } - - IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - - assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); - - // Make sure no restoration started or finished. - assertTrue(restoreSingleResultsReceivedLatch.get() > 0); - assertFalse("Snapshot future has finished", restoreFut.isDone()); - - int code = execute("--snapshot", "status"); - - // Ensures that there is a status despite unstarted restore process. - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - - var out = testOut.toString(); - - assertContains(log, out, "Check snapshot operation is in progress"); - assertContains(log, out, "Snapshot name: test_snapshot"); - assertContains(log, out, "Incremental: false"); - assertContains(log, out, "Estimated operation progress:"); - - proceedCheckLatch.countDown(); - - // Wait for future to finish in order to avoid excessive message about task cancellation. - restoreFut.get(); - - assertTrue(restoreSingleResultsReceivedLatch.get() == 0); - } - - /** */ - private static class TestCommunicationSpi extends TcpCommunicationSpi { - /** */ - private volatile @Nullable Consumer msgCsmr; - - /** {@inheritDoc} */ - @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { - var msgCsmr = this.msgCsmr; - - if (msgCsmr != null) - msgCsmr.accept(msg); - - super.sendMessage(node, msg); - } - - /** {@inheritDoc} */ - @Override public void sendMessage( - ClusterNode node, - Message msg, - IgniteInClosure ackC - ) throws IgniteSpiException { - var msgCsmr = this.msgCsmr; - - if (msgCsmr != null) - msgCsmr.accept(msg); - - super.sendMessage(node, msg, ackC); - } - } -} diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index b5f2659b1fa0b..70496ce45b127 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -48,6 +48,7 @@ import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -121,6 +122,7 @@ import org.apache.ignite.internal.util.distributed.DistributedProcess; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.lang.GridFunc; import org.apache.ignite.internal.util.typedef.F; @@ -135,6 +137,7 @@ import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.spi.metric.Metric; @@ -148,10 +151,12 @@ import org.apache.ignite.transactions.TransactionRollbackException; import org.apache.ignite.transactions.TransactionTimeoutException; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.junit.Assume; import org.junit.Test; import static java.io.File.separatorChar; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CLUSTER_NAME; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -182,6 +187,7 @@ import static org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.DEFAULT_TARGET_FOLDER; import static org.apache.ignite.internal.processors.job.GridJobProcessor.JOBS_VIEW; import static org.apache.ignite.internal.processors.task.GridTaskProcessor.TASKS_VIEW; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.testframework.GridTestUtils.assertContains; import static org.apache.ignite.testframework.GridTestUtils.assertNotContains; @@ -233,13 +239,16 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected ListeningTestLogger listeningLog; + /** */ + protected @Nullable Supplier communicationSpiSupp; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); initDiagnosticDir(); - cleanDiagnosticDir(); + cleanPersistenceDir(); } /** {@inheritDoc} */ @@ -263,6 +272,9 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (listeningLog != null) cfg.setGridLogger(listeningLog); + if (communicationSpiSupp != null) + cfg.setCommunicationSpi(communicationSpiSupp.get()); + return cfg; } @@ -3658,6 +3670,80 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { assertNull(ig.cache(DEFAULT_CACHE_NAME)); } + /** Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. */ + @Test + public void testRestoreSnapshotMetricsAtStart() throws Exception { + communicationSpiSupp = TestCommunicationSpi::new; + + startGrids(3).cluster().state(ClusterState.ACTIVE); + + createCacheAndPreload(grid(1), 8192); + + IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + + snapshotMgr.createSnapshot("test_snapshot").get(getTestTimeout()); + + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + + var checkSingleResultsReceivedLatch = new CountDownLatch(2); + var restoreSingleResultsReceivedLatch = new AtomicInteger(2); + var proceedCheckLatch = new CountDownLatch(1); + + for (var ig : Arrays.asList(grid(1), grid(2))) { + ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { + if (!(msg instanceof GridIoMessage ioMsg)) + return; + + if (!(ioMsg.message() instanceof SingleNodeMessage sm)) + return; + + if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) + restoreSingleResultsReceivedLatch.decrementAndGet(); + else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { + checkSingleResultsReceivedLatch.countDown(); + + try { + assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + } + + IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); + + assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); + + // Make sure no restoration started or finished. + assertTrue(restoreSingleResultsReceivedLatch.get() == 2); + assertFalse("Snapshot future has finished", restoreFut.isDone()); + + injectTestSystemOut(); + + int code = execute("--snapshot", "status"); + + // Ensures that there is a status despite unstarted restore process. + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + + var out = testOut.toString(); + + assertContains(log, out, "Check snapshot operation is in progress"); + assertContains(log, out, "Snapshot name: test_snapshot"); + assertContains(log, out, "Incremental: false"); + assertContains(log, out, "Estimated operation progress:"); + + proceedCheckLatch.countDown(); + + // Wait for future to finish in order to avoid excessive message about task cancellation. + restoreFut.get(); + + assertTrue(restoreSingleResultsReceivedLatch.get() == 0); + } + /** @throws Exception If fails. */ @Test public void testSnapshotStatusInMemory() throws Exception { @@ -4066,4 +4152,34 @@ public void input(String input) { this.input = input; } } + + /** */ + private static class TestCommunicationSpi extends TcpCommunicationSpi { + /** */ + private volatile @Nullable Consumer msgCsmr; + + /** {@inheritDoc} */ + @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; + + if (msgCsmr != null) + msgCsmr.accept(msg); + + super.sendMessage(node, msg); + } + + /** {@inheritDoc} */ + @Override public void sendMessage( + ClusterNode node, + Message msg, + IgniteInClosure ackC + ) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; + + if (msgCsmr != null) + msgCsmr.accept(msg); + + super.sendMessage(node, msg, ackC); + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 6ee811b8858c8..f069919248f91 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -724,8 +724,6 @@ private boolean baseline(UUID nodeId) { /** */ private void registerMetrics(SnapshotCheckContext ctx) { - log.error("TEST | registerMetrics"); - MetricRegistryImpl mreg = kctx.metric().registry(MetricUtils.metricName(SNAPSHOT_CHECK_METRIC, ctx.req.snapshotName())); assert !mreg.iterator().hasNext(); From e8d79ea619188d4364e73ce7f2655dd4f6ad5540 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 19:04:13 +0300 Subject: [PATCH 06/17] test fixes --- .../ignite/util/GridCommandHandlerTest.java | 2 +- .../snapshot/SnapshotStatusTask.java | 155 +++++------------- .../snapshot/SnapshotRestoreStatusTask.java | 2 +- 3 files changed, 42 insertions(+), 117 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 70496ce45b127..f5b6de0aea469 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -248,7 +248,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb initDiagnosticDir(); - cleanPersistenceDir(); + cleanDiagnosticDir(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 02775544c1b9e..81f89753bc705 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -60,37 +60,25 @@ */ @GridInternal public class SnapshotStatusTask extends VisorMultiNodeTask { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * - */ + /** */ @LoggerResource private transient IgniteLogger log; - /** - * - */ + /** */ private transient @Nullable Boolean checkStatusSupported; - /** - * {@inheritDoc} - */ + /** @inheritDoc} */ @Override protected VisorJob job(NoArg arg) { if (checkStatusSupported == null) resolveCheckStatusSupported(); - assert checkStatusSupported != null; - return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); } - /** - * - */ + /** */ private void resolveCheckStatusSupported() { var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); @@ -137,16 +125,12 @@ private void resolveCheckStatusSupported() { checkStatusSupported = true; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected Collection jobNodes(VisorTaskArgument arg) { return nodeIds(ignite.cluster().forServers().nodes()); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected @Nullable SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -201,13 +185,9 @@ private void resolveCheckStatusSupported() { return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); } - /** - * - */ + /** */ private static class SnapshotStatusJob extends SnapshotJob { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; /** @@ -218,9 +198,7 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { if (!CU.isPersistenceEnabled(ignite.context().config())) return null; @@ -284,48 +262,30 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { } } - /** - * Snapshot operation status. - */ + /** Snapshot operation status. */ public static class SnapshotStatus implements Serializable { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * Operation type. {@code Null} for other operation types. - */ + /** Operation type. {@code Null} for other operation types. */ private final @Nullable SnapshotOperation op; - /** - * Snapshot name. - */ + /** Snapshot name. */ private final String name; - /** - * Incremental snapshot index. - */ + /** Incremental snapshot index. */ private final int incIdx; - /** - * Request ID. - */ + /** Request ID. */ private final String reqId; - /** - * Start time. - */ + /**vStart time. */ private final long startTime; - /** - * Progress of operation on nodes. - */ + /** Progress of operation on nodes. */ private final Map> progress; - /** - * - */ + /** */ private SnapshotStatus( @Nullable SnapshotOperation op, String name, @@ -342,88 +302,63 @@ private SnapshotStatus( this.progress = progress; } - /** - * @return Operation type. {@code Null} for other operation types. - */ + /** @return Operation type. {@code Null} for other operation types. */ @Nullable SnapshotOperation operation() { return op; } - /** - * @return Snapshot name. - */ + /** @return Snapshot name. */ String name() { return name; } - /** - * @return Incremental snapshot index. - */ + /** @return Incremental snapshot index. */ int incrementIndex() { return incIdx; } - /** - * @return Request ID. - */ + /** @return Request ID. */ String requestId() { return reqId; } - /** - * @return Start time. - */ + /** @return Start time. */ long startTime() { return startTime; } - /** - * @return Progress of operation on nodes. - */ + /** @return Progress of operation on nodes. */ Map> progress() { return progress; } } - /** - * Snapshot operation type. - */ + /** Snapshot operation type. */ enum SnapshotOperation { - /** - * Snapshot creation. - */ + /** Snapshot creation. */ CREATE, - /** - * Snapshot restoration. - */ + /** Snapshot restoration. */ RESTORE } - /** - * - */ + /** */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * - */ + /** */ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** - * {@inheritDoc} - */ - @Override protected @Nullable SnapshotStatusV2 run(@Nullable NoArg arg) throws IgniteException { + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { var res1 = super.run(arg); + // Create or restore status detected. if (res1 != null) - return new SnapshotStatusV2(res1); + return res1; List checkStatuses = null; @@ -475,30 +410,20 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { } } - /** - * Supports snapsho status. - */ + /** Supports snapsho status. */ public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * Nodes' statuses of all snapshot check operations - */ + /** Nodes' statuses of all snapshot check operations. */ @Nullable List allCheckStatuses; - /** - * - */ + /** */ private SnapshotStatusV2(SnapshotStatus s1) { super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); } - /** - * - */ + /** */ private SnapshotStatusV2(List allCheckStatuses) { // Single, V1 status holds first found check status. super( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java index 6df4081be8c29..14a395779c9d4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java @@ -40,7 +40,7 @@ */ @GridInternal @Deprecated -public class SnapshotRestoreStatusTask extends ComputeTaskAdapter { +class SnapshotRestoreStatusTask extends ComputeTaskAdapter { /** Serial version uid. */ private static final long serialVersionUID = 0L; From a57d8004f1842055cc2b2715257e2a93b145d60b Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 19:04:13 +0300 Subject: [PATCH 07/17] test fixes --- .../ignite/util/GridCommandHandlerTest.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 40 ++--- .../snapshot/SnapshotStatusTask.java | 157 +++++------------- .../snapshot/SnapshotRestoreProcess.java | 2 +- .../snapshot/SnapshotRestoreStatusTask.java | 2 +- 5 files changed, 64 insertions(+), 139 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 70496ce45b127..f5b6de0aea469 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -248,7 +248,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb initDiagnosticDir(); - cleanPersistenceDir(); + cleanDiagnosticDir(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 1fd77f760c4fe..819aaa585d1c2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -53,7 +53,7 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand taskClass() { + @Override public Class taskClass() { return SnapshotStatusTask.class; } @@ -65,14 +65,14 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; + boolean isCreating = SnapshotStatusTask.SnapshotOperation.CREATE == status.operation(); + boolean isRestoring = SnapshotStatusTask.SnapshotOperation.RESTORE == status.operation(); + boolean isIncremental = status.incrementIndex() > 0; - assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(creating || restoring); + assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(isCreating || isRestoring); - // The check oeration can be run in parallel for different snapshots. - List multipleOpsView = creating || restoring + // The check operation can be run in parallel for different snapshots. + List multipleOpsView = isCreating || isRestoring ? Collections.singletonList(status) : ((SnapshotStatusTask.SnapshotStatusV2)status).allCheckStatuses; @@ -85,12 +85,12 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand> rows = s0.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) .map(e -> desc.buildRow(e.getKey(), e.getValue())) @@ -275,7 +275,7 @@ private static class RestoreIncrementalSnapshotTaskProgressDesc extends Snapshot /** */ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - private final boolean inc; + private final boolean incremental; /** */ CheckSnapshotTaskProgressDesc(boolean incremental) { @@ -285,12 +285,12 @@ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressD "processedSnapshotParts", "snapshotPartsToProcess", "percent") ); - inc = incremental; + this.incremental = incremental; } /** {@inheritDoc} */ @Override public List buildRow(UUID nodeId, T5 progress) { - if (inc) { + if (incremental) { long processed = progress.get1(); long total = progress.get2(); @@ -308,7 +308,7 @@ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressD if (partitionsToCheck <= 0 || partsToCheck <= 0) return F.asList(nodeId, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown"); - // Ration of checked partitions in current snapshot part * total parts ratio. + // Ratio of checked partitions in current snapshot part * total parts ratio. double totalRatio = ((double)progress.get2() / partitionsToCheck) * ((double)progress.get4() / partsToCheck); return F.asList( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 02775544c1b9e..9ce1fe5613989 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -60,37 +60,25 @@ */ @GridInternal public class SnapshotStatusTask extends VisorMultiNodeTask { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * - */ + /** */ @LoggerResource private transient IgniteLogger log; - /** - * - */ + /** */ private transient @Nullable Boolean checkStatusSupported; - /** - * {@inheritDoc} - */ + /** @inheritDoc} */ @Override protected VisorJob job(NoArg arg) { if (checkStatusSupported == null) resolveCheckStatusSupported(); - assert checkStatusSupported != null; - return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); } - /** - * - */ + /** */ private void resolveCheckStatusSupported() { var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); @@ -137,16 +125,12 @@ private void resolveCheckStatusSupported() { checkStatusSupported = true; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected Collection jobNodes(VisorTaskArgument arg) { return nodeIds(ignite.cluster().forServers().nodes()); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected @Nullable SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -201,26 +185,20 @@ private void resolveCheckStatusSupported() { return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); } - /** - * - */ + /** */ private static class SnapshotStatusJob extends SnapshotJob { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; /** - * @param arg Job argument. + * @param arg Job argument. * @param debug Flag indicating whether debug information should be printed into node log. */ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { if (!CU.isPersistenceEnabled(ignite.context().config())) return null; @@ -284,48 +262,30 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { } } - /** - * Snapshot operation status. - */ + /** Snapshot operation status. */ public static class SnapshotStatus implements Serializable { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * Operation type. {@code Null} for other operation types. - */ + /** Operation type. {@code Null} for other operation types. */ private final @Nullable SnapshotOperation op; - /** - * Snapshot name. - */ + /** Snapshot name. */ private final String name; - /** - * Incremental snapshot index. - */ + /** Incremental snapshot index. */ private final int incIdx; - /** - * Request ID. - */ + /** Request ID. */ private final String reqId; - /** - * Start time. - */ + /** Start time. */ private final long startTime; - /** - * Progress of operation on nodes. - */ + /** Progress of operation on nodes. */ private final Map> progress; - /** - * - */ + /** */ private SnapshotStatus( @Nullable SnapshotOperation op, String name, @@ -342,88 +302,63 @@ private SnapshotStatus( this.progress = progress; } - /** - * @return Operation type. {@code Null} for other operation types. - */ + /** @return Operation type. {@code Null} for other operation types. */ @Nullable SnapshotOperation operation() { return op; } - /** - * @return Snapshot name. - */ + /** @return Snapshot name. */ String name() { return name; } - /** - * @return Incremental snapshot index. - */ + /** @return Incremental snapshot index. */ int incrementIndex() { return incIdx; } - /** - * @return Request ID. - */ + /** @return Request ID. */ String requestId() { return reqId; } - /** - * @return Start time. - */ + /** @return Start time. */ long startTime() { return startTime; } - /** - * @return Progress of operation on nodes. - */ + /** @return Progress of operation on nodes. */ Map> progress() { return progress; } } - /** - * Snapshot operation type. - */ + /** Snapshot operation type. */ enum SnapshotOperation { - /** - * Snapshot creation. - */ + /** Snapshot creation. */ CREATE, - /** - * Snapshot restoration. - */ + /** Snapshot restoration. */ RESTORE } - /** - * - */ + /** */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * - */ + /** */ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** - * {@inheritDoc} - */ - @Override protected @Nullable SnapshotStatusV2 run(@Nullable NoArg arg) throws IgniteException { + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { var res1 = super.run(arg); + // Create or restore status detected. if (res1 != null) - return new SnapshotStatusV2(res1); + return res1; List checkStatuses = null; @@ -475,30 +410,20 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { } } - /** - * Supports snapsho status. - */ + /** {@link SnapshotStatus} with support of snapshot check process. */ public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * Nodes' statuses of all snapshot check operations - */ + /** Nodes' statuses of all snapshot check operations. */ @Nullable List allCheckStatuses; - /** - * - */ + /** */ private SnapshotStatusV2(SnapshotStatus s1) { super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); } - /** - * - */ + /** */ private SnapshotStatusV2(List allCheckStatuses) { // Single, V1 status holds first found check status. super( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 50c589407d154..77f56f4d4cc7f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -351,7 +351,7 @@ public IgniteFutureImpl start( .stream() .findFirst(); - if (firstMeta.isEmpty()) { + if (!firstMeta.isPresent()) { finishProcess( fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot metadata read") diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java index 6df4081be8c29..14a395779c9d4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java @@ -40,7 +40,7 @@ */ @GridInternal @Deprecated -public class SnapshotRestoreStatusTask extends ComputeTaskAdapter { +class SnapshotRestoreStatusTask extends ComputeTaskAdapter { /** Serial version uid. */ private static final long serialVersionUID = 0L; From 375edb418ebc0055b80af2bd60cd0c01c7e7b034 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Wed, 2 Sep 2026 11:09:14 +0300 Subject: [PATCH 08/17] test fixes --- .../ignite/util/GridCommandHandlerTest.java | 99 +++++++------------ .../TestRecordingCommunicationSpi.java | 8 ++ 2 files changed, 42 insertions(+), 65 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index f5b6de0aea469..470388d878ae2 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -137,7 +137,6 @@ import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.spi.metric.Metric; @@ -156,7 +155,6 @@ import org.junit.Test; import static java.io.File.separatorChar; -import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CLUSTER_NAME; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -3673,7 +3671,7 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { /** Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. */ @Test public void testRestoreSnapshotMetricsAtStart() throws Exception { - communicationSpiSupp = TestCommunicationSpi::new; + communicationSpiSupp = TestRecordingCommunicationSpi::new; startGrids(3).cluster().state(ClusterState.ACTIVE); @@ -3687,40 +3685,29 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { awaitPartitionMapExchange(); - var checkSingleResultsReceivedLatch = new CountDownLatch(2); - var restoreSingleResultsReceivedLatch = new AtomicInteger(2); - var proceedCheckLatch = new CountDownLatch(1); + TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); - for (var ig : Arrays.asList(grid(1), grid(2))) { - ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { - if (!(msg instanceof GridIoMessage ioMsg)) - return; - - if (!(ioMsg.message() instanceof SingleNodeMessage sm)) - return; - - if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) - restoreSingleResultsReceivedLatch.decrementAndGet(); - else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { - checkSingleResultsReceivedLatch.countDown(); - - try { - assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - }; + // Block one of the process' first messages of snapshot restoring or snapshot checking. + for (var cm : Arrays.asList(cm1, cm2)) { + cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm + && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) + ); } + // Snapshot restoration should get paused at the preceeding checking. IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); - - // Make sure no restoration started or finished. - assertTrue(restoreSingleResultsReceivedLatch.get() == 2); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + // Waiting for the nodes each to send snapshot check single result. + for (var cm : Arrays.asList(cm1, cm2)) { + assertTrue(waitForCondition( + () -> cm.blockedMessages().stream().anyMatch( + m -> m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() + ), + getTestTimeout() + )); + } injectTestSystemOut(); @@ -3736,12 +3723,24 @@ else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { assertContains(log, out, "Incremental: false"); assertContains(log, out, "Estimated operation progress:"); - proceedCheckLatch.countDown(); + // Let's suppose the restoration could start and wait for a while. + Thread.sleep(3000L); + + // Ensure that no snapshot restoration started or finished. + assertFalse("Snapshot future has finished", restoreFut.isDone()); + + for (var cm : Arrays.asList(cm1, cm2)) { + // Ensure that nthe restoration process didn't start. + assertTrue(cm.blockedMessages().stream().noneMatch( + m -> m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) + ); + + cm.stopBlock(); + } // Wait for future to finish in order to avoid excessive message about task cancellation. restoreFut.get(); - - assertTrue(restoreSingleResultsReceivedLatch.get() == 0); } /** @throws Exception If fails. */ @@ -4152,34 +4151,4 @@ public void input(String input) { this.input = input; } } - - /** */ - private static class TestCommunicationSpi extends TcpCommunicationSpi { - /** */ - private volatile @Nullable Consumer msgCsmr; - - /** {@inheritDoc} */ - @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { - var msgCsmr = this.msgCsmr; - - if (msgCsmr != null) - msgCsmr.accept(msg); - - super.sendMessage(node, msg); - } - - /** {@inheritDoc} */ - @Override public void sendMessage( - ClusterNode node, - Message msg, - IgniteInClosure ackC - ) throws IgniteSpiException { - var msgCsmr = this.msgCsmr; - - if (msgCsmr != null) - msgCsmr.accept(msg); - - super.sendMessage(node, msg, ackC); - } - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index cee31a0224a8f..5d08d2d18ca0a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -32,6 +32,7 @@ import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiInClosure; @@ -248,6 +249,13 @@ public void waitForRecorded() throws InterruptedException { } } + /** */ + public List blockedMessages() { + var res = this.blockedMsgs; + + return F.isEmpty(res) ? Collections.emptyList() : Collections.unmodifiableList(res); + } + /** * @param cls Message class. * @param nodeName Node name. From 4f18f1b84b3a441d64a0f7b0ae041002809cc771 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 11:51:11 +0300 Subject: [PATCH 09/17] tets fixes --- .../ignite/util/GridCommandHandlerTest.java | 7 ++--- .../snapshot/SnapshotStatusTask.java | 28 ++++--------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 470388d878ae2..8cd6c3d950b22 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3723,14 +3723,11 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { assertContains(log, out, "Incremental: false"); assertContains(log, out, "Estimated operation progress:"); - // Let's suppose the restoration could start and wait for a while. - Thread.sleep(3000L); - // Ensure that no snapshot restoration started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); for (var cm : Arrays.asList(cm1, cm2)) { - // Ensure that nthe restoration process didn't start. + // Ensure that the restore process didn't start. assertTrue(cm.blockedMessages().stream().noneMatch( m -> m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) @@ -3740,7 +3737,7 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { } // Wait for future to finish in order to avoid excessive message about task cancellation. - restoreFut.get(); + restoreFut.get(getTestTimeout()); } /** @throws Exception If fails. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 9ce1fe5613989..34a54020b459f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -94,20 +94,7 @@ private void resolveCheckStatusSupported() { } for (var n : ignite.cluster().nodes()) { - if (!(n instanceof IgniteClusterNode)) { - if (log.isInfoEnabled()) { - log.info(String.format( - "Cannot extract features of node %s. The status is available only for snapshot creation and restoration.", - n.id() - )); - } - - checkStatusSupported = false; - - return; - } - - if (!((IgniteClusterNode)n).features().contains(feature)) { + if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { if (log.isInfoEnabled()) { log.info(String.format( "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + @@ -141,16 +128,16 @@ private void resolveCheckStatusSupported() { if (error != null) throw new IgniteException("Failed to get the snapshot status.", error); - Collection res0 = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); + Collection res = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); // There is no snapshot operation. - if (res0.isEmpty()) + if (res.isEmpty()) return null; - SnapshotStatus firstRes = F.first(res0); + SnapshotStatus firstRes = F.first(res); // Filter out differing requests due to concurrent updates on nodes. - Collection sameRqRes = F.view(res0, s -> s.reqId.equals(firstRes.reqId)); + Collection sameRqRes = F.view(res, s -> s.reqId.equals(firstRes.reqId)); if (firstRes instanceof SnapshotStatusV2 firstResV2) { assert !F.isEmpty(firstResV2.allCheckStatuses); @@ -218,10 +205,7 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { metrics = new T5<>( mreg.findMetric("CurrentSnapshotProcessedSize").value(), mreg.findMetric("CurrentSnapshotTotalSize").value(), - -1L, - -1L, - -1L - ); + -1L, -1L, -1L); } return new SnapshotStatus( From 89e50c8aa26f02783b0f667a1b46f723cf0de207 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 12:05:20 +0300 Subject: [PATCH 10/17] test fixes --- .../snapshot/SnapshotStatusTask.java | 18 +++++++----------- .../TestIgniteReleaseFeatures_2_19_0.java | 3 --- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 34a54020b459f..1093e892789c9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -83,10 +83,8 @@ private void resolveCheckStatusSupported() { var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); if (!ignite.context().rollingUpgrade().features().isActive(feature)) { - if (log.isInfoEnabled()) { - log.info("The snapshot-check-aware status feature isn't enabled. The status is available only for " + - "snapshot creation and restoration."); - } + log.warning("The snapshot-check-aware status feature isn't enabled. The status is available only for " + + "snapshot creation and restoration."); checkStatusSupported = false; @@ -95,13 +93,11 @@ private void resolveCheckStatusSupported() { for (var n : ignite.cluster().nodes()) { if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { - if (log.isInfoEnabled()) { - log.info(String.format( - "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + - "for snapshot creation and restoration.", - n.id() - )); - } + log.warning(String.format( + "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + + "for snapshot creation and restoration.", + n.id() + )); checkStatusSupported = false; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java index 7fb49a2744440..3cb497ee64701 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java @@ -21,7 +21,4 @@ public class TestIgniteReleaseFeatures_2_19_0 { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); - - /** */ - public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(1); } From 5e91a9397278773757a7cb98207afd4a867a9a1f Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 14:19:21 +0300 Subject: [PATCH 11/17] extended tests --- .../ignite/util/GridCommandHandlerTest.java | 141 ++++++++++++++---- 1 file changed, 114 insertions(+), 27 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 8cd6c3d950b22..6ed41f1176605 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -78,6 +78,7 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.IgniteVersionUtils; import org.apache.ignite.internal.Order; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.dto.IgniteDataTransferObject; @@ -117,6 +118,10 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor; import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest; import org.apache.ignite.internal.processors.metric.MetricRegistryImpl; +import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; +import org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeProcessor; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSet; import org.apache.ignite.internal.util.BasicRateLimiter; import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.distributed.DistributedProcess; @@ -136,7 +141,11 @@ import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteNodeValidationResult; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.spi.metric.Metric; @@ -240,20 +249,16 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected @Nullable Supplier communicationSpiSupp; + /** */ + protected @Nullable PluginProvider pluginProvider; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); initDiagnosticDir(); - cleanDiagnosticDir(); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - listeningLog = null; + cleanPersistenceDir(); } /** {@inheritDoc} */ @@ -273,6 +278,9 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (communicationSpiSupp != null) cfg.setCommunicationSpi(communicationSpiSupp.get()); + if (pluginProvider != null) + cfg.setPluginProviders(pluginProvider); + return cfg; } @@ -469,7 +477,7 @@ public void testIdleVerifyCancelWhileCalcPartitionHashRunning() throws Exception * @param checkCrc If {@code true} then run idle verify with --check-crc argument. * @param waitAtStart If {@code true} then forces nodes to wait after the procedure start and to proceed only after * the {@code prepare}'s before-latch switched. - */ + */ private void doTestCancelIdleVerify( BiConsumer prepare, boolean checkCrc, @@ -1245,7 +1253,7 @@ public void testConnectivityCommandWithFailedNodes() throws Exception { String what = "There is no connectivity between the following nodes"; assertContains(log, out.replaceAll("[\\W_]+", "").trim(), - what.replaceAll("[\\W_]+", "").trim()); + what.replaceAll("[\\W_]+", "").trim()); } /** @@ -1679,14 +1687,14 @@ else if (entry.getKey().equals(node2)) { // Test kill by xid. validate(h, map -> { - assertEquals(1, map.size()); + assertEquals(1, map.size()); - Map.Entry killedEntry = map.entrySet().iterator().next(); + Map.Entry killedEntry = map.entrySet().iterator().next(); - TxInfo info = killedEntry.getValue().getInfos().get(0); + TxInfo info = killedEntry.getValue().getInfos().get(0); - assertEquals(toKill[0].getXid(), info.getXid()); - }, "--tx", "--kill", + assertEquals(toKill[0].getXid(), info.getXid()); + }, "--tx", "--kill", "--xid", toKill[0].getXid().toString(), // Use saved on first run value. "--nodes", grid(0).localNode().consistentId().toString()); @@ -1917,8 +1925,8 @@ public void testBaselineAddOnNotActiveCluster() throws Exception { // Ignite instase 1 can be logged only in arguments list. boolean isInstance1Found = Arrays.stream(testOutStr.split("\n")) - .filter(s -> s.contains("Arguments:")) - .noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1")); + .filter(s -> s.contains("Arguments:")) + .noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1")); assertTrue(testOutStr, testOutStr.contains("Node not found for consistent ID:")); @@ -3052,7 +3060,7 @@ public void testClusterCreateSnapshotWarning() throws Exception { String expWarn = dataStmrDetected ? DataStreamerUpdatesHandler.WRN_MSG : String.format("Cache partitions differ for cache groups [%s]. ", CU.cacheId(DEFAULT_CACHE_NAME)) - + SnapshotPartitionsQuickVerifyHandler.WRN_MSG; + + SnapshotPartitionsQuickVerifyHandler.WRN_MSG; assertContains(log, out, expWarn); @@ -3668,11 +3676,70 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { assertNull(ig.cache(DEFAULT_CACHE_NAME)); } - /** Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. */ + /** + * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. + * All nodes support the check status. + */ + @Test + public void testRestoreSnapshotMetricsAtStartAllNodesSupport() throws Exception { + doTestRestoreSnapshotMetricsAtStart(null); + } + + /** + * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. + * One node doesn't support the check status. + */ + @Test + public void testRestoreSnapshotMetricsAtStartOneNodeDoesntSupport() throws Exception { + doTestRestoreSnapshotMetricsAtStart(false); + } + + /** + * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. + * All nodes don't support the check status. + */ @Test - public void testRestoreSnapshotMetricsAtStart() throws Exception { + public void testRestoreSnapshotMetricsAtStartNoSupport() throws Exception { + doTestRestoreSnapshotMetricsAtStart(true); + } + + /** + * @param allNodesNotSupporting Flag of how many nodes don't support the snapshot check status. + * If {@code null}, all nodes support. + * If {@code true}, none of nodes support the snapshot check status. + * If {@code false}, just one node doesn't support the snapshot check status. + */ + private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSupporting) throws Exception { communicationSpiSupp = TestRecordingCommunicationSpi::new; + // Creates empty feature set unsupporting the snpshot check ststus if required. + pluginProvider = allNodesNotSupporting == null ? null : new AbstractTestPluginProvider() { + @Override public String name() { + return "Test features provider"; + } + + @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { + if (!cls.equals(DiscoveryNodeValidationProcessor.class)) + return null; + + assert allNodesNotSupporting != null; + + boolean doNotSupport = allNodesNotSupporting + || ctx.igniteConfiguration().getIgniteInstanceName().equals(getTestIgniteInstanceName(2)); + + return (T)new RollingUpgradeProcessor( + ((IgniteEx)ctx.grid()).context(), + doNotSupport ? new IgniteCoreFeatureSet(IgniteVersionUtils.VER, new IgniteFeatureSet()) : IgniteCoreFeatureSet.local() + ) { + @Override public @Nullable IgniteNodeValidationResult validateNode(ClusterNode joiningNode) { + return null; + } + }; + } + }; + + listeningLog = new ListeningTestLogger(log); + startGrids(3).cluster().state(ClusterState.ACTIVE); createCacheAndPreload(grid(1), 8192); @@ -3711,17 +3778,37 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { injectTestSystemOut(); + LogListener logLsnr = null; + + if (Boolean.FALSE.equals(allNodesNotSupporting)) { + logLsnr = LogListener.matches("Node %s doesn't support the snapshot-check-aware status feature" + .formatted(grid(2).localNode().id())).build(); + + listeningLog.registerListener(logLsnr); + } else if (Boolean.TRUE.equals(allNodesNotSupporting)) { + logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); + + listeningLog.registerListener(logLsnr); + } + int code = execute("--snapshot", "status"); - // Ensures that there is a status despite unstarted restore process. assertEquals("Unexpected exit code", EXIT_CODE_OK, code); var out = testOut.toString(); - assertContains(log, out, "Check snapshot operation is in progress"); - assertContains(log, out, "Snapshot name: test_snapshot"); - assertContains(log, out, "Incremental: false"); - assertContains(log, out, "Estimated operation progress:"); + if (allNodesNotSupporting == null) { + assertContains(log, out, "Check snapshot operation is in progress"); + assertContains(log, out, "Snapshot name: test_snapshot"); + assertContains(log, out, "Incremental: false"); + assertContains(log, out, "Estimated operation progress:"); + } else { + assert logLsnr != null; + + assertTrue(logLsnr.check(getTestTimeout())); + + assertContains(log, out, "There is no create or restore snapshot operation in progress"); + } // Ensure that no snapshot restoration started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); @@ -3961,8 +4048,8 @@ public void testCacheIdleVerifyLogLevelDebug() throws Exception { ignite.cluster().state(ACTIVE); IgniteCache cache = ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1)); + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1)); cache.put("key", "value"); From ab62f2dd9eae022ea1473aa1f9b930bbb35c97bf Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 15:54:46 +0300 Subject: [PATCH 12/17] minority --- .../internal/management/snapshot/SnapshotStatusCommand.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 819aaa585d1c2..5b5c702615ac4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -65,8 +65,8 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(isCreating || isRestoring); From c223b8fb7213a22005d7227a467bdc1c6dd6c88e Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 16:31:22 +0300 Subject: [PATCH 13/17] fix --- .../ignite/util/GridCommandHandlerTest.java | 24 +- .../snapshot/SnapshotStatusCommand.java | 9 +- .../snapshot/SnapshotStatusTask.java | 232 +++-------------- .../snapshot/SnapshotStatusTaskV2.java | 239 ++++++++++++++++++ .../util/distributed/SingleNodeMessage.java | 6 - .../ignite/spi/discovery/tcp/ClientImpl.java | 28 +- .../resources/META-INF/classnames.properties | 5 +- 7 files changed, 303 insertions(+), 240 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 6ed41f1176605..2fcfeca5c7b18 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -477,7 +477,7 @@ public void testIdleVerifyCancelWhileCalcPartitionHashRunning() throws Exception * @param checkCrc If {@code true} then run idle verify with --check-crc argument. * @param waitAtStart If {@code true} then forces nodes to wait after the procedure start and to proceed only after * the {@code prepare}'s before-latch switched. - */ + */ private void doTestCancelIdleVerify( BiConsumer prepare, boolean checkCrc, @@ -1253,7 +1253,7 @@ public void testConnectivityCommandWithFailedNodes() throws Exception { String what = "There is no connectivity between the following nodes"; assertContains(log, out.replaceAll("[\\W_]+", "").trim(), - what.replaceAll("[\\W_]+", "").trim()); + what.replaceAll("[\\W_]+", "").trim()); } /** @@ -1687,14 +1687,14 @@ else if (entry.getKey().equals(node2)) { // Test kill by xid. validate(h, map -> { - assertEquals(1, map.size()); + assertEquals(1, map.size()); - Map.Entry killedEntry = map.entrySet().iterator().next(); + Map.Entry killedEntry = map.entrySet().iterator().next(); - TxInfo info = killedEntry.getValue().getInfos().get(0); + TxInfo info = killedEntry.getValue().getInfos().get(0); - assertEquals(toKill[0].getXid(), info.getXid()); - }, "--tx", "--kill", + assertEquals(toKill[0].getXid(), info.getXid()); + }, "--tx", "--kill", "--xid", toKill[0].getXid().toString(), // Use saved on first run value. "--nodes", grid(0).localNode().consistentId().toString()); @@ -1925,8 +1925,8 @@ public void testBaselineAddOnNotActiveCluster() throws Exception { // Ignite instase 1 can be logged only in arguments list. boolean isInstance1Found = Arrays.stream(testOutStr.split("\n")) - .filter(s -> s.contains("Arguments:")) - .noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1")); + .filter(s -> s.contains("Arguments:")) + .noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1")); assertTrue(testOutStr, testOutStr.contains("Node not found for consistent ID:")); @@ -3060,7 +3060,7 @@ public void testClusterCreateSnapshotWarning() throws Exception { String expWarn = dataStmrDetected ? DataStreamerUpdatesHandler.WRN_MSG : String.format("Cache partitions differ for cache groups [%s]. ", CU.cacheId(DEFAULT_CACHE_NAME)) - + SnapshotPartitionsQuickVerifyHandler.WRN_MSG; + + SnapshotPartitionsQuickVerifyHandler.WRN_MSG; assertContains(log, out, expWarn); @@ -4048,8 +4048,8 @@ public void testCacheIdleVerifyLogLevelDebug() throws Exception { ignite.cluster().state(ACTIVE); IgniteCache cache = ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1)); + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1)); cache.put("key", "value"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 5b5c702615ac4..52238cc119075 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -53,8 +53,8 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand taskClass() { - return SnapshotStatusTask.class; + @Override public Class taskClass() { + return SnapshotStatusTaskV2.class; } /** {@inheritDoc} */ @@ -69,12 +69,13 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; - assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(isCreating || isRestoring); + assert (status instanceof SnapshotStatusTaskV2.SnapshotStatusV2) == !(isCreating || isRestoring) + : "No create or restore snapshot operation found but the status os not of V2 status."; // The check operation can be run in parallel for different snapshots. List multipleOpsView = isCreating || isRestoring ? Collections.singletonList(status) - : ((SnapshotStatusTask.SnapshotStatusV2)status).allCheckStatuses; + : ((SnapshotStatusTaskV2.SnapshotStatusV2)status).allCheckStatuses; // Flag of additional line delimiter. AtomicBoolean oneOp = new AtomicBoolean(true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 1093e892789c9..ecc13fd9c7e95 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -18,34 +18,24 @@ package org.apache.ignite.internal.management.snapshot; import java.io.Serializable; -import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.management.api.NoArg; -import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; -import org.apache.ignite.internal.processors.metric.impl.MetricUtils; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; -import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; import org.apache.ignite.internal.processors.task.GridInternal; -import org.apache.ignite.internal.util.lang.GridFunc; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; import org.apache.ignite.internal.util.typedef.internal.CU; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.metric.MetricRegistry; -import org.apache.ignite.resources.LoggerResource; -import org.apache.ignite.spi.metric.BooleanMetric; import org.apache.ignite.spi.metric.IntMetric; import org.apache.ignite.spi.metric.LongMetric; import org.jetbrains.annotations.Nullable; @@ -63,49 +53,9 @@ public class SnapshotStatusTask extends VisorMultiNodeTask job(NoArg arg) { - if (checkStatusSupported == null) - resolveCheckStatusSupported(); - - return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); - } - - /** */ - private void resolveCheckStatusSupported() { - var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); - - if (!ignite.context().rollingUpgrade().features().isActive(feature)) { - log.warning("The snapshot-check-aware status feature isn't enabled. The status is available only for " + - "snapshot creation and restoration."); - - checkStatusSupported = false; - - return; - } - - for (var n : ignite.cluster().nodes()) { - if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { - log.warning(String.format( - "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + - "for snapshot creation and restoration.", - n.id() - )); - - checkStatusSupported = false; - - return; - } - } - - checkStatusSupported = true; + return new SnapshotStatusJob(arg, debug); } /** {@inheritDoc} */ @@ -114,7 +64,7 @@ private void resolveCheckStatusSupported() { } /** {@inheritDoc} */ - @Override protected @Nullable SnapshotStatus reduce0(List results) { + @Nullable @Override protected SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -130,46 +80,21 @@ private void resolveCheckStatusSupported() { if (res.isEmpty()) return null; - SnapshotStatus firstRes = F.first(res); + SnapshotStatus s0 = F.first(res); // Filter out differing requests due to concurrent updates on nodes. - Collection sameRqRes = F.view(res, s -> s.reqId.equals(firstRes.reqId)); - - if (firstRes instanceof SnapshotStatusV2 firstResV2) { - assert !F.isEmpty(firstResV2.allCheckStatuses); - - // Check status: snpName, per node collection. - Map statusesMap = U.newHashMap(sameRqRes.size()); - - sameRqRes.forEach(s -> { - assert s instanceof SnapshotStatusTask.SnapshotStatusV2; - - for (SnapshotStatus s0 : ((SnapshotStatusTask.SnapshotStatusV2)s).allCheckStatuses) { - var prev = statusesMap.putIfAbsent(s0.name, s0); - - if (prev == null) - continue; - - // Merge nodes progress. - prev.progress().putAll(s0.progress()); - } - - firstResV2.allCheckStatuses = new ArrayList<>(statusesMap.values()); - }); - - return firstResV2; - } + res = F.view(res, s -> s.requestId.equals(s0.requestId)); // Merge nodes progress. - Map> mergedProgress = U.newHashMap(sameRqRes.size()); + Map> progress = new HashMap<>(); - sameRqRes.forEach(s -> mergedProgress.putAll(s.progress)); + res.forEach(s -> progress.putAll(s.progress)); - return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); + return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.requestId, s0.startTime, progress); } /** */ - private static class SnapshotStatusJob extends SnapshotJob { + protected static class SnapshotStatusJob extends SnapshotJob { /** */ private static final long serialVersionUID = 0L; @@ -177,12 +102,12 @@ private static class SnapshotStatusJob extends SnapshotJob> progress; /** */ - private SnapshotStatus( - @Nullable SnapshotOperation op, + public SnapshotStatus( + SnapshotOperation op, String name, int incIdx, - String reqId, + String requestId, long startTime, Map> progress ) { this.op = op; this.name = name; this.incIdx = incIdx; - this.reqId = reqId; + this.requestId = requestId; this.startTime = startTime; this.progress = progress; } - /** @return Operation type. {@code Null} for other operation types. */ - @Nullable SnapshotOperation operation() { + /** @return Operation type. */ + public SnapshotOperation operation() { return op; } /** @return Snapshot name. */ - String name() { + public String name() { return name; } /** @return Incremental snapshot index. */ - int incrementIndex() { + public int incrementIndex() { return incIdx; } /** @return Request ID. */ - String requestId() { - return reqId; + public String requestId() { + return requestId; } /** @return Start time. */ - long startTime() { + public long startTime() { return startTime; } /** @return Progress of operation on nodes. */ - Map> progress() { + public Map> progress() { return progress; } } /** Snapshot operation type. */ - enum SnapshotOperation { - /** Snapshot creation. */ + public enum SnapshotOperation { + /** Create snapshot. */ CREATE, - /** Snapshot restoration. */ + /** Restore snapshot. */ RESTORE } - - /** */ - private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { - super(arg, debug); - } - - /** {@inheritDoc} */ - @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { - var res1 = super.run(arg); - - // Create or restore status detected. - if (res1 != null) - return res1; - - List checkStatuses = null; - - for (var snpCheckMReg : ignite.context().metric()) { - if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) - continue; - - if (checkStatuses == null) - checkStatuses = new ArrayList<>(); - - int incIdx = snpCheckMReg.findMetric("incrementIndex") == null - ? 0 - : ((IntMetric)snpCheckMReg.findMetric("incrementIndex")).value(); - - T5 metrics; - - if (incIdx > 0) { - metrics = new T5<>( - (long)snpCheckMReg.findMetric("processedWalSegments").value(), - (long)snpCheckMReg.findMetric("totalWalSegments").value(), - -1L, - -1L, - -1L - ); - } - else { - metrics = new T5<>( - snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, - (long)snpCheckMReg.findMetric("processedPartitions").value(), - (long)snpCheckMReg.findMetric("totalPartitions").value(), - (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), - (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() - ); - } - - var status = new SnapshotStatus( - null, - MetricUtils.fromFullName(snpCheckMReg.name()).get2(), - incIdx, - snpCheckMReg.findMetric("requestId").getAsString(), - ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), - GridFunc.asMap(ignite.localNode().id(), metrics) - ); - - checkStatuses.add(status); - } - - return checkStatuses == null ? null : new SnapshotStatusV2(checkStatuses); - } - } - - /** {@link SnapshotStatus} with support of snapshot check process. */ - public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** */ - private static final long serialVersionUID = 0L; - - /** Nodes' statuses of all snapshot check operations. */ - @Nullable List allCheckStatuses; - - /** */ - private SnapshotStatusV2(SnapshotStatus s1) { - super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); - } - - /** */ - private SnapshotStatusV2(List allCheckStatuses) { - // Single, V1 status holds first found check status. - super( - null, - allCheckStatuses.get(0).name, - allCheckStatuses.get(0).incIdx, - allCheckStatuses.get(0).reqId, - allCheckStatuses.get(0).startTime, - allCheckStatuses.get(0).progress - ); - - this.allCheckStatuses = allCheckStatuses; - } - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java new file mode 100644 index 0000000000000..af629fd394b59 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.management.api.NoArg; +import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; +import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.util.lang.GridFunc; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T5; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.resources.LoggerResource; +import org.apache.ignite.spi.metric.BooleanMetric; +import org.apache.ignite.spi.metric.IntMetric; +import org.apache.ignite.spi.metric.LongMetric; +import org.jetbrains.annotations.Nullable; + +/** + * Task to get the status of the current snapshot operation in the cluster. + */ +@GridInternal +public class SnapshotStatusTaskV2 extends SnapshotStatusTask { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + @LoggerResource + private transient IgniteLogger log; + + /** */ + private transient @Nullable Boolean checkStatusSupported; + + /** @inheritDoc} */ + @Override protected VisorJob job(NoArg arg) { + if (checkStatusSupported == null) + resolveCheckStatusSupported(); + + return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); + } + + /** */ + private void resolveCheckStatusSupported() { + var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); + + if (!ignite.context().rollingUpgrade().features().isActive(feature)) { + log.warning("The snapshot-check-aware status feature isn't enabled. The status is available only for " + + "snapshot creation and restoration."); + + checkStatusSupported = false; + + return; + } + + for (var n : ignite.cluster().nodes()) { + if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { + log.warning(String.format( + "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + + "for snapshot creation and restoration.", + n.id() + )); + + checkStatusSupported = false; + + return; + } + } + + checkStatusSupported = true; + } + + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatus reduce0(List results) { + SnapshotStatus res0 = super.reduce0(results); + + // No results received at all. + if (res0 == null) + return null; + + // Found crate or restore result. + if (res0.operation() != null) + return res0; + + Collection sameRqRes = F.viewReadOnly(results, ComputeJobResult::getData, + r -> r.getData() != null && ((SnapshotStatus)r.getData()).requestId().equals(res0.requestId())); + + assert !F.isEmpty(sameRqRes); + + SnapshotStatus firstRes = F.first(sameRqRes); + + assert firstRes instanceof SnapshotStatusV2 : "Expected V2 snapshot status result"; + + SnapshotStatusV2 firstResV2 = (SnapshotStatusV2)firstRes; + + // Check status: snpName, per node collection. + Map statusesMap = U.newHashMap(sameRqRes.size()); + + sameRqRes.forEach(s -> { + assert s instanceof SnapshotStatusV2; + + for (SnapshotStatus s0 : ((SnapshotStatusV2)s).allCheckStatuses) { + var prev = statusesMap.putIfAbsent(s0.name(), s0); + + if (prev == null) + continue; + + // Merge nodes progress. + prev.progress().putAll(s0.progress()); + } + + firstResV2.allCheckStatuses = new ArrayList<>(statusesMap.values()); + }); + + return firstResV2; + } + + /** */ + private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { + var res1 = super.run(arg); + + // Create or restore status detected. + if (res1 != null) + return res1; + + List checkStatuses = null; + + for (var snpCheckMReg : ignite.context().metric()) { + if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) + continue; + + if (checkStatuses == null) + checkStatuses = new ArrayList<>(); + + int incIdx = snpCheckMReg.findMetric("incrementIndex") == null + ? 0 + : ((IntMetric)snpCheckMReg.findMetric("incrementIndex")).value(); + + T5 metrics; + + if (incIdx > 0) { + metrics = new T5<>( + (long)snpCheckMReg.findMetric("processedWalSegments").value(), + (long)snpCheckMReg.findMetric("totalWalSegments").value(), + -1L, + -1L, + -1L + ); + } + else { + metrics = new T5<>( + snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, + (long)snpCheckMReg.findMetric("processedPartitions").value(), + (long)snpCheckMReg.findMetric("totalPartitions").value(), + (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() + ); + } + + var status = new SnapshotStatus( + null, + MetricUtils.fromFullName(snpCheckMReg.name()).get2(), + incIdx, + snpCheckMReg.findMetric("requestId").getAsString(), + ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), + GridFunc.asMap(ignite.localNode().id(), metrics) + ); + + checkStatuses.add(status); + } + + return checkStatuses == null ? null : new SnapshotStatusV2(checkStatuses); + } + } + + /** {@link SnapshotStatus} with support of snapshot check process. */ + public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { + /** */ + private static final long serialVersionUID = 0L; + + /** Nodes' statuses of all snapshot check operations. */ + @Nullable List allCheckStatuses; + + /** */ + private SnapshotStatusV2(SnapshotStatus s1) { + super(s1.operation(), s1.name(), s1.incrementIndex(), s1.requestId(), s1.startTime(), s1.progress()); + } + + /** */ + private SnapshotStatusV2(List allCheckStatuses) { + // Single, V1 status holds first found check status. + super( + null, + allCheckStatuses.get(0).name(), + allCheckStatuses.get(0).incrementIndex(), + allCheckStatuses.get(0).requestId(), + allCheckStatuses.get(0).startTime(), + allCheckStatuses.get(0).progress() + ); + + this.allCheckStatuses = allCheckStatuses; + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java index 5c9d9cb276ef8..fc6d5943bbee1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java @@ -22,7 +22,6 @@ import org.apache.ignite.internal.Order; import org.apache.ignite.internal.util.ErrorMessage; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; -import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageFactory; import org.jetbrains.annotations.Nullable; @@ -100,9 +99,4 @@ public boolean hasError() { @Nullable public Throwable error() { return ErrorMessage.error(errMsg); } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(SingleNodeMessage.class, this); - } } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java index 3cfacdd43b9a9..7af45bd9de712 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java @@ -535,7 +535,7 @@ else if (state == DISCONNECTED) { * @throws IgniteSpiException If failed. * @see TcpDiscoverySpi#joinTimeout */ - @Nullable private TcpDiscoveryIoSession joinTopology( + @Nullable private SocketStream joinTopology( InetSocketAddress prevAddr, long timeout, @Nullable Runnable beforeEachSleep, @@ -581,12 +581,12 @@ else if (state == DISCONNECTED) { Collection addrs0 = new ArrayList<>(addrs); - T2> waitAndRes = sendJoinRequests(prevAddr != null, addrs); + T2> waitAndRes = sendJoinRequests(prevAddr != null, addrs); addrs.clear(); boolean wait = waitAndRes.get1(); - T2 res = waitAndRes.get2(); + T2 res = waitAndRes.get2(); if (res != null) return res.get1(); @@ -611,7 +611,7 @@ else if (addrs.isEmpty()) { } /** */ - private T2> sendJoinRequests( + private T2> sendJoinRequests( boolean recon, Collection addrs ) throws InterruptedException { @@ -619,21 +619,21 @@ private T2> sendJoinRequests( if (Thread.currentThread().isInterrupted()) throw new InterruptedException(); - T2 joinRes = sendJoinRequest(recon, addr); + T2 sockAndRes = sendJoinRequest(recon, addr); - if (joinRes == null) + if (sockAndRes == null) continue; - assert joinRes.get1() != null && joinRes.get2() != null : joinRes; + assert sockAndRes.get1() != null && sockAndRes.get2() != null : sockAndRes; - Socket sock = joinRes.get1().socket(); + Socket sock = sockAndRes.get1().socket(); if (log.isDebugEnabled()) - log.debug("Received response to join request [addr=" + addr + ", res=" + joinRes.get2() + ']'); + log.debug("Received response to join request [addr=" + addr + ", res=" + sockAndRes.get2() + ']'); - switch (joinRes.get2()) { + switch (sockAndRes.get2()) { case RES_OK: - return new T2<>(false, joinRes); + return new T2<>(false, sockAndRes); case RES_CONTINUE_JOIN: case RES_WAIT: @@ -643,7 +643,7 @@ private T2> sendJoinRequests( default: if (log.isDebugEnabled()) - log.debug("Received unexpected response to join request: " + joinRes.get2()); + log.debug("Received unexpected response to join request: " + sockAndRes.get2()); U.closeQuiet(sock); } @@ -671,7 +671,7 @@ private static void sleepEx(long millis, Runnable before, Runnable after) throws * @param addr Address. * @return Socket, connect response and client acknowledge support flag. */ - @Nullable private T2 sendJoinRequest(boolean recon, + @Nullable private T2 sendJoinRequest(boolean recon, InetSocketAddress addr) throws InterruptedException { assert addr != null; @@ -731,7 +731,7 @@ private static void sleepEx(long millis, Runnable before, Runnable after) throws if (log.isInfoEnabled()) log.info("Reconnecting to the addresses of a proper DC [addrs=" + redirectAddrs + ']'); - T2> redirectedRes = sendJoinRequests(recon, redirectAddrs); + T2> redirectedRes = sendJoinRequests(recon, redirectAddrs); return redirectedRes.get2(); } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 11594924fe6fd..54dd1595b9b00 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -658,9 +658,10 @@ org.apache.ignite.internal.management.snapshot.SnapshotRestoreTask$SnapshotStart org.apache.ignite.internal.management.snapshot.SnapshotStatusTask org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatus -org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusV2 org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJob -org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJobV2 +org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2 +org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2$SnapshotStatusV2 +org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2$SnapshotStatusJobV2 org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetAllCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetCommandArg From 77dbb35838c5acae8d93a4764f3e4c0cb39ac2d7 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 16:38:07 +0300 Subject: [PATCH 14/17] + taskV2 for thin client --- .../ignite/util/GridCommandHandlerTest.java | 3 +- .../snapshot/SnapshotStatusTaskV2.java | 9 +++--- .../ignite/spi/discovery/tcp/ClientImpl.java | 28 +++++++++---------- .../TestRecordingCommunicationSpi.java | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 2fcfeca5c7b18..741a8bcea99e5 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3785,7 +3785,8 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu .formatted(grid(2).localNode().id())).build(); listeningLog.registerListener(logLsnr); - } else if (Boolean.TRUE.equals(allNodesNotSupporting)) { + } + else if (Boolean.TRUE.equals(allNodesNotSupporting)) { logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); listeningLog.registerListener(logLsnr); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java index af629fd394b59..9b185a67007c4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java @@ -42,9 +42,7 @@ import org.apache.ignite.spi.metric.LongMetric; import org.jetbrains.annotations.Nullable; -/** - * Task to get the status of the current snapshot operation in the cluster. - */ +/** V2 of {@link SnapshotStatusTask} with the support of snapshot check status. */ @GridInternal public class SnapshotStatusTaskV2 extends SnapshotStatusTask { /** */ @@ -140,7 +138,7 @@ private void resolveCheckStatusSupported() { return firstResV2; } - /** */ + /** V2 of {@link SnapshotStatusJob} with the support of snapshot check status. */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { /** */ private static final long serialVersionUID = 0L; @@ -208,7 +206,8 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { } } - /** {@link SnapshotStatus} with support of snapshot check process. */ + /** V2 of {@link SnapshotStatus} with the support of snapshot check status. */ + public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java index 7af45bd9de712..3cfacdd43b9a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java @@ -535,7 +535,7 @@ else if (state == DISCONNECTED) { * @throws IgniteSpiException If failed. * @see TcpDiscoverySpi#joinTimeout */ - @Nullable private SocketStream joinTopology( + @Nullable private TcpDiscoveryIoSession joinTopology( InetSocketAddress prevAddr, long timeout, @Nullable Runnable beforeEachSleep, @@ -581,12 +581,12 @@ else if (state == DISCONNECTED) { Collection addrs0 = new ArrayList<>(addrs); - T2> waitAndRes = sendJoinRequests(prevAddr != null, addrs); + T2> waitAndRes = sendJoinRequests(prevAddr != null, addrs); addrs.clear(); boolean wait = waitAndRes.get1(); - T2 res = waitAndRes.get2(); + T2 res = waitAndRes.get2(); if (res != null) return res.get1(); @@ -611,7 +611,7 @@ else if (addrs.isEmpty()) { } /** */ - private T2> sendJoinRequests( + private T2> sendJoinRequests( boolean recon, Collection addrs ) throws InterruptedException { @@ -619,21 +619,21 @@ private T2> sendJoinRequests( if (Thread.currentThread().isInterrupted()) throw new InterruptedException(); - T2 sockAndRes = sendJoinRequest(recon, addr); + T2 joinRes = sendJoinRequest(recon, addr); - if (sockAndRes == null) + if (joinRes == null) continue; - assert sockAndRes.get1() != null && sockAndRes.get2() != null : sockAndRes; + assert joinRes.get1() != null && joinRes.get2() != null : joinRes; - Socket sock = sockAndRes.get1().socket(); + Socket sock = joinRes.get1().socket(); if (log.isDebugEnabled()) - log.debug("Received response to join request [addr=" + addr + ", res=" + sockAndRes.get2() + ']'); + log.debug("Received response to join request [addr=" + addr + ", res=" + joinRes.get2() + ']'); - switch (sockAndRes.get2()) { + switch (joinRes.get2()) { case RES_OK: - return new T2<>(false, sockAndRes); + return new T2<>(false, joinRes); case RES_CONTINUE_JOIN: case RES_WAIT: @@ -643,7 +643,7 @@ private T2> sendJoinRequests( default: if (log.isDebugEnabled()) - log.debug("Received unexpected response to join request: " + sockAndRes.get2()); + log.debug("Received unexpected response to join request: " + joinRes.get2()); U.closeQuiet(sock); } @@ -671,7 +671,7 @@ private static void sleepEx(long millis, Runnable before, Runnable after) throws * @param addr Address. * @return Socket, connect response and client acknowledge support flag. */ - @Nullable private T2 sendJoinRequest(boolean recon, + @Nullable private T2 sendJoinRequest(boolean recon, InetSocketAddress addr) throws InterruptedException { assert addr != null; @@ -731,7 +731,7 @@ private static void sleepEx(long millis, Runnable before, Runnable after) throws if (log.isInfoEnabled()) log.info("Reconnecting to the addresses of a proper DC [addrs=" + redirectAddrs + ']'); - T2> redirectedRes = sendJoinRequests(recon, redirectAddrs); + T2> redirectedRes = sendJoinRequests(recon, redirectAddrs); return redirectedRes.get2(); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index 5d08d2d18ca0a..c8e529f84b789 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -251,7 +251,7 @@ public void waitForRecorded() throws InterruptedException { /** */ public List blockedMessages() { - var res = this.blockedMsgs; + var res = blockedMsgs; return F.isEmpty(res) ? Collections.emptyList() : Collections.unmodifiableList(res); } From 65addb2af6191a1316787fb31bc175685cad7001 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 17:07:40 +0300 Subject: [PATCH 15/17] minor --- .../java/org/apache/ignite/util/GridCommandHandlerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 741a8bcea99e5..5e814445a892f 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3801,8 +3801,6 @@ else if (Boolean.TRUE.equals(allNodesNotSupporting)) { if (allNodesNotSupporting == null) { assertContains(log, out, "Check snapshot operation is in progress"); assertContains(log, out, "Snapshot name: test_snapshot"); - assertContains(log, out, "Incremental: false"); - assertContains(log, out, "Estimated operation progress:"); } else { assert logLsnr != null; From 0b534fee3d260bd7b7c4572baa33e1b5f0d8a1f2 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 3 Sep 2026 08:06:28 +0300 Subject: [PATCH 16/17] fixes, + check status test --- .../ignite/util/GridCommandHandlerTest.java | 137 +++++++++++++----- .../snapshot/SnapshotStatusCommand.java | 35 ++--- .../snapshot/SnapshotStatusTaskV2.java | 43 +++--- 3 files changed, 132 insertions(+), 83 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 5e814445a892f..2275cab14a5df 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3706,24 +3706,22 @@ public void testRestoreSnapshotMetricsAtStartNoSupport() throws Exception { /** * @param allNodesNotSupporting Flag of how many nodes don't support the snapshot check status. * If {@code null}, all nodes support. - * If {@code true}, none of nodes support the snapshot check status. - * If {@code false}, just one node doesn't support the snapshot check status. + * If {@code true}, none supports. + * If {@code false}, just one node doesn't support. */ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSupporting) throws Exception { communicationSpiSupp = TestRecordingCommunicationSpi::new; - // Creates empty feature set unsupporting the snpshot check ststus if required. + // Creates empty feature set unsupporting the snapshot check ststus if required. pluginProvider = allNodesNotSupporting == null ? null : new AbstractTestPluginProvider() { @Override public String name() { - return "Test features provider"; + return "Test Ignite features provider"; } @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { if (!cls.equals(DiscoveryNodeValidationProcessor.class)) return null; - assert allNodesNotSupporting != null; - boolean doNotSupport = allNodesNotSupporting || ctx.igniteConfiguration().getIgniteInstanceName().equals(getTestIgniteInstanceName(2)); @@ -3756,38 +3754,31 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); // Block one of the process' first messages of snapshot restoring or snapshot checking. - for (var cm : Arrays.asList(cm1, cm2)) { - cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm - && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) - ); - } + F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm + && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()))); - // Snapshot restoration should get paused at the preceeding checking. + // Snapshot restoration should get paused at the preceeding snapshot check. IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - // Waiting for the nodes each to send snapshot check single result. - for (var cm : Arrays.asList(cm1, cm2)) { - assertTrue(waitForCondition( - () -> cm.blockedMessages().stream().anyMatch( - m -> m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() - ), - getTestTimeout() - )); + // Waiting for each node to send snapshot check single result. + for (var cm : F.asList(cm1, cm2)) { + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()), + getTestTimeout())); } injectTestSystemOut(); LogListener logLsnr = null; - if (Boolean.FALSE.equals(allNodesNotSupporting)) { - logLsnr = LogListener.matches("Node %s doesn't support the snapshot-check-aware status feature" - .formatted(grid(2).localNode().id())).build(); + if (Boolean.TRUE.equals(allNodesNotSupporting)) { + logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); listeningLog.registerListener(logLsnr); } - else if (Boolean.TRUE.equals(allNodesNotSupporting)) { - logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); + else if (Boolean.FALSE.equals(allNodesNotSupporting)) { + logLsnr = LogListener.matches("Node %s doesn't support the snapshot check status feature" + .formatted(grid(2).localNode().id())).build(); listeningLog.registerListener(logLsnr); } @@ -3801,31 +3792,99 @@ else if (Boolean.TRUE.equals(allNodesNotSupporting)) { if (allNodesNotSupporting == null) { assertContains(log, out, "Check snapshot operation is in progress"); assertContains(log, out, "Snapshot name: test_snapshot"); - } else { - assert logLsnr != null; - + } + else { assertTrue(logLsnr.check(getTestTimeout())); - assertContains(log, out, "There is no create or restore snapshot operation in progress"); } - // Ensure that no snapshot restoration started or finished. + // Ensure that no snapshot restore started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); - for (var cm : Arrays.asList(cm1, cm2)) { - // Ensure that the restore process didn't start. - assertTrue(cm.blockedMessages().stream().noneMatch( - m -> m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) - ); + F.asList(cm1, cm2).forEach(cm -> { + assertTrue(cm.blockedMessages().stream().noneMatch(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal())); cm.stopBlock(); - } + }); - // Wait for future to finish in order to avoid excessive message about task cancellation. restoreFut.get(getTestTimeout()); } + /** */ + @Test + public void testOneSnapshotCheckStatus() throws Exception { + doTestSnapshotsChecksStatus(false); + } + + /** */ + @Test + public void testTwoSnapshotsChecksStatus() throws Exception { + doTestSnapshotsChecksStatus(true); + } + + /** */ + private void doTestSnapshotsChecksStatus(boolean twoSnapshots) throws Exception { + communicationSpiSupp = TestRecordingCommunicationSpi::new; + + startGrids(3).cluster().state(ClusterState.ACTIVE); + + IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 4096, 64, null); + snapshotMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); + + if (twoSnapshots) { + createCacheAndPreload(grid(1), "cache2", 4096, 64, null); + snapshotMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); + } + + grid(0).destroyCaches(twoSnapshots ? F.asList(DEFAULT_CACHE_NAME, "cache2") : F.asList(DEFAULT_CACHE_NAME)); + + awaitPartitionMapExchange(); + + TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + + F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()))); + + var checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); + var checkFut1 = twoSnapshots ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; + + // Waiting for the nodes each to send snapshot check single result. + for (var cm : F.asList(cm1, cm2)) { + assertTrue(waitForCondition( + () -> cm.blockedMessages().stream().filter( + m -> m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1), + getTestTimeout() + )); + } + + injectTestSystemOut(); + + assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status")); + + var out = testOut.toString(); + + if (log.isInfoEnabled()) + log.info("Test out:" + U.nl() + out); + + assertTrue(out.contains(twoSnapshots ? "Check snapshot operations are in progress" : "Check snapshot operation is in progress")); + assertTrue(out.contains("Snapshot name: testSnapshot0")); + if (twoSnapshots) + assertTrue(out.contains("Snapshot name: testSnapshot1")); + + F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); + + checkFut0.get(); + + if (twoSnapshots) + checkFut1.get(); + } + /** @throws Exception If fails. */ @Test public void testSnapshotStatusInMemory() throws Exception { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 52238cc119075..53735c84000cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.ignite.internal.management.SystemViewCommand; @@ -75,29 +74,21 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand multipleOpsView = isCreating || isRestoring ? Collections.singletonList(status) - : ((SnapshotStatusTaskV2.SnapshotStatusV2)status).allCheckStatuses; + : ((SnapshotStatusTaskV2.SnapshotStatusV2)status).checkStatuses(); - // Flag of additional line delimiter. - AtomicBoolean oneOp = new AtomicBoolean(true); + assert multipleOpsView.size() == 1 || !(isCreating || isRestoring) : "Only snapshot check supports multiple operations."; - multipleOpsView.forEach(s0 -> { - GridStringBuilder s = new GridStringBuilder(); - - if (!oneOp.get()) - printer.accept(U.nl()); + if (isCreating) + printer.accept("Create snapshot operation is in progress."); + else if (isRestoring) + printer.accept("Restore snapshot operation is in progress."); + else + printer.accept("Check snapshot operation" + (multipleOpsView.size() < 2 ? " is " : "s are ") + "in progress."); - if (isCreating) { - assert multipleOpsView.size() == 1; - - s.a("Create snapshot operation is in progress.").nl(); - } - else if (isRestoring) { - assert multipleOpsView.size() == 1; + printer.accept(U.nl()); - s.a("Restore snapshot operation is in progress.").nl(); - } - else - s.a("Check snapshot operation" + (multipleOpsView.size() < 2 ? " is " : "s are ") + "in progress.").nl(); + multipleOpsView.forEach(s0 -> { + GridStringBuilder s = new GridStringBuilder(); s.a("Snapshot name: ").a(s0.name()).nl(); s.a("Incremental: ").a(isIncremental).nl(); @@ -129,8 +120,6 @@ else if (isRestoring) SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); printer.accept(U.nl()); - - oneOp.set(false); }); } @@ -279,7 +268,7 @@ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressD private final boolean incremental; /** */ - CheckSnapshotTaskProgressDesc(boolean incremental) { + private CheckSnapshotTaskProgressDesc(boolean incremental) { super(incremental ? F.asList("Node ID", "processedWalSegments", "totalWalSegments", "percent") : F.asList("Node ID", "fullCheck", "processedPartitions", "totalPartitions", diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java index 9b185a67007c4..c699a097ab876 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java @@ -42,7 +42,7 @@ import org.apache.ignite.spi.metric.LongMetric; import org.jetbrains.annotations.Nullable; -/** V2 of {@link SnapshotStatusTask} with the support of snapshot check status. */ +/** V2 of {@link SnapshotStatusTask} with support of the snapshot check status. */ @GridInternal public class SnapshotStatusTaskV2 extends SnapshotStatusTask { /** */ @@ -78,11 +78,8 @@ private void resolveCheckStatusSupported() { for (var n : ignite.cluster().nodes()) { if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { - log.warning(String.format( - "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + - "for snapshot creation and restoration.", - n.id() - )); + log.warning(String.format("Node %s doesn't support the snapshot check status feature. The status " + + "is available only for snapshot creation and restoration.", n.id())); checkStatusSupported = false; @@ -101,7 +98,7 @@ private void resolveCheckStatusSupported() { if (res0 == null) return null; - // Found crate or restore result. + // Found create or restore result. if (res0.operation() != null) return res0; @@ -122,7 +119,7 @@ private void resolveCheckStatusSupported() { sameRqRes.forEach(s -> { assert s instanceof SnapshotStatusV2; - for (SnapshotStatus s0 : ((SnapshotStatusV2)s).allCheckStatuses) { + for (SnapshotStatus s0 : ((SnapshotStatusV2)s).checkStatuses) { var prev = statusesMap.putIfAbsent(s0.name(), s0); if (prev == null) @@ -132,13 +129,13 @@ private void resolveCheckStatusSupported() { prev.progress().putAll(s0.progress()); } - firstResV2.allCheckStatuses = new ArrayList<>(statusesMap.values()); + firstResV2.checkStatuses = new ArrayList<>(statusesMap.values()); }); return firstResV2; } - /** V2 of {@link SnapshotStatusJob} with the support of snapshot check status. */ + /** V2 of {@link SnapshotStatusJob} with support of the snapshot check status. */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { /** */ private static final long serialVersionUID = 0L; @@ -206,14 +203,13 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { } } - /** V2 of {@link SnapshotStatus} with the support of snapshot check status. */ - + /** V2 of {@link SnapshotStatus} with support of the snapshot check status. */ public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { /** */ private static final long serialVersionUID = 0L; /** Nodes' statuses of all snapshot check operations. */ - @Nullable List allCheckStatuses; + private List checkStatuses; /** */ private SnapshotStatusV2(SnapshotStatus s1) { @@ -221,18 +217,23 @@ private SnapshotStatusV2(SnapshotStatus s1) { } /** */ - private SnapshotStatusV2(List allCheckStatuses) { - // Single, V1 status holds first found check status. + private SnapshotStatusV2(List checkStatuses) { + // Single, V1 status holds first check status. super( null, - allCheckStatuses.get(0).name(), - allCheckStatuses.get(0).incrementIndex(), - allCheckStatuses.get(0).requestId(), - allCheckStatuses.get(0).startTime(), - allCheckStatuses.get(0).progress() + checkStatuses.get(0).name(), + checkStatuses.get(0).incrementIndex(), + checkStatuses.get(0).requestId(), + checkStatuses.get(0).startTime(), + checkStatuses.get(0).progress() ); - this.allCheckStatuses = allCheckStatuses; + this.checkStatuses = checkStatuses; + } + + /** */ + public List checkStatuses() { + return checkStatuses; } } } From 215c7dd18fbf559bfdd3e26f813aec1093d0092c Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 3 Sep 2026 09:07:57 +0300 Subject: [PATCH 17/17] + incremental snp test --- .../ignite/util/GridCommandHandlerTest.java | 88 +++++++++++++++---- 1 file changed, 73 insertions(+), 15 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 2275cab14a5df..6739ecafea6d2 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -252,6 +252,9 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected @Nullable PluginProvider pluginProvider; + /** */ + protected boolean walCompaction; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -281,6 +284,8 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (pluginProvider != null) cfg.setPluginProviders(pluginProvider); + cfg.getDataStorageConfiguration().setWalCompactionEnabled(walCompaction); + return cfg; } @@ -3740,11 +3745,11 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu startGrids(3).cluster().state(ClusterState.ACTIVE); - createCacheAndPreload(grid(1), 8192); + createCacheAndPreload(grid(1), 1000); - IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); - snapshotMgr.createSnapshot("test_snapshot").get(getTestTimeout()); + snpMgr.createSnapshot("test_snapshot").get(getTestTimeout()); grid(0).destroyCache(DEFAULT_CACHE_NAME); @@ -3758,7 +3763,7 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()))); // Snapshot restoration should get paused at the preceeding snapshot check. - IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); + IgniteFutureImpl restoreFut = snpMgr.restoreSnapshot("test_snapshot", null, null, 0, true); // Waiting for each node to send snapshot check single result. for (var cm : F.asList(cm1, cm2)) { @@ -3815,29 +3820,61 @@ else if (Boolean.FALSE.equals(allNodesNotSupporting)) { /** */ @Test public void testOneSnapshotCheckStatus() throws Exception { - doTestSnapshotsChecksStatus(false); + doTestSnapshotsChecksStatus(false, false); + } + + /** */ + @Test + public void testOneIncrementalSnapshotCheckStatus() throws Exception { + doTestSnapshotsChecksStatus(false, true); } /** */ @Test public void testTwoSnapshotsChecksStatus() throws Exception { - doTestSnapshotsChecksStatus(true); + doTestSnapshotsChecksStatus(true, false); } /** */ - private void doTestSnapshotsChecksStatus(boolean twoSnapshots) throws Exception { + @Test + public void testTwoIncrementalsSnapshotsChecksStatus() throws Exception { + doTestSnapshotsChecksStatus(true, true); + } + + /** */ + private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean incremental) throws Exception { communicationSpiSupp = TestRecordingCommunicationSpi::new; + walCompaction = incremental; + startGrids(3).cluster().state(ClusterState.ACTIVE); - IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); - createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 4096, 64, null); - snapshotMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 1000, 32, null); + snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); + + if (incremental) { + try (IgniteDataStreamer streamer = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 1000; i < 2000; i++) + streamer.addData(i, i); + } + + snpMgr.createIncrementalSnapshot("testSnapshot0").get(getTestTimeout()); + } if (twoSnapshots) { - createCacheAndPreload(grid(1), "cache2", 4096, 64, null); - snapshotMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); + createCacheAndPreload(grid(1), "cache2", 1000, 32, null); + snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); + + if (incremental) { + try (IgniteDataStreamer streamer = grid(0).dataStreamer("cache2")) { + for (int i = 1000; i < 2000; i++) + streamer.addData(i, i); + } + + snpMgr.createIncrementalSnapshot("testSnapshot1").get(getTestTimeout()); + } } grid(0).destroyCaches(twoSnapshots ? F.asList(DEFAULT_CACHE_NAME, "cache2") : F.asList(DEFAULT_CACHE_NAME)); @@ -3850,8 +3887,15 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots) throws Exception F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()))); - var checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); - var checkFut1 = twoSnapshots ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; + var checkFut0 = runAsync(() -> incremental + ? execute("--snapshot", "check", "testSnapshot0", "--increment", "1") + : execute("--snapshot", "check", "testSnapshot0")); + + var checkFut1 = twoSnapshots + ? runAsync(() -> incremental + ? execute("--snapshot", "check", "testSnapshot1", "--increment", "1") + : execute("--snapshot", "check", "testSnapshot1")) + : null; // Waiting for the nodes each to send snapshot check single result. for (var cm : F.asList(cm1, cm2)) { @@ -3874,9 +3918,23 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots) throws Exception assertTrue(out.contains(twoSnapshots ? "Check snapshot operations are in progress" : "Check snapshot operation is in progress")); assertTrue(out.contains("Snapshot name: testSnapshot0")); - if (twoSnapshots) + + if (incremental) + assertTrue(out.contains("Increment index: 1")); + + if (twoSnapshots) { assertTrue(out.contains("Snapshot name: testSnapshot1")); + if (incremental) { + // Number of 'Increment index: 1' entries. + var sum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace("Increment index: 1", "").length()) / "Increment index: 1".length()) + .sum(); + + assertEquals(2, sum); + } + } + F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); checkFut0.get();