From 22cfebce07553421c2c67e44dd5a24ff7cf64fd9 Mon Sep 17 00:00:00 2001 From: Shawn McKee Date: Wed, 18 Nov 2026 20:35:04 -0500 Subject: [PATCH] pool: add result code, file size, I/O mode, queue time and pool name to firefly storage statistics Motivation: With pool.firefly.storage-statistics enabled, the 'end' firefly carries a "storage-stats" section with the bytes transferred, the duration and the disk read/write rates and active/idle times. Several things a consumer needs to interpret these numbers are missing: - The outcome. A transfer that failed before any data moved sends an 'end' firefly in which every value is zero; nothing distinguishes it from an empty file that was transferred successfully. - The file size. Without it a partial transfer cannot be told from a complete one. - The direction. Whether the pool read or wrote the file has to be inferred from which of usage.sent and usage.received is non-zero. - The time spent waiting for a mover. connection-time starts when the mover runs, so a transfer delayed by a full mover queue looks the same as one that started at once. - The pool. The syslog header names the host, but a host commonly runs several pools. All but the queue time are already in the MoverInfoMessage the statistics are taken from. The queue time is not: InfoMessage has a timeQueued field, but the pool never sets it for a mover. Modification: FlowMarkerBuilder#withStats adds five keys to "storage-stats": result-code return code of the transfer, 0 is success file-size size of the file in bytes io-mode "write" if the pool received the file, "read" otherwise queue-time time the request waited in the mover queue, milliseconds pool-name cell name of the pool Mover gains getQueueTime(). AbstractMover records when it is created, which is when the request is added to the mover queue, and when execute() is called, which is when the queue grants it a slot; the difference is the queue time. DefaultPostTransferService copies it into the MoverInfoMessage. The SciTags chapter of TheBook and the description of pool.firefly.storage-statistics list the content of the section. Unit tests cover the new keys for a successful upload and a failed download, and the queue time before, at and after execution. Result: When storage statistics are enabled, 'end' fireflies tell whether the transfer succeeded, how large the file is, whether the pool read or wrote it, how long the request was queued and which pool served it. Nothing changes if pool.firefly.storage-statistics is false, which is the default. As a side effect the mover's billing record now carries its queue time: the "queuingTime" field of the Kafka record of a mover, which was always 0, has the real value. Acked-by: Target: master Require-book: yes Require-notes: yes Signed-off-by: Shawn McKee Co-Authored-By: Claude Fable 5.2 --- .../src/main/markdown/config-scitags.md | 14 ++- .../main/java/org/dcache/net/FlowMarker.java | 5 ++ .../classic/DefaultPostTransferService.java | 1 + .../org/dcache/pool/movers/AbstractMover.java | 14 +++ .../java/org/dcache/pool/movers/Mover.java | 7 ++ .../net/FlowMarkerStorageStatsTest.java | 86 +++++++++++++++++++ .../pool/movers/MoverQueueTimeTest.java | 55 ++++++++++++ skel/share/defaults/pool.properties | 6 ++ 8 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java create mode 100644 modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java diff --git a/docs/TheBook/src/main/markdown/config-scitags.md b/docs/TheBook/src/main/markdown/config-scitags.md index fb1bda2e13e..42f96c904b1 100644 --- a/docs/TheBook/src/main/markdown/config-scitags.md +++ b/docs/TheBook/src/main/markdown/config-scitags.md @@ -74,7 +74,19 @@ Pools and WebDAV doors each have their own properties, which default to the site properties. Set these only where a service must differ from the rest of the site. In addition, `pool.firefly.storage-statistics` controls whether pools include storage statistics -in end markers. +in end markers. The `storage-stats` section then holds: + +| Key | Meaning | +|-----|---------| +| `bytes-transferred` | bytes moved by the mover | +| `connection-time` | duration of the transfer in milliseconds | +| `read-bw`, `write-bw` | mean disk read and write rate in bytes per second while the mover was reading or writing | +| `read-active`, `read-idle`, `write-active`, `write-idle` | time spent in, and between, disk reads and writes (ISO-8601 duration, or `"0"` if not measured) | +| `result-code` | return code of the transfer; `0` is success | +| `file-size` | size of the file in bytes; compare with `bytes-transferred` to recognise a partial transfer | +| `io-mode` | `write` if the pool received the file, `read` if it sent it | +| `queue-time` | time the request waited in the pool's mover queue, in milliseconds | +| `pool-name` | name of the pool that served the transfer | Reference defaults: `skel/share/defaults/dcache.properties`, `skel/share/defaults/pool.properties` and `skel/share/defaults/webdav.properties`. diff --git a/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java b/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java index a7a787d4e55..2485bc43945 100644 --- a/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java +++ b/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java @@ -147,6 +147,11 @@ public FlowMarkerBuilder withStats(MoverInfoMessage message) { stats.put("read-idle", message.getReadIdle().isPresent() ? message.getReadIdle().get().toString() : "0"); stats.put("write-active", message.getWriteActive().isPresent() ? message.getWriteActive().get().toString() : "0"); stats.put("write-idle", message.getWriteIdle().isPresent() ? message.getWriteIdle().get().toString() : "0"); + stats.put("result-code", message.getResultCode()); + stats.put("file-size", message.getFileSize()); + stats.put("io-mode", message.isFileCreated() ? "write" : "read"); + stats.put("queue-time", message.getTimeQueued()); + stats.put("pool-name", message.getCellAddress().getCellName()); return this; } diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java index 07257bd591b..1aa44374a5c 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java @@ -169,6 +169,7 @@ public MoverInfoMessage generateBillingMessage(Mover mover, long fileSize) { info.setP2P(mover.isPoolToPoolTransfer()); info.setFileSize(fileSize); info.setResult(mover.getErrorCode(), mover.getErrorMessage()); + info.setTimeQueued(mover.getQueueTime()); info.setTransferAttributes(mover.getBytesTransferred(), mover.getTransferTime(), mover.getProtocolInfo()); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java index b69727bc4d5..9620b833078 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java @@ -40,6 +40,7 @@ import java.util.HashSet; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import javax.security.auth.Subject; import org.dcache.pool.classic.Cancellable; @@ -87,6 +88,13 @@ public abstract class AbstractMover

transferService) { TypeToken type = new TypeToken(getClass()) { @@ -227,8 +235,14 @@ public void close(CompletionHandler completionHandler) { _transferService.closeMover((M) this, completionHandler); } + @Override + public long getQueueTime() { + return _queueTime; + } + @Override public Cancellable execute(CompletionHandler completionHandler) { + _queueTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - _createdAt); return new TryCatchTemplate(completionHandler) { @Override @SuppressWarnings("unchecked") diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java index 8c679832be1..410db7f92f0 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java @@ -61,6 +61,13 @@ public interface Mover { */ long getTransferTime(); + /** + * Returns how long this mover waited in the pool's mover queue before it was executed. + * + * @return queue time in milliseconds, or zero if the mover has not been executed. + */ + long getQueueTime(); + /** * Number of bytes transferred by this mover. */ diff --git a/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java b/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java new file mode 100644 index 00000000000..50e71692185 --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java @@ -0,0 +1,86 @@ +package org.dcache.net; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import diskCacheV111.util.CacheException; +import diskCacheV111.util.PnfsId; +import diskCacheV111.vehicles.MoverInfoMessage; +import dmg.cells.nucleus.CellAddressCore; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.time.Instant; +import org.dcache.net.FlowMarker.FlowMarkerBuilder; +import org.json.JSONObject; +import org.junit.Test; + +public class FlowMarkerStorageStatsTest { + + private static final PnfsId PNFSID = new PnfsId("000000000000000000000000000000000001"); + + @Test + public void shouldDescribeSuccessfulUpload() { + var info = new MoverInfoMessage(new CellAddressCore("pool_a", "poolDomain"), PNFSID); + info.setFileCreated(true); + info.setFileSize(4096); + info.setResult(0, ""); + info.setTimeQueued(250); + info.setTransferAttributes(4096, 1500, null); + + JSONObject stats = storageStats(info); + + assertEquals(0, stats.getInt("result-code")); + assertEquals(4096L, stats.getLong("file-size")); + assertEquals("write", stats.getString("io-mode")); + assertEquals(250L, stats.getLong("queue-time")); + assertEquals("pool_a", stats.getString("pool-name")); + assertEquals(4096L, stats.getLong("bytes-transferred")); + assertEquals(1500L, stats.getLong("connection-time")); + } + + @Test + public void shouldDescribeFailedDownload() { + var info = new MoverInfoMessage(new CellAddressCore("pool_b", "poolDomain"), PNFSID); + info.setFileCreated(false); + info.setFileSize(8192); + info.setResult(CacheException.DEFAULT_ERROR_CODE, "General problem"); + info.setTransferAttributes(1024, 300, null); + + JSONObject stats = storageStats(info); + + assertEquals(CacheException.DEFAULT_ERROR_CODE, stats.getInt("result-code")); + assertEquals(8192L, stats.getLong("file-size")); + assertEquals("read", stats.getString("io-mode")); + assertEquals(0L, stats.getLong("queue-time")); + assertEquals("pool_b", stats.getString("pool-name")); + assertEquals(1024L, stats.getLong("bytes-transferred")); + } + + @Test + public void shouldOmitStorageStatsUnlessRequested() { + String firefly = builder().build("end"); + + assertFalse(payload(firefly).has("storage-stats")); + } + + private static JSONObject storageStats(MoverInfoMessage info) { + return payload(builder().withStats(info).build("end")).getJSONObject("storage-stats"); + } + + private static FlowMarkerBuilder builder() { + return new FlowMarkerBuilder() + .withStartedAt(Instant.now()) + .withFinishedAt(Instant.now()) + .withExperimentId(2) + .withActivityId(1) + .wittApplication("https") + .withProtocol("tcp") + .withAFI("ipv4") + .withDestination(new InetSocketAddress(InetAddress.getLoopbackAddress(), 1234)) + .withSource(new InetSocketAddress(InetAddress.getLoopbackAddress(), 5678)); + } + + private static JSONObject payload(String firefly) { + return new JSONObject(firefly.substring(firefly.indexOf("{"))); + } +} diff --git a/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java b/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java new file mode 100644 index 00000000000..dab4f55ea0e --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java @@ -0,0 +1,55 @@ +package org.dcache.pool.movers; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +import diskCacheV111.vehicles.PoolDeliverFileMessage; +import dmg.cells.nucleus.CellPath; +import java.nio.channels.CompletionHandler; +import org.dcache.pool.classic.TransferService; +import org.dcache.pool.repository.ReplicaDescriptor; +import org.junit.Before; +import org.junit.Test; + +public class MoverQueueTimeTest { + + private MoverProtocolMover mover; + + @Before + @SuppressWarnings("unchecked") + public void setUp() { + mover = new MoverProtocolMover(mock(ReplicaDescriptor.class), + mock(PoolDeliverFileMessage.class), new CellPath("door"), + mock(TransferService.class), mock(MoverProtocol.class)); + } + + @Test + public void shouldReportNoQueueTimeBeforeExecution() { + assertEquals(0L, mover.getQueueTime()); + } + + @Test + @SuppressWarnings("unchecked") + public void shouldReportTimeBetweenCreationAndExecution() throws Exception { + Thread.sleep(50); + + mover.execute(mock(CompletionHandler.class)); + + assertThat(mover.getQueueTime(), greaterThanOrEqualTo(40L)); + assertThat(mover.getQueueTime(), lessThan(10_000L)); + } + + @Test + @SuppressWarnings("unchecked") + public void shouldNotCountTimeAfterExecutionStarted() throws Exception { + mover.execute(mock(CompletionHandler.class)); + long queueTime = mover.getQueueTime(); + + Thread.sleep(50); + + assertEquals(queueTime, mover.getQueueTime()); + } +} diff --git a/skel/share/defaults/pool.properties b/skel/share/defaults/pool.properties index 27b1a0f7a90..a249fa736be 100644 --- a/skel/share/defaults/pool.properties +++ b/skel/share/defaults/pool.properties @@ -898,6 +898,12 @@ pool.firefly.vo-mapping=${dcache.firefly.vo-mapping} # # Enable sending storage statistics as part of the firefly packets # +# If enabled, end markers carry a "storage-stats" section with the bytes +# transferred, the duration, disk read and write rates and active/idle +# times, the transfer's return code, the file size, whether the pool read +# or wrote the file, the time the request waited in the mover queue and the +# pool name. +# pool.firefly.storage-statistics=false #