diff --git a/docs/TheBook/src/main/markdown/config-scitags.md b/docs/TheBook/src/main/markdown/config-scitags.md
index fb1bda2e13e..42f96c904b1 100644
--- a/docs/TheBook/src/main/markdown/config-scitags.md
+++ b/docs/TheBook/src/main/markdown/config-scitags.md
@@ -74,7 +74,19 @@ Pools and WebDAV doors each have their own properties, which default to the site
properties. Set these only where a service must differ from the rest of the site.
In addition, `pool.firefly.storage-statistics` controls whether pools include storage statistics
-in end markers.
+in end markers. The `storage-stats` section then holds:
+
+| Key | Meaning |
+|-----|---------|
+| `bytes-transferred` | bytes moved by the mover |
+| `connection-time` | duration of the transfer in milliseconds |
+| `read-bw`, `write-bw` | mean disk read and write rate in bytes per second while the mover was reading or writing |
+| `read-active`, `read-idle`, `write-active`, `write-idle` | time spent in, and between, disk reads and writes (ISO-8601 duration, or `"0"` if not measured) |
+| `result-code` | return code of the transfer; `0` is success |
+| `file-size` | size of the file in bytes; compare with `bytes-transferred` to recognise a partial transfer |
+| `io-mode` | `write` if the pool received the file, `read` if it sent it |
+| `queue-time` | time the request waited in the pool's mover queue, in milliseconds |
+| `pool-name` | name of the pool that served the transfer |
Reference defaults: `skel/share/defaults/dcache.properties`, `skel/share/defaults/pool.properties`
and `skel/share/defaults/webdav.properties`.
diff --git a/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java b/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java
index a7a787d4e55..2485bc43945 100644
--- a/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java
+++ b/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java
@@ -147,6 +147,11 @@ public FlowMarkerBuilder withStats(MoverInfoMessage message) {
stats.put("read-idle", message.getReadIdle().isPresent() ? message.getReadIdle().get().toString() : "0");
stats.put("write-active", message.getWriteActive().isPresent() ? message.getWriteActive().get().toString() : "0");
stats.put("write-idle", message.getWriteIdle().isPresent() ? message.getWriteIdle().get().toString() : "0");
+ stats.put("result-code", message.getResultCode());
+ stats.put("file-size", message.getFileSize());
+ stats.put("io-mode", message.isFileCreated() ? "write" : "read");
+ stats.put("queue-time", message.getTimeQueued());
+ stats.put("pool-name", message.getCellAddress().getCellName());
return this;
}
diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java
index 07257bd591b..1aa44374a5c 100644
--- a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java
+++ b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java
@@ -169,6 +169,7 @@ public MoverInfoMessage generateBillingMessage(Mover> mover, long fileSize) {
info.setP2P(mover.isPoolToPoolTransfer());
info.setFileSize(fileSize);
info.setResult(mover.getErrorCode(), mover.getErrorMessage());
+ info.setTimeQueued(mover.getQueueTime());
info.setTransferAttributes(mover.getBytesTransferred(),
mover.getTransferTime(),
mover.getProtocolInfo());
diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java
index b69727bc4d5..9620b833078 100644
--- a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java
+++ b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java
@@ -40,6 +40,7 @@
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.security.auth.Subject;
import org.dcache.pool.classic.Cancellable;
@@ -87,6 +88,13 @@ public abstract class AbstractMover
transferService) {
TypeToken type = new TypeToken(getClass()) {
@@ -227,8 +235,14 @@ public void close(CompletionHandler completionHandler) {
_transferService.closeMover((M) this, completionHandler);
}
+ @Override
+ public long getQueueTime() {
+ return _queueTime;
+ }
+
@Override
public Cancellable execute(CompletionHandler completionHandler) {
+ _queueTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - _createdAt);
return new TryCatchTemplate(completionHandler) {
@Override
@SuppressWarnings("unchecked")
diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java
index 8c679832be1..410db7f92f0 100644
--- a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java
+++ b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java
@@ -61,6 +61,13 @@ public interface Mover {
*/
long getTransferTime();
+ /**
+ * Returns how long this mover waited in the pool's mover queue before it was executed.
+ *
+ * @return queue time in milliseconds, or zero if the mover has not been executed.
+ */
+ long getQueueTime();
+
/**
* Number of bytes transferred by this mover.
*/
diff --git a/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java b/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java
new file mode 100644
index 00000000000..50e71692185
--- /dev/null
+++ b/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java
@@ -0,0 +1,86 @@
+package org.dcache.net;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import diskCacheV111.util.CacheException;
+import diskCacheV111.util.PnfsId;
+import diskCacheV111.vehicles.MoverInfoMessage;
+import dmg.cells.nucleus.CellAddressCore;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.time.Instant;
+import org.dcache.net.FlowMarker.FlowMarkerBuilder;
+import org.json.JSONObject;
+import org.junit.Test;
+
+public class FlowMarkerStorageStatsTest {
+
+ private static final PnfsId PNFSID = new PnfsId("000000000000000000000000000000000001");
+
+ @Test
+ public void shouldDescribeSuccessfulUpload() {
+ var info = new MoverInfoMessage(new CellAddressCore("pool_a", "poolDomain"), PNFSID);
+ info.setFileCreated(true);
+ info.setFileSize(4096);
+ info.setResult(0, "");
+ info.setTimeQueued(250);
+ info.setTransferAttributes(4096, 1500, null);
+
+ JSONObject stats = storageStats(info);
+
+ assertEquals(0, stats.getInt("result-code"));
+ assertEquals(4096L, stats.getLong("file-size"));
+ assertEquals("write", stats.getString("io-mode"));
+ assertEquals(250L, stats.getLong("queue-time"));
+ assertEquals("pool_a", stats.getString("pool-name"));
+ assertEquals(4096L, stats.getLong("bytes-transferred"));
+ assertEquals(1500L, stats.getLong("connection-time"));
+ }
+
+ @Test
+ public void shouldDescribeFailedDownload() {
+ var info = new MoverInfoMessage(new CellAddressCore("pool_b", "poolDomain"), PNFSID);
+ info.setFileCreated(false);
+ info.setFileSize(8192);
+ info.setResult(CacheException.DEFAULT_ERROR_CODE, "General problem");
+ info.setTransferAttributes(1024, 300, null);
+
+ JSONObject stats = storageStats(info);
+
+ assertEquals(CacheException.DEFAULT_ERROR_CODE, stats.getInt("result-code"));
+ assertEquals(8192L, stats.getLong("file-size"));
+ assertEquals("read", stats.getString("io-mode"));
+ assertEquals(0L, stats.getLong("queue-time"));
+ assertEquals("pool_b", stats.getString("pool-name"));
+ assertEquals(1024L, stats.getLong("bytes-transferred"));
+ }
+
+ @Test
+ public void shouldOmitStorageStatsUnlessRequested() {
+ String firefly = builder().build("end");
+
+ assertFalse(payload(firefly).has("storage-stats"));
+ }
+
+ private static JSONObject storageStats(MoverInfoMessage info) {
+ return payload(builder().withStats(info).build("end")).getJSONObject("storage-stats");
+ }
+
+ private static FlowMarkerBuilder builder() {
+ return new FlowMarkerBuilder()
+ .withStartedAt(Instant.now())
+ .withFinishedAt(Instant.now())
+ .withExperimentId(2)
+ .withActivityId(1)
+ .wittApplication("https")
+ .withProtocol("tcp")
+ .withAFI("ipv4")
+ .withDestination(new InetSocketAddress(InetAddress.getLoopbackAddress(), 1234))
+ .withSource(new InetSocketAddress(InetAddress.getLoopbackAddress(), 5678));
+ }
+
+ private static JSONObject payload(String firefly) {
+ return new JSONObject(firefly.substring(firefly.indexOf("{")));
+ }
+}
diff --git a/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java b/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java
new file mode 100644
index 00000000000..dab4f55ea0e
--- /dev/null
+++ b/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java
@@ -0,0 +1,55 @@
+package org.dcache.pool.movers;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+import diskCacheV111.vehicles.PoolDeliverFileMessage;
+import dmg.cells.nucleus.CellPath;
+import java.nio.channels.CompletionHandler;
+import org.dcache.pool.classic.TransferService;
+import org.dcache.pool.repository.ReplicaDescriptor;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MoverQueueTimeTest {
+
+ private MoverProtocolMover mover;
+
+ @Before
+ @SuppressWarnings("unchecked")
+ public void setUp() {
+ mover = new MoverProtocolMover(mock(ReplicaDescriptor.class),
+ mock(PoolDeliverFileMessage.class), new CellPath("door"),
+ mock(TransferService.class), mock(MoverProtocol.class));
+ }
+
+ @Test
+ public void shouldReportNoQueueTimeBeforeExecution() {
+ assertEquals(0L, mover.getQueueTime());
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void shouldReportTimeBetweenCreationAndExecution() throws Exception {
+ Thread.sleep(50);
+
+ mover.execute(mock(CompletionHandler.class));
+
+ assertThat(mover.getQueueTime(), greaterThanOrEqualTo(40L));
+ assertThat(mover.getQueueTime(), lessThan(10_000L));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void shouldNotCountTimeAfterExecutionStarted() throws Exception {
+ mover.execute(mock(CompletionHandler.class));
+ long queueTime = mover.getQueueTime();
+
+ Thread.sleep(50);
+
+ assertEquals(queueTime, mover.getQueueTime());
+ }
+}
diff --git a/skel/share/defaults/pool.properties b/skel/share/defaults/pool.properties
index 27b1a0f7a90..a249fa736be 100644
--- a/skel/share/defaults/pool.properties
+++ b/skel/share/defaults/pool.properties
@@ -898,6 +898,12 @@ pool.firefly.vo-mapping=${dcache.firefly.vo-mapping}
#
# Enable sending storage statistics as part of the firefly packets
#
+# If enabled, end markers carry a "storage-stats" section with the bytes
+# transferred, the duration, disk read and write rates and active/idle
+# times, the transfer's return code, the file size, whether the pool read
+# or wrote the file, the time the request waited in the mover queue and the
+# pool name.
+#
pool.firefly.storage-statistics=false
#