diff --git a/docs/TheBook/src/main/markdown/config-scitags.md b/docs/TheBook/src/main/markdown/config-scitags.md index fb1bda2e13e..42f96c904b1 100644 --- a/docs/TheBook/src/main/markdown/config-scitags.md +++ b/docs/TheBook/src/main/markdown/config-scitags.md @@ -74,7 +74,19 @@ Pools and WebDAV doors each have their own properties, which default to the site properties. Set these only where a service must differ from the rest of the site. In addition, `pool.firefly.storage-statistics` controls whether pools include storage statistics -in end markers. +in end markers. The `storage-stats` section then holds: + +| Key | Meaning | +|-----|---------| +| `bytes-transferred` | bytes moved by the mover | +| `connection-time` | duration of the transfer in milliseconds | +| `read-bw`, `write-bw` | mean disk read and write rate in bytes per second while the mover was reading or writing | +| `read-active`, `read-idle`, `write-active`, `write-idle` | time spent in, and between, disk reads and writes (ISO-8601 duration, or `"0"` if not measured) | +| `result-code` | return code of the transfer; `0` is success | +| `file-size` | size of the file in bytes; compare with `bytes-transferred` to recognise a partial transfer | +| `io-mode` | `write` if the pool received the file, `read` if it sent it | +| `queue-time` | time the request waited in the pool's mover queue, in milliseconds | +| `pool-name` | name of the pool that served the transfer | Reference defaults: `skel/share/defaults/dcache.properties`, `skel/share/defaults/pool.properties` and `skel/share/defaults/webdav.properties`. diff --git a/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java b/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java index a7a787d4e55..2485bc43945 100644 --- a/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java +++ b/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java @@ -147,6 +147,11 @@ public FlowMarkerBuilder withStats(MoverInfoMessage message) { stats.put("read-idle", message.getReadIdle().isPresent() ? message.getReadIdle().get().toString() : "0"); stats.put("write-active", message.getWriteActive().isPresent() ? message.getWriteActive().get().toString() : "0"); stats.put("write-idle", message.getWriteIdle().isPresent() ? message.getWriteIdle().get().toString() : "0"); + stats.put("result-code", message.getResultCode()); + stats.put("file-size", message.getFileSize()); + stats.put("io-mode", message.isFileCreated() ? "write" : "read"); + stats.put("queue-time", message.getTimeQueued()); + stats.put("pool-name", message.getCellAddress().getCellName()); return this; } diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java index 07257bd591b..1aa44374a5c 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java @@ -169,6 +169,7 @@ public MoverInfoMessage generateBillingMessage(Mover mover, long fileSize) { info.setP2P(mover.isPoolToPoolTransfer()); info.setFileSize(fileSize); info.setResult(mover.getErrorCode(), mover.getErrorMessage()); + info.setTimeQueued(mover.getQueueTime()); info.setTransferAttributes(mover.getBytesTransferred(), mover.getTransferTime(), mover.getProtocolInfo()); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java index b69727bc4d5..9620b833078 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java @@ -40,6 +40,7 @@ import java.util.HashSet; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import javax.security.auth.Subject; import org.dcache.pool.classic.Cancellable; @@ -87,6 +88,13 @@ public abstract class AbstractMover

transferService) { TypeToken type = new TypeToken(getClass()) { @@ -227,8 +235,14 @@ public void close(CompletionHandler completionHandler) { _transferService.closeMover((M) this, completionHandler); } + @Override + public long getQueueTime() { + return _queueTime; + } + @Override public Cancellable execute(CompletionHandler completionHandler) { + _queueTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - _createdAt); return new TryCatchTemplate(completionHandler) { @Override @SuppressWarnings("unchecked") diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java index 8c679832be1..410db7f92f0 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java @@ -61,6 +61,13 @@ public interface Mover { */ long getTransferTime(); + /** + * Returns how long this mover waited in the pool's mover queue before it was executed. + * + * @return queue time in milliseconds, or zero if the mover has not been executed. + */ + long getQueueTime(); + /** * Number of bytes transferred by this mover. */ diff --git a/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java b/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java new file mode 100644 index 00000000000..50e71692185 --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/net/FlowMarkerStorageStatsTest.java @@ -0,0 +1,86 @@ +package org.dcache.net; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import diskCacheV111.util.CacheException; +import diskCacheV111.util.PnfsId; +import diskCacheV111.vehicles.MoverInfoMessage; +import dmg.cells.nucleus.CellAddressCore; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.time.Instant; +import org.dcache.net.FlowMarker.FlowMarkerBuilder; +import org.json.JSONObject; +import org.junit.Test; + +public class FlowMarkerStorageStatsTest { + + private static final PnfsId PNFSID = new PnfsId("000000000000000000000000000000000001"); + + @Test + public void shouldDescribeSuccessfulUpload() { + var info = new MoverInfoMessage(new CellAddressCore("pool_a", "poolDomain"), PNFSID); + info.setFileCreated(true); + info.setFileSize(4096); + info.setResult(0, ""); + info.setTimeQueued(250); + info.setTransferAttributes(4096, 1500, null); + + JSONObject stats = storageStats(info); + + assertEquals(0, stats.getInt("result-code")); + assertEquals(4096L, stats.getLong("file-size")); + assertEquals("write", stats.getString("io-mode")); + assertEquals(250L, stats.getLong("queue-time")); + assertEquals("pool_a", stats.getString("pool-name")); + assertEquals(4096L, stats.getLong("bytes-transferred")); + assertEquals(1500L, stats.getLong("connection-time")); + } + + @Test + public void shouldDescribeFailedDownload() { + var info = new MoverInfoMessage(new CellAddressCore("pool_b", "poolDomain"), PNFSID); + info.setFileCreated(false); + info.setFileSize(8192); + info.setResult(CacheException.DEFAULT_ERROR_CODE, "General problem"); + info.setTransferAttributes(1024, 300, null); + + JSONObject stats = storageStats(info); + + assertEquals(CacheException.DEFAULT_ERROR_CODE, stats.getInt("result-code")); + assertEquals(8192L, stats.getLong("file-size")); + assertEquals("read", stats.getString("io-mode")); + assertEquals(0L, stats.getLong("queue-time")); + assertEquals("pool_b", stats.getString("pool-name")); + assertEquals(1024L, stats.getLong("bytes-transferred")); + } + + @Test + public void shouldOmitStorageStatsUnlessRequested() { + String firefly = builder().build("end"); + + assertFalse(payload(firefly).has("storage-stats")); + } + + private static JSONObject storageStats(MoverInfoMessage info) { + return payload(builder().withStats(info).build("end")).getJSONObject("storage-stats"); + } + + private static FlowMarkerBuilder builder() { + return new FlowMarkerBuilder() + .withStartedAt(Instant.now()) + .withFinishedAt(Instant.now()) + .withExperimentId(2) + .withActivityId(1) + .wittApplication("https") + .withProtocol("tcp") + .withAFI("ipv4") + .withDestination(new InetSocketAddress(InetAddress.getLoopbackAddress(), 1234)) + .withSource(new InetSocketAddress(InetAddress.getLoopbackAddress(), 5678)); + } + + private static JSONObject payload(String firefly) { + return new JSONObject(firefly.substring(firefly.indexOf("{"))); + } +} diff --git a/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java b/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java new file mode 100644 index 00000000000..dab4f55ea0e --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/pool/movers/MoverQueueTimeTest.java @@ -0,0 +1,55 @@ +package org.dcache.pool.movers; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +import diskCacheV111.vehicles.PoolDeliverFileMessage; +import dmg.cells.nucleus.CellPath; +import java.nio.channels.CompletionHandler; +import org.dcache.pool.classic.TransferService; +import org.dcache.pool.repository.ReplicaDescriptor; +import org.junit.Before; +import org.junit.Test; + +public class MoverQueueTimeTest { + + private MoverProtocolMover mover; + + @Before + @SuppressWarnings("unchecked") + public void setUp() { + mover = new MoverProtocolMover(mock(ReplicaDescriptor.class), + mock(PoolDeliverFileMessage.class), new CellPath("door"), + mock(TransferService.class), mock(MoverProtocol.class)); + } + + @Test + public void shouldReportNoQueueTimeBeforeExecution() { + assertEquals(0L, mover.getQueueTime()); + } + + @Test + @SuppressWarnings("unchecked") + public void shouldReportTimeBetweenCreationAndExecution() throws Exception { + Thread.sleep(50); + + mover.execute(mock(CompletionHandler.class)); + + assertThat(mover.getQueueTime(), greaterThanOrEqualTo(40L)); + assertThat(mover.getQueueTime(), lessThan(10_000L)); + } + + @Test + @SuppressWarnings("unchecked") + public void shouldNotCountTimeAfterExecutionStarted() throws Exception { + mover.execute(mock(CompletionHandler.class)); + long queueTime = mover.getQueueTime(); + + Thread.sleep(50); + + assertEquals(queueTime, mover.getQueueTime()); + } +} diff --git a/skel/share/defaults/pool.properties b/skel/share/defaults/pool.properties index 27b1a0f7a90..a249fa736be 100644 --- a/skel/share/defaults/pool.properties +++ b/skel/share/defaults/pool.properties @@ -898,6 +898,12 @@ pool.firefly.vo-mapping=${dcache.firefly.vo-mapping} # # Enable sending storage statistics as part of the firefly packets # +# If enabled, end markers carry a "storage-stats" section with the bytes +# transferred, the duration, disk read and write rates and active/idle +# times, the transfer's return code, the file size, whether the pool read +# or wrote the file, the time the request waited in the mover queue and the +# pool name. +# pool.firefly.storage-statistics=false #