From 9709fb7f5e270bb08545fe4fe4176e26d465af72 Mon Sep 17 00:00:00 2001 From: Shawn McKee Date: Fri, 13 Nov 2026 12:03:22 -0500 Subject: [PATCH] pool: fix start-time and end-time of firefly end markers Motivation: The 'end' firefly reports the time interval of the transfer as start-time = MoverInfoMessage#getTimestamp and end-time = start-time + connection time. The mover info message is created when the transfer has finished, so its timestamp is the end of the transfer, not its start. As a result both times are too late by the duration of the transfer: start-time is the moment the transfer ended and end-time lies in the future when the marker is sent. The start and end markers of the same transfer also disagree about when it started. Observed on a pool for one HTTP-TPC pull of 50.4 s: start marker, sent 16:23:15.710Z: start-time 16:23:15.710Z end marker, sent 16:24:06.215Z: start-time 16:24:06.214Z end-time 16:24:56.639Z The transfer ran from 16:23:15.7 to 16:24:06.2. The start-time was derived from the mover info timestamp following a review comment on #8044; the timestamp was then used as the start itself. Modification: Use the mover info timestamp as end-time and derive start-time by subtracting the connection time. The duration is unchanged. The unit test that pinned the previous behaviour is updated, and now also checks that end-time does not lie after the marker's current-time. The SciTags chapter of TheBook described the times of end markers the old way and is corrected. Rebased onto the onEnd refactoring of 985824cd56: the fix is now in onEnd(src, dst, MoverInfoMessage), which hands the derived start-time and the duration to the common onEnd. End markers of doors, which pass their own start time, are not affected. Result: The start-time and end-time of 'end' fireflies describe the interval in which the transfer took place, and agree with the start-time of the corresponding 'start' firefly. For the example above the derived start-time is 16:23:15.789Z. Acked-by: Target: master, 12.1, 12.0, 11.2 Require-book: no Require-notes: yes Signed-off-by: Shawn McKee Co-Authored-By: Claude Fable 5.2 --- docs/TheBook/src/main/markdown/config-scitags.md | 4 ++-- .../java/org/dcache/pool/movers/TransferLifeCycle.java | 6 +++++- .../org/dcache/pool/movers/TransferLifeCycleTest.java | 10 +++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/TheBook/src/main/markdown/config-scitags.md b/docs/TheBook/src/main/markdown/config-scitags.md index fb1bda2e13e..ac06d263f40 100644 --- a/docs/TheBook/src/main/markdown/config-scitags.md +++ b/docs/TheBook/src/main/markdown/config-scitags.md @@ -359,8 +359,8 @@ Main JSON sections: - `usage` (`received`, `sent`) - optional `storage-stats` when `pool.firefly.storage-statistics=true` -For end markers, `start-time` is reconstructed from mover submit time and `end-time` -is derived as `start-time + max(connectionTime, 0)`. +For end markers sent by a pool, `end-time` is the time the transfer finished and +`start-time` is derived as `end-time - max(connectionTime, 0)`. Test schema file: diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java b/modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java index 26ea20a5511..353f96b19da 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java @@ -152,8 +152,12 @@ public void onStart(InetSocketAddress src, InetSocketAddress dst, ProtocolInfo p * @param mover the mover's description of the completed transfer */ public void onEnd(InetSocketAddress src, InetSocketAddress dst, MoverInfoMessage mover) { + // The mover info message is created when the transfer has finished, so its + // timestamp is the end of the transfer; the start is derived from the duration. + long durationMillis = Math.max(0L, mover.getConnectionTime()); onEnd(src, dst, mover.getProtocolInfo(), mover.getSubject(), - Instant.ofEpochMilli(mover.getTimestamp()), mover.getConnectionTime(), + Instant.ofEpochMilli(mover.getTimestamp()).minusMillis(durationMillis), + durationMillis, mover.getBytesRead(), mover.getBytesWritten(), storageStatisticsEnabled ? mover : null); } diff --git a/modules/dcache/src/test/java/org/dcache/pool/movers/TransferLifeCycleTest.java b/modules/dcache/src/test/java/org/dcache/pool/movers/TransferLifeCycleTest.java index e9db2da30ef..81d064ae02b 100644 --- a/modules/dcache/src/test/java/org/dcache/pool/movers/TransferLifeCycleTest.java +++ b/modules/dcache/src/test/java/org/dcache/pool/movers/TransferLifeCycleTest.java @@ -138,7 +138,7 @@ public void shouldSendEndMarkerForTransferWithoutMover() throws Exception { } @Test - public void shouldSetEndMarkerStartTimeFromTransferDuration() throws Exception { + public void shouldDeriveEndMarkerTimesFromTransferEnd() throws Exception { try (DatagramSocket socket = new DatagramSocket(0, InetAddress.getByName("127.0.0.1"))) { socket.setSoTimeout(700); @@ -156,8 +156,9 @@ public void shouldSetEndMarkerStartTimeFromTransferDuration() throws Exception { mover.setBytesRead(1024); mover.setBytesWritten(0); - Instant expectedStart = Instant.ofEpochMilli(mover.getTimestamp()); - Instant expectedEnd = expectedStart.plusMillis(transferDuration); + // the mover info message is created when the transfer has finished + Instant expectedEnd = Instant.ofEpochMilli(mover.getTimestamp()); + Instant expectedStart = expectedEnd.minusMillis(transferDuration); lifecycle.onEnd( new InetSocketAddress("203.0.113.20", 42000), @@ -174,10 +175,13 @@ public void shouldSetEndMarkerStartTimeFromTransferDuration() throws Exception { Instant startTime = Instant.parse(lifecyclePayload.getString("start-time")); Instant endTime = Instant.parse(lifecyclePayload.getString("end-time")); + Instant currentTime = Instant.parse(lifecyclePayload.getString("current-time")); assertEquals(expectedStart, startTime); assertEquals(expectedEnd, endTime); assertEquals(transferDuration, Duration.between(startTime, endTime).toMillis()); + assertFalse("end-time must not lie after the time the marker was built", + endTime.isAfter(currentTime)); } }