Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion docs/TheBook/src/main/markdown/config-scitags.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,19 @@ Pools and WebDAV doors each have their own properties, which default to the site
properties. Set these only where a service must differ from the rest of the site.

In addition, `pool.firefly.storage-statistics` controls whether pools include storage statistics
in end markers.
in end markers. The `storage-stats` section then holds:

| Key | Meaning |
|-----|---------|
| `bytes-transferred` | bytes moved by the mover |
| `connection-time` | duration of the transfer in milliseconds |
| `read-bw`, `write-bw` | mean disk read and write rate in bytes per second while the mover was reading or writing |
| `read-active`, `read-idle`, `write-active`, `write-idle` | time spent in, and between, disk reads and writes (ISO-8601 duration, or `"0"` if not measured) |
| `result-code` | return code of the transfer; `0` is success |
| `file-size` | size of the file in bytes; compare with `bytes-transferred` to recognise a partial transfer |
| `io-mode` | `write` if the pool received the file, `read` if it sent it |
| `queue-time` | time the request waited in the pool's mover queue, in milliseconds |
| `pool-name` | name of the pool that served the transfer |

Reference defaults: `skel/share/defaults/dcache.properties`, `skel/share/defaults/pool.properties`
and `skel/share/defaults/webdav.properties`.
Expand Down
5 changes: 5 additions & 0 deletions modules/dcache/src/main/java/org/dcache/net/FlowMarker.java
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,11 @@ public FlowMarkerBuilder withStats(MoverInfoMessage message) {
stats.put("read-idle", message.getReadIdle().isPresent() ? message.getReadIdle().get().toString() : "0");
stats.put("write-active", message.getWriteActive().isPresent() ? message.getWriteActive().get().toString() : "0");
stats.put("write-idle", message.getWriteIdle().isPresent() ? message.getWriteIdle().get().toString() : "0");
stats.put("result-code", message.getResultCode());
stats.put("file-size", message.getFileSize());
stats.put("io-mode", message.isFileCreated() ? "write" : "read");
stats.put("queue-time", message.getTimeQueued());
stats.put("pool-name", message.getCellAddress().getCellName());
return this;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ public MoverInfoMessage generateBillingMessage(Mover<?> mover, long fileSize) {
info.setP2P(mover.isPoolToPoolTransfer());
info.setFileSize(fileSize);
info.setResult(mover.getErrorCode(), mover.getErrorMessage());
info.setTimeQueued(mover.getQueueTime());
info.setTransferAttributes(mover.getBytesTransferred(),
mover.getTransferTime(),
mover.getProtocolInfo());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.security.auth.Subject;
import org.dcache.pool.classic.Cancellable;
Expand Down Expand Up @@ -87,6 +88,13 @@ public abstract class AbstractMover<P extends ProtocolInfo, M extends AbstractMo
private volatile InetSocketAddress _localEndpoint;
private volatile InetSocketAddress _remoteEndpoint;

/**
* The mover is created when the request is added to the mover queue and executed when the
* queue grants it a slot; the time between the two is the time spent queued.
*/
private final long _createdAt = System.nanoTime();
private volatile long _queueTime;

public AbstractMover(ReplicaDescriptor handle, PoolIoFileMessage message, CellPath pathToDoor,
TransferService<M> transferService) {
TypeToken<M> type = new TypeToken<M>(getClass()) {
Expand Down Expand Up @@ -227,8 +235,14 @@ public void close(CompletionHandler<Void, Void> completionHandler) {
_transferService.closeMover((M) this, completionHandler);
}

@Override
public long getQueueTime() {
return _queueTime;
}

@Override
public Cancellable execute(CompletionHandler<Void, Void> completionHandler) {
_queueTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - _createdAt);
return new TryCatchTemplate<Void, Void>(completionHandler) {
@Override
@SuppressWarnings("unchecked")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ public interface Mover<T extends ProtocolInfo> {
*/
long getTransferTime();

/**
* Returns how long this mover waited in the pool's mover queue before it was executed.
*
* @return queue time in milliseconds, or zero if the mover has not been executed.
*/
long getQueueTime();

/**
* Number of bytes transferred by this mover.
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package org.dcache.net;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;

import diskCacheV111.util.CacheException;
import diskCacheV111.util.PnfsId;
import diskCacheV111.vehicles.MoverInfoMessage;
import dmg.cells.nucleus.CellAddressCore;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.time.Instant;
import org.dcache.net.FlowMarker.FlowMarkerBuilder;
import org.json.JSONObject;
import org.junit.Test;

public class FlowMarkerStorageStatsTest {

private static final PnfsId PNFSID = new PnfsId("000000000000000000000000000000000001");

@Test
public void shouldDescribeSuccessfulUpload() {
var info = new MoverInfoMessage(new CellAddressCore("pool_a", "poolDomain"), PNFSID);
info.setFileCreated(true);
info.setFileSize(4096);
info.setResult(0, "");
info.setTimeQueued(250);
info.setTransferAttributes(4096, 1500, null);

JSONObject stats = storageStats(info);

assertEquals(0, stats.getInt("result-code"));
assertEquals(4096L, stats.getLong("file-size"));
assertEquals("write", stats.getString("io-mode"));
assertEquals(250L, stats.getLong("queue-time"));
assertEquals("pool_a", stats.getString("pool-name"));
assertEquals(4096L, stats.getLong("bytes-transferred"));
assertEquals(1500L, stats.getLong("connection-time"));
}

@Test
public void shouldDescribeFailedDownload() {
var info = new MoverInfoMessage(new CellAddressCore("pool_b", "poolDomain"), PNFSID);
info.setFileCreated(false);
info.setFileSize(8192);
info.setResult(CacheException.DEFAULT_ERROR_CODE, "General problem");
info.setTransferAttributes(1024, 300, null);

JSONObject stats = storageStats(info);

assertEquals(CacheException.DEFAULT_ERROR_CODE, stats.getInt("result-code"));
assertEquals(8192L, stats.getLong("file-size"));
assertEquals("read", stats.getString("io-mode"));
assertEquals(0L, stats.getLong("queue-time"));
assertEquals("pool_b", stats.getString("pool-name"));
assertEquals(1024L, stats.getLong("bytes-transferred"));
}

@Test
public void shouldOmitStorageStatsUnlessRequested() {
String firefly = builder().build("end");

assertFalse(payload(firefly).has("storage-stats"));
}

private static JSONObject storageStats(MoverInfoMessage info) {
return payload(builder().withStats(info).build("end")).getJSONObject("storage-stats");
}

private static FlowMarkerBuilder builder() {
return new FlowMarkerBuilder()
.withStartedAt(Instant.now())
.withFinishedAt(Instant.now())
.withExperimentId(2)
.withActivityId(1)
.wittApplication("https")
.withProtocol("tcp")
.withAFI("ipv4")
.withDestination(new InetSocketAddress(InetAddress.getLoopbackAddress(), 1234))
.withSource(new InetSocketAddress(InetAddress.getLoopbackAddress(), 5678));
}

private static JSONObject payload(String firefly) {
return new JSONObject(firefly.substring(firefly.indexOf("{")));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package org.dcache.pool.movers;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.lessThan;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;

import diskCacheV111.vehicles.PoolDeliverFileMessage;
import dmg.cells.nucleus.CellPath;
import java.nio.channels.CompletionHandler;
import org.dcache.pool.classic.TransferService;
import org.dcache.pool.repository.ReplicaDescriptor;
import org.junit.Before;
import org.junit.Test;

public class MoverQueueTimeTest {

private MoverProtocolMover mover;

@Before
@SuppressWarnings("unchecked")
public void setUp() {
mover = new MoverProtocolMover(mock(ReplicaDescriptor.class),
mock(PoolDeliverFileMessage.class), new CellPath("door"),
mock(TransferService.class), mock(MoverProtocol.class));
}

@Test
public void shouldReportNoQueueTimeBeforeExecution() {
assertEquals(0L, mover.getQueueTime());
}

@Test
@SuppressWarnings("unchecked")
public void shouldReportTimeBetweenCreationAndExecution() throws Exception {
Thread.sleep(50);

mover.execute(mock(CompletionHandler.class));

assertThat(mover.getQueueTime(), greaterThanOrEqualTo(40L));
assertThat(mover.getQueueTime(), lessThan(10_000L));
}

@Test
@SuppressWarnings("unchecked")
public void shouldNotCountTimeAfterExecutionStarted() throws Exception {
mover.execute(mock(CompletionHandler.class));
long queueTime = mover.getQueueTime();

Thread.sleep(50);

assertEquals(queueTime, mover.getQueueTime());
}
}
6 changes: 6 additions & 0 deletions skel/share/defaults/pool.properties
Original file line number Diff line number Diff line change
Expand Up @@ -898,6 +898,12 @@ pool.firefly.vo-mapping=${dcache.firefly.vo-mapping}
#
# Enable sending storage statistics as part of the firefly packets
#
# If enabled, end markers carry a "storage-stats" section with the bytes
# transferred, the duration, disk read and write rates and active/idle
# times, the transfer's return code, the file size, whether the pool read
# or wrote the file, the time the request waited in the mover queue and the
# pool name.
#
pool.firefly.storage-statistics=false

#
Expand Down
Loading