Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 35 additions & 5 deletions docs/TheBook/src/main/markdown/config-scitags.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,10 +252,15 @@ Current invocation is from:

- `modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java`

`NettyTransferService.executeMover(...)` calls `transferLifeCycle.onStart(...)` with:
When a client connects to the mover, `NettyTransferService` calls
`transferLifeCycle.onStart(...)` with:

- source = remote endpoint from `IpProtocolInfo.getSocketAddress()`
- destination = local pool endpoint chosen for mover
- the remote endpoint of that connection,
- the pool's endpoint of that connection,
- the direction of the data: inbound if the mover writes the file, outbound if it reads it.

`TransferLifeCycle` turns these into the source and destination of the flow (see
"Source and destination" below).

### End marker invocation path

Expand All @@ -265,8 +270,9 @@ Current invocation is from:

`DefaultPostTransferService.sendFinished(...)`:

- uses `((IpProtocolInfo) mover.getProtocolInfo()).getSocketAddress()` as source,
- uses `mover.getLocalEndpoint()` as destination when available,
- uses the mover's remote endpoint, or else
`((IpProtocolInfo) mover.getProtocolInfo()).getSocketAddress()`, as the remote endpoint,
- uses `mover.getLocalEndpoint()` as the local endpoint when available,
- otherwise tries `deriveLocalEndpoint(remoteEndpoint)` via `NetworkUtils.getLocalAddress(...)`,
- calls `transferLifeCycle.onEnd(...)` when either direct or derived local endpoint is available,
- logs structured skip reasons when end-marker invocation cannot proceed.
Expand Down Expand Up @@ -353,12 +359,36 @@ Marker wire format:
Main JSON sections:

- `version`
- `dcache-version`, the dCache release that sent the firefly
- `flow-lifecycle` with `state`, `start-time`, optional `end-time`, `current-time`
- `flow-id` with AFI, IPs, ports, and protocol
- `context` with `experiment-id`, `activity-id`, and application/protocol name
- `usage` (`received`, `sent`)
- optional `storage-stats` when `pool.firefly.storage-statistics=true`

### Source and destination

The source of the flow is the supplier of the data, as in XRootD's fireflies:

| Transfer | `src` | `dst` |
|----------|-------|-------|
| Upload, third-party pull | remote party | dCache node (pool or door) |
| Download, third-party push | dCache node (pool or door) | remote party |

Both ends of a transfer between two dCache instances therefore report the same `flow-id`.

`usage` is counted from the side of the source: `sent` is the number of bytes the source sent
to the destination, that is the data of the transfer, and `received` is the number of bytes it
received from the destination.

Independently of the direction, the firefly is sent to the remote party, the syslog header
carries the host name of the dCache node, and the excludes apply when both the remote and
the local address are inside the configured subnets.

Earlier dCache versions always reported the remote party as `src` and the dCache node as
`dst`, and counted `usage` from the side of the dCache node. `dcache-version` is absent from
their fireflies, which lets a consumer tell the two conventions apart.

For end markers sent by a pool, `end-time` is the time the transfer finished and
`start-time` is derived as `end-time - max(connectionTime, 0)`.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@
import org.dcache.missingfiles.MissingFileStrategy;
import org.dcache.namespace.FileAttribute;
import org.dcache.pool.movers.TransferLifeCycle;
import org.dcache.pool.movers.TransferLifeCycle.DataDirection;
import org.dcache.poolmanager.PoolManagerStub;
import org.dcache.poolmanager.PoolMonitor;
import org.dcache.util.Args;
Expand Down Expand Up @@ -1906,6 +1907,7 @@ private class HttpTransfer extends RedirectedTransfer<String> {
private final InetSocketAddress _doorEndpoint;
private final boolean _isRequestSecure;
private Instant _clientFlowStartedAt;
private DataDirection _clientFlowDirection;
protected HttpProtocolInfo.Disposition _disposition;
private boolean _isSSL;

Expand Down Expand Up @@ -1955,13 +1957,16 @@ private static InetSocketAddress localEndpoint(HttpServletRequest request) {
/**
* Send the firefly start marker for data this door is about to relay to or from the
* client that made the request. Does nothing if the data is relayed for a third party.
*
* @param direction {@code OUTBOUND} for a download, {@code INBOUND} for an upload
*/
protected void clientFlowStarted() {
protected void clientFlowStarted(DataDirection direction) {
if (_originAddress != null || _doorEndpoint == null) {
return;
}
_clientFlowStartedAt = Instant.now();
_transferLifeCycle.onStart(getClientAddress(), _doorEndpoint,
_clientFlowDirection = direction;
_transferLifeCycle.onStart(getClientAddress(), _doorEndpoint, direction,
createProtocolInfo(getClientAddress(), _isRequestSecure), _subject);
}

Expand All @@ -1977,7 +1982,7 @@ protected void clientFlowEnded(long sent, long received) {
return;
}
_clientFlowStartedAt = null;
_transferLifeCycle.onEnd(getClientAddress(), _doorEndpoint,
_transferLifeCycle.onEnd(getClientAddress(), _doorEndpoint, _clientFlowDirection,
createProtocolInfo(getClientAddress(), _isRequestSecure), _subject,
startedAt, Duration.between(startedAt, Instant.now()).toMillis(),
sent, received);
Expand Down Expand Up @@ -2153,7 +2158,7 @@ public void relayData(OutputStream outputStream, io.milton.http.Range range)
setStatus("Mover " + getPool() + "/" + getMoverId() +
": Sending data");
CountingOutputStream counting = new CountingOutputStream(outputStream);
clientFlowStarted();
clientFlowStarted(DataDirection.OUTBOUND);
try {
inputStream.transferTo(counting);
counting.flush();
Expand Down Expand Up @@ -2322,7 +2327,7 @@ public void relayData(InputStream inputStream)
setStatus("Mover " + getPool() + "/" + getMoverId() +
": Receiving data");
CountingInputStream counting = new CountingInputStream(inputStream);
clientFlowStarted();
clientFlowStarted(DataDirection.INBOUND);
try {
counting.transferTo(outputStream);
outputStream.flush();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1581,7 +1581,17 @@ private void flowConnected() {
_flowLocal = local.get();
_flowStartedAt = Instant.now();
_flowStartBytes = _relayedBytes.get();
_transferLifeCycle.onStart(_flowRemote, _flowLocal, _proxyInfo, _subject);
_transferLifeCycle.onStart(_flowRemote, _flowLocal, flowDirection(), _proxyInfo,
_subject);
}

/**
* A pull brings the data from the remote server to this door.
*/
private TransferLifeCycle.DataDirection flowDirection() {
return _direction == Direction.PULL
? TransferLifeCycle.DataDirection.INBOUND
: TransferLifeCycle.DataDirection.OUTBOUND;
}

/**
Expand All @@ -1596,7 +1606,8 @@ private void flowEnded() {

long bytes = _relayedBytes.get() - _flowStartBytes;
boolean isPull = _direction == Direction.PULL;
_transferLifeCycle.onEnd(_flowRemote, _flowLocal, _proxyInfo, _subject, startedAt,
_transferLifeCycle.onEnd(_flowRemote, _flowLocal, flowDirection(), _proxyInfo,
_subject, startedAt,
Duration.between(startedAt, Instant.now()).toMillis(),
isPull ? 0 : bytes, isPull ? bytes : 0);
}
Expand Down
22 changes: 17 additions & 5 deletions modules/dcache/src/main/java/org/dcache/net/FlowMarker.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.time.Instant;
import java.time.format.DateTimeFormatter;

import org.dcache.util.Version;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -40,6 +41,12 @@ public static class FlowMarkerBuilder {

private final static int VERSION = 1;

/**
* The dCache release that sends the firefly, so that a consumer can tell which
* conventions the firefly follows.
*/
private final static String DCACHE_VERSION = Version.of(FlowMarker.class).getVersion();

private final JSONObject payload = new JSONObject();
private final JSONObject lifecycle = new JSONObject();
private final JSONObject context = new JSONObject();
Expand Down Expand Up @@ -130,11 +137,15 @@ public FlowMarkerBuilder withProtocol(String proto) {
return this;
}

public FlowMarkerBuilder withUsage(double bytesRead, double bytesWritten) {
// approx. - assumes bytesWritten to disk are bytes "received" over the network
// and bytesRead from disk are bytes "sent" to the network
usage.put("received", Double.isFinite(bytesWritten) ? (long) bytesWritten : 0L);
usage.put("sent", Double.isFinite(bytesRead) ? (long) bytesRead : 0L);
/**
* The bytes that crossed the flow, counted from the side of its source.
*
* @param sent bytes the source sent to the destination: the data of the transfer
* @param received bytes the source received from the destination
*/
public FlowMarkerBuilder withUsage(double sent, double received) {
usage.put("received", Double.isFinite(received) ? (long) received : 0L);
usage.put("sent", Double.isFinite(sent) ? (long) sent : 0L);
return this;
}

Expand Down Expand Up @@ -167,6 +178,7 @@ public String build(String state) {
}

payload.put("version", VERSION);
payload.put("dcache-version", DCACHE_VERSION);
payload.put("flow-lifecycle", lifecycle);
payload.put("context", context);
payload.put("flow-id", flow);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.CompletionHandler;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
Expand All @@ -68,6 +69,7 @@
import org.dcache.pool.classic.Cancellable;
import org.dcache.pool.classic.PostTransferService;
import org.dcache.pool.classic.TransferService;
import org.dcache.pool.movers.TransferLifeCycle.DataDirection;
import org.dcache.pool.repository.ReplicaDescriptor;
import org.dcache.util.CDCThreadFactory;
import org.dcache.util.ChannelCdcSessionHandlerWrapper;
Expand Down Expand Up @@ -508,7 +510,10 @@ public void connected(SocketAddress remote, SocketAddress local) {
mover.setRemoteEndpoint(remoteEndpoint);
if (transferLifeCycle != null) {
try (CDC ignored = cdc.restore()) {
// a mover that writes the file receives the data
transferLifeCycle.onStart(remoteEndpoint, localEndpoint,
DataDirection.ofWrite(
mover.getIoMode().contains(StandardOpenOption.WRITE)),
mover.getProtocolInfo(), mover.getSubject());
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import javax.security.auth.Subject;
import org.apache.http.impl.client.CloseableHttpClient;
import org.dcache.http.tpc.RemoteHttpTransferClient;
import org.dcache.pool.movers.TransferLifeCycle.DataDirection;
import org.dcache.pool.repository.RepositoryChannel;
import org.dcache.util.Checksum;
import org.dcache.vehicles.FileAttributes;
Expand Down Expand Up @@ -102,6 +103,7 @@ public class RemoteHttpDataTransferProtocol implements MoverProtocol,
private final TransferLifeCycle _transferLifeCycle;
private Subject _subject;
private boolean _startMarkerSent;
private volatile DataDirection _direction;

public RemoteHttpDataTransferProtocol(CloseableHttpClient client,
TransferLifeCycle transferLifeCycle) {
Expand Down Expand Up @@ -146,6 +148,9 @@ public void runIO(FileAttributes attributes, RepositoryChannel channel,
});
});

// a pull writes the file: the remote server supplies the data
_direction = DataDirection.ofWrite(access.contains(StandardOpenOption.WRITE));

try {
if (access.contains(StandardOpenOption.WRITE)) {
receiveFile();
Expand Down Expand Up @@ -253,7 +258,7 @@ private void startFlowMarker() {
return;
}

_transferLifeCycle.onStart(remoteEndpoint, _localEndpoint,
_transferLifeCycle.onStart(remoteEndpoint, _localEndpoint, _direction,
protocolInfo, _subject);
_startMarkerSent = true;
}
Expand Down
Loading
Loading