diff --git a/docs/TheBook/src/main/markdown/config-scitags.md b/docs/TheBook/src/main/markdown/config-scitags.md index ac06d263f40..c13dbdc061c 100644 --- a/docs/TheBook/src/main/markdown/config-scitags.md +++ b/docs/TheBook/src/main/markdown/config-scitags.md @@ -252,10 +252,15 @@ Current invocation is from: - `modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java` -`NettyTransferService.executeMover(...)` calls `transferLifeCycle.onStart(...)` with: +When a client connects to the mover, `NettyTransferService` calls +`transferLifeCycle.onStart(...)` with: -- source = remote endpoint from `IpProtocolInfo.getSocketAddress()` -- destination = local pool endpoint chosen for mover +- the remote endpoint of that connection, +- the pool's endpoint of that connection, +- the direction of the data: inbound if the mover writes the file, outbound if it reads it. + +`TransferLifeCycle` turns these into the source and destination of the flow (see +"Source and destination" below). ### End marker invocation path @@ -265,8 +270,9 @@ Current invocation is from: `DefaultPostTransferService.sendFinished(...)`: -- uses `((IpProtocolInfo) mover.getProtocolInfo()).getSocketAddress()` as source, -- uses `mover.getLocalEndpoint()` as destination when available, +- uses the mover's remote endpoint, or else + `((IpProtocolInfo) mover.getProtocolInfo()).getSocketAddress()`, as the remote endpoint, +- uses `mover.getLocalEndpoint()` as the local endpoint when available, - otherwise tries `deriveLocalEndpoint(remoteEndpoint)` via `NetworkUtils.getLocalAddress(...)`, - calls `transferLifeCycle.onEnd(...)` when either direct or derived local endpoint is available, - logs structured skip reasons when end-marker invocation cannot proceed. @@ -353,12 +359,36 @@ Marker wire format: Main JSON sections: - `version` +- `dcache-version`, the dCache release that sent the firefly - `flow-lifecycle` with `state`, `start-time`, optional `end-time`, `current-time` - `flow-id` with AFI, IPs, ports, and protocol - `context` with `experiment-id`, `activity-id`, and application/protocol name - `usage` (`received`, `sent`) - optional `storage-stats` when `pool.firefly.storage-statistics=true` +### Source and destination + +The source of the flow is the supplier of the data, as in XRootD's fireflies: + +| Transfer | `src` | `dst` | +|----------|-------|-------| +| Upload, third-party pull | remote party | dCache node (pool or door) | +| Download, third-party push | dCache node (pool or door) | remote party | + +Both ends of a transfer between two dCache instances therefore report the same `flow-id`. + +`usage` is counted from the side of the source: `sent` is the number of bytes the source sent +to the destination, that is the data of the transfer, and `received` is the number of bytes it +received from the destination. + +Independently of the direction, the firefly is sent to the remote party, the syslog header +carries the host name of the dCache node, and the excludes apply when both the remote and +the local address are inside the configured subnets. + +Earlier dCache versions always reported the remote party as `src` and the dCache node as +`dst`, and counted `usage` from the side of the dCache node. `dcache-version` is absent from +their fireflies, which lets a consumer tell the two conventions apart. + For end markers sent by a pool, `end-time` is the time the transfer finished and `start-time` is derived as `end-time - max(connectionTime, 0)`. diff --git a/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java b/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java index d64a726e2ac..3660893beb4 100644 --- a/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java +++ b/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java @@ -140,6 +140,7 @@ import org.dcache.missingfiles.MissingFileStrategy; import org.dcache.namespace.FileAttribute; import org.dcache.pool.movers.TransferLifeCycle; +import org.dcache.pool.movers.TransferLifeCycle.DataDirection; import org.dcache.poolmanager.PoolManagerStub; import org.dcache.poolmanager.PoolMonitor; import org.dcache.util.Args; @@ -1906,6 +1907,7 @@ private class HttpTransfer extends RedirectedTransfer { private final InetSocketAddress _doorEndpoint; private final boolean _isRequestSecure; private Instant _clientFlowStartedAt; + private DataDirection _clientFlowDirection; protected HttpProtocolInfo.Disposition _disposition; private boolean _isSSL; @@ -1955,13 +1957,16 @@ private static InetSocketAddress localEndpoint(HttpServletRequest request) { /** * Send the firefly start marker for data this door is about to relay to or from the * client that made the request. Does nothing if the data is relayed for a third party. + * + * @param direction {@code OUTBOUND} for a download, {@code INBOUND} for an upload */ - protected void clientFlowStarted() { + protected void clientFlowStarted(DataDirection direction) { if (_originAddress != null || _doorEndpoint == null) { return; } _clientFlowStartedAt = Instant.now(); - _transferLifeCycle.onStart(getClientAddress(), _doorEndpoint, + _clientFlowDirection = direction; + _transferLifeCycle.onStart(getClientAddress(), _doorEndpoint, direction, createProtocolInfo(getClientAddress(), _isRequestSecure), _subject); } @@ -1977,7 +1982,7 @@ protected void clientFlowEnded(long sent, long received) { return; } _clientFlowStartedAt = null; - _transferLifeCycle.onEnd(getClientAddress(), _doorEndpoint, + _transferLifeCycle.onEnd(getClientAddress(), _doorEndpoint, _clientFlowDirection, createProtocolInfo(getClientAddress(), _isRequestSecure), _subject, startedAt, Duration.between(startedAt, Instant.now()).toMillis(), sent, received); @@ -2153,7 +2158,7 @@ public void relayData(OutputStream outputStream, io.milton.http.Range range) setStatus("Mover " + getPool() + "/" + getMoverId() + ": Sending data"); CountingOutputStream counting = new CountingOutputStream(outputStream); - clientFlowStarted(); + clientFlowStarted(DataDirection.OUTBOUND); try { inputStream.transferTo(counting); counting.flush(); @@ -2322,7 +2327,7 @@ public void relayData(InputStream inputStream) setStatus("Mover " + getPool() + "/" + getMoverId() + ": Receiving data"); CountingInputStream counting = new CountingInputStream(inputStream); - clientFlowStarted(); + clientFlowStarted(DataDirection.INBOUND); try { counting.transferTo(outputStream); outputStream.flush(); diff --git a/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java b/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java index c137a863e66..56d1522b104 100644 --- a/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java +++ b/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java @@ -1581,7 +1581,17 @@ private void flowConnected() { _flowLocal = local.get(); _flowStartedAt = Instant.now(); _flowStartBytes = _relayedBytes.get(); - _transferLifeCycle.onStart(_flowRemote, _flowLocal, _proxyInfo, _subject); + _transferLifeCycle.onStart(_flowRemote, _flowLocal, flowDirection(), _proxyInfo, + _subject); + } + + /** + * A pull brings the data from the remote server to this door. + */ + private TransferLifeCycle.DataDirection flowDirection() { + return _direction == Direction.PULL + ? TransferLifeCycle.DataDirection.INBOUND + : TransferLifeCycle.DataDirection.OUTBOUND; } /** @@ -1596,7 +1606,8 @@ private void flowEnded() { long bytes = _relayedBytes.get() - _flowStartBytes; boolean isPull = _direction == Direction.PULL; - _transferLifeCycle.onEnd(_flowRemote, _flowLocal, _proxyInfo, _subject, startedAt, + _transferLifeCycle.onEnd(_flowRemote, _flowLocal, flowDirection(), _proxyInfo, + _subject, startedAt, Duration.between(startedAt, Instant.now()).toMillis(), isPull ? 0 : bytes, isPull ? bytes : 0); } diff --git a/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java b/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java index a7a787d4e55..de502aee156 100644 --- a/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java +++ b/modules/dcache/src/main/java/org/dcache/net/FlowMarker.java @@ -25,6 +25,7 @@ import java.time.Instant; import java.time.format.DateTimeFormatter; +import org.dcache.util.Version; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +41,12 @@ public static class FlowMarkerBuilder { private final static int VERSION = 1; + /** + * The dCache release that sends the firefly, so that a consumer can tell which + * conventions the firefly follows. + */ + private final static String DCACHE_VERSION = Version.of(FlowMarker.class).getVersion(); + private final JSONObject payload = new JSONObject(); private final JSONObject lifecycle = new JSONObject(); private final JSONObject context = new JSONObject(); @@ -130,11 +137,15 @@ public FlowMarkerBuilder withProtocol(String proto) { return this; } - public FlowMarkerBuilder withUsage(double bytesRead, double bytesWritten) { - // approx. - assumes bytesWritten to disk are bytes "received" over the network - // and bytesRead from disk are bytes "sent" to the network - usage.put("received", Double.isFinite(bytesWritten) ? (long) bytesWritten : 0L); - usage.put("sent", Double.isFinite(bytesRead) ? (long) bytesRead : 0L); + /** + * The bytes that crossed the flow, counted from the side of its source. + * + * @param sent bytes the source sent to the destination: the data of the transfer + * @param received bytes the source received from the destination + */ + public FlowMarkerBuilder withUsage(double sent, double received) { + usage.put("received", Double.isFinite(received) ? (long) received : 0L); + usage.put("sent", Double.isFinite(sent) ? (long) sent : 0L); return this; } @@ -167,6 +178,7 @@ public String build(String state) { } payload.put("version", VERSION); + payload.put("dcache-version", DCACHE_VERSION); payload.put("flow-lifecycle", lifecycle); payload.put("context", context); payload.put("flow-id", flow); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java index 73350f2815d..08354a865d5 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java @@ -53,6 +53,7 @@ import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.channels.CompletionHandler; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -68,6 +69,7 @@ import org.dcache.pool.classic.Cancellable; import org.dcache.pool.classic.PostTransferService; import org.dcache.pool.classic.TransferService; +import org.dcache.pool.movers.TransferLifeCycle.DataDirection; import org.dcache.pool.repository.ReplicaDescriptor; import org.dcache.util.CDCThreadFactory; import org.dcache.util.ChannelCdcSessionHandlerWrapper; @@ -508,7 +510,10 @@ public void connected(SocketAddress remote, SocketAddress local) { mover.setRemoteEndpoint(remoteEndpoint); if (transferLifeCycle != null) { try (CDC ignored = cdc.restore()) { + // a mover that writes the file receives the data transferLifeCycle.onStart(remoteEndpoint, localEndpoint, + DataDirection.ofWrite( + mover.getIoMode().contains(StandardOpenOption.WRITE)), mover.getProtocolInfo(), mover.getSubject()); } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java index 2bb3ce1daba..126fcc3528a 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java @@ -23,6 +23,7 @@ import javax.security.auth.Subject; import org.apache.http.impl.client.CloseableHttpClient; import org.dcache.http.tpc.RemoteHttpTransferClient; +import org.dcache.pool.movers.TransferLifeCycle.DataDirection; import org.dcache.pool.repository.RepositoryChannel; import org.dcache.util.Checksum; import org.dcache.vehicles.FileAttributes; @@ -102,6 +103,7 @@ public class RemoteHttpDataTransferProtocol implements MoverProtocol, private final TransferLifeCycle _transferLifeCycle; private Subject _subject; private boolean _startMarkerSent; + private volatile DataDirection _direction; public RemoteHttpDataTransferProtocol(CloseableHttpClient client, TransferLifeCycle transferLifeCycle) { @@ -146,6 +148,9 @@ public void runIO(FileAttributes attributes, RepositoryChannel channel, }); }); + // a pull writes the file: the remote server supplies the data + _direction = DataDirection.ofWrite(access.contains(StandardOpenOption.WRITE)); + try { if (access.contains(StandardOpenOption.WRITE)) { receiveFile(); @@ -253,7 +258,7 @@ private void startFlowMarker() { return; } - _transferLifeCycle.onStart(remoteEndpoint, _localEndpoint, + _transferLifeCycle.onStart(remoteEndpoint, _localEndpoint, _direction, protocolInfo, _subject); _startMarkerSent = true; } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java b/modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java index 353f96b19da..144adf69fca 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java @@ -71,7 +71,7 @@ public class TransferLifeCycle { */ private final static int UDP_PORT = 10514; - // by default, send the firefly to the destination host. + // by default, send the firefly to the remote party of the flow. private Function toFireflyDestination = a -> new InetSocketAddress( a.getAddress(), UDP_PORT); @@ -81,7 +81,7 @@ public class TransferLifeCycle { // optional additional collector destination for firefly markers private InetSocketAddress fireflyCollector; - // syslog HOSTNAME of emitted fireflies: the FQDN of this pool node. Resolved once; + // syslog HOSTNAME of emitted fireflies: the FQDN of this node. Resolved once; // it must not depend on the flow's src/dst, as either may be the remote party. private final String hostname = NetworkUtils.getCanonicalHostName(); private boolean enabled; @@ -89,73 +89,105 @@ public class TransferLifeCycle { private Map voToExpId = new HashMap<>(); + /** + * Which way the data of a transfer moves, seen from this node. A firefly names the supplier + * of the data as the source of the flow, so the direction decides which end of the + * connection is reported as source and which as destination. + */ + public enum DataDirection { + /** + * The remote party supplies the data: an upload, or a third-party pull. + */ + INBOUND, + + /** + * This node supplies the data: a download, or a third-party push. + */ + OUTBOUND; + + /** + * The direction of a transfer in which this node writes, or reads, the file. + */ + public static DataDirection ofWrite(boolean isWrite) { + return isWrite ? INBOUND : OUTBOUND; + } + } + /** * Mark transfer start. - * @param src remote client endpoint - * @param dst local endpoint + * @param remote the remote party's endpoint of the connection that carries the data + * @param local this node's endpoint of that connection + * @param direction which way the data moves * @param protocolInfo access protocol information * @param subject associated with the transfer */ - public void onStart(InetSocketAddress src, InetSocketAddress dst, ProtocolInfo protocolInfo, - Subject subject) { + public void onStart(InetSocketAddress remote, InetSocketAddress local, + DataDirection direction, ProtocolInfo protocolInfo, Subject subject) { + + // the supplier of the data is the source of the flow + InetSocketAddress src = direction == DataDirection.INBOUND ? remote : local; + InetSocketAddress dst = direction == DataDirection.INBOUND ? local : remote; if (!enabled) { - logSkippedMarker("disabled", src, dst, protocolInfo); + logSkippedMarker("disabled", src, dst, direction, protocolInfo); return; } if (isRelayed(protocolInfo)) { - logSkippedMarker("relayed-by-door", src, dst, protocolInfo); + logSkippedMarker("relayed-by-door", src, dst, direction, protocolInfo); return; } - if (isExcludedTransfer(src, dst)) { - logSkippedMarker("excluded", src, dst, protocolInfo); + if (isExcludedTransfer(remote, local)) { + logSkippedMarker("excluded", src, dst, direction, protocolInfo); return; } if (!needMarker(protocolInfo)) { - logSkippedMarker("unsupported-protocol", src, dst, protocolInfo); + logSkippedMarker("unsupported-protocol", src, dst, direction, protocolInfo); return; } - var optionalExpId = getExperimentId(protocolInfo, subject); - if (optionalExpId.isEmpty()) { - logSkippedMarker("no-experiment-id", src, dst, protocolInfo); + var optionalExpId = getExperimentId(protocolInfo, subject); + if (optionalExpId.isEmpty()) { + logSkippedMarker("no-experiment-id", src, dst, direction, protocolInfo); return; - } + } - int activityId = getActivity(protocolInfo); + int activityId = getActivity(protocolInfo); - var data = new FlowMarkerBuilder() + var data = new FlowMarkerBuilder() .withStartedAt(Instant.now()) .withExperimentId(optionalExpId.getAsInt()) .withActivityId(activityId) .wittApplication(getApplication(protocolInfo)) .withProtocol("tcp") .withHostname(hostname) - .withAFI(toAFI(dst)) + .withAFI(toAFI(local)) .withDestination(dst) .withSource(src) .build("start"); - InetSocketAddress fireflyDestination = toFireflyDestination.apply(src); - sendToMultipleDestinations(fireflyDestination, data); - logMarkerEvent("start", src, dst, protocolInfo, optionalExpId.getAsInt(), activityId, - null, null, fireflyDestination); + // the firefly goes to the remote party, whichever end of the flow that is + InetSocketAddress fireflyDestination = toFireflyDestination.apply(remote); + sendToMultipleDestinations(fireflyDestination, data); + logMarkerEvent("start", src, dst, direction, protocolInfo, optionalExpId.getAsInt(), + activityId, null, null, fireflyDestination); } /** - * Mark transfer end. - * @param src remote client endpoint - * @param dst local endpoint + * Mark transfer end. The direction is taken from the mover: a mover that created the file + * received the data. + * @param remote the remote party's endpoint of the connection that carried the data + * @param local this node's endpoint of that connection * @param mover the mover's description of the completed transfer */ - public void onEnd(InetSocketAddress src, InetSocketAddress dst, MoverInfoMessage mover) { + public void onEnd(InetSocketAddress remote, InetSocketAddress local, MoverInfoMessage mover) { // The mover info message is created when the transfer has finished, so its // timestamp is the end of the transfer; the start is derived from the duration. long durationMillis = Math.max(0L, mover.getConnectionTime()); - onEnd(src, dst, mover.getProtocolInfo(), mover.getSubject(), + onEnd(remote, local, DataDirection.ofWrite(mover.isFileCreated()), + mover.getProtocolInfo(), mover.getSubject(), Instant.ofEpochMilli(mover.getTimestamp()).minusMillis(durationMillis), durationMillis, mover.getBytesRead(), mover.getBytesWritten(), @@ -164,8 +196,9 @@ public void onEnd(InetSocketAddress src, InetSocketAddress dst, MoverInfoMessage /** * Mark transfer end, for a transfer that has no mover. - * @param src remote client endpoint - * @param dst local endpoint + * @param remote the remote party's endpoint of the connection that carried the data + * @param local this node's endpoint of that connection + * @param direction which way the data moved * @param protocolInfo access protocol information * @param subject associated with the transfer * @param startedAt when the transfer started @@ -173,56 +206,68 @@ public void onEnd(InetSocketAddress src, InetSocketAddress dst, MoverInfoMessage * @param bytesRead number of bytes sent to the remote endpoint * @param bytesWritten number of bytes received from the remote endpoint */ - public void onEnd(InetSocketAddress src, InetSocketAddress dst, ProtocolInfo protocolInfo, + public void onEnd(InetSocketAddress remote, InetSocketAddress local, + DataDirection direction, ProtocolInfo protocolInfo, Subject subject, Instant startedAt, long durationMillis, long bytesRead, long bytesWritten) { - onEnd(src, dst, protocolInfo, subject, startedAt, durationMillis, bytesRead, - bytesWritten, null); + onEnd(remote, local, direction, protocolInfo, subject, startedAt, durationMillis, + bytesRead, bytesWritten, null); } - private void onEnd(InetSocketAddress src, InetSocketAddress dst, ProtocolInfo protocolInfo, + private void onEnd(InetSocketAddress remote, InetSocketAddress local, + DataDirection direction, ProtocolInfo protocolInfo, Subject subject, Instant startedAt, long durationMillis, double bytesRead, double bytesWritten, @Nullable MoverInfoMessage storageStatistics) { + // the supplier of the data is the source of the flow + boolean isInbound = direction == DataDirection.INBOUND; + InetSocketAddress src = isInbound ? remote : local; + InetSocketAddress dst = isInbound ? local : remote; + if (!enabled) { - logSkippedMarker("disabled", src, dst, protocolInfo); + logSkippedMarker("disabled", src, dst, direction, protocolInfo); return; } if (isRelayed(protocolInfo)) { - logSkippedMarker("relayed-by-door", src, dst, protocolInfo); + logSkippedMarker("relayed-by-door", src, dst, direction, protocolInfo); return; } - if (isExcludedTransfer(src, dst)) { - logSkippedMarker("excluded", src, dst, protocolInfo); + if (isExcludedTransfer(remote, local)) { + logSkippedMarker("excluded", src, dst, direction, protocolInfo); return; } if (!needMarker(protocolInfo)) { - logSkippedMarker("unsupported-protocol", src, dst, protocolInfo); + logSkippedMarker("unsupported-protocol", src, dst, direction, protocolInfo); return; } var optionalExpId = getExperimentId(protocolInfo, subject); if (optionalExpId.isEmpty()) { - logSkippedMarker("no-experiment-id", src, dst, protocolInfo); + logSkippedMarker("no-experiment-id", src, dst, direction, protocolInfo); return; } int activityId = getActivity(protocolInfo); Instant finishedAt = startedAt.plusMillis(Math.max(0L, durationMillis)); + // Usage is counted from the side of the source, like the flow itself: what this node + // received is what a remote source sent. + double sentBySource = isInbound ? bytesWritten : bytesRead; + double receivedBySource = isInbound ? bytesRead : bytesWritten; + var data = new FlowMarkerBuilder() .withStartedAt(startedAt) .withFinishedAt(finishedAt) .withExperimentId(optionalExpId.getAsInt()) .withActivityId(activityId) .wittApplication(getApplication(protocolInfo)) - .withUsage(bytesRead, bytesWritten) + .withUsage(sentBySource, receivedBySource) .withProtocol("tcp") .withHostname(hostname) - .withAFI(toAFI(dst)) + .withAFI(toAFI(local)) .withDestination(dst) .withSource(src); if (storageStatistics != null) { @@ -230,10 +275,11 @@ private void onEnd(InetSocketAddress src, InetSocketAddress dst, ProtocolInfo pr } var firefly = data.build("end"); - InetSocketAddress fireflyDestination = toFireflyDestination.apply(src); + // the firefly goes to the remote party, whichever end of the flow that is + InetSocketAddress fireflyDestination = toFireflyDestination.apply(remote); sendToMultipleDestinations(fireflyDestination, firefly); - logMarkerEvent("end", src, dst, protocolInfo, optionalExpId.getAsInt(), activityId, - bytesRead, bytesWritten, fireflyDestination); + logMarkerEvent("end", src, dst, direction, protocolInfo, optionalExpId.getAsInt(), + activityId, bytesRead, bytesWritten, fireflyDestination); } /** @@ -402,12 +448,12 @@ private OptionalInt getExperimentId(ProtocolInfo protocolInfo, Subject subject) : OptionalInt.empty(); } - private boolean isExcludedTransfer(InetSocketAddress src, InetSocketAddress dst) { - InetAddress srcAddress = src.getAddress(); - InetAddress dstAddress = dst.getAddress(); - return srcAddress != null && dstAddress != null - && localSubnet.test(srcAddress) - && localSubnet.test(dstAddress); + private boolean isExcludedTransfer(InetSocketAddress remote, InetSocketAddress local) { + InetAddress remoteAddress = remote.getAddress(); + InetAddress localAddress = local.getAddress(); + return remoteAddress != null && localAddress != null + && localSubnet.test(remoteAddress) + && localSubnet.test(localAddress); } private int getActivity(ProtocolInfo protocolInfo) { @@ -425,34 +471,37 @@ private boolean hasTransferTag(ProtocolInfo protocolInfo) { } private void logSkippedMarker(String reason, InetSocketAddress src, InetSocketAddress dst, - ProtocolInfo protocolInfo) { + DataDirection direction, ProtocolInfo protocolInfo) { if (SCITAGS_LOGGER.isDebugEnabled()) { SCITAGS_LOGGER.debug( - "scitags event=marker-skip reason={} protocol={} source={} sourcePort={} destination={} destinationPort={} transferTag={}", + "scitags event=marker-skip reason={} protocol={} source={} sourcePort={} destination={} destinationPort={} direction={} transferTag={}", reason, protocolInfo.getProtocol().toLowerCase(), formatAddress(src), formatPort(src), formatAddress(dst), formatPort(dst), + direction.name().toLowerCase(), formatTransferTag(protocolInfo)); } } private void logMarkerEvent(String event, InetSocketAddress src, InetSocketAddress dst, + DataDirection direction, ProtocolInfo protocolInfo, int experimentId, int activityId, Double bytesRead, Double bytesWritten, InetSocketAddress fireflyDestination) { if (SCITAGS_LOGGER.isDebugEnabled()) { String classification = hasTransferTag(protocolInfo) ? "transfer-tag" : "fqan"; if (bytesRead == null || bytesWritten == null) { SCITAGS_LOGGER.debug( - "scitags event=marker state={} protocol={} source={} sourcePort={} destination={} destinationPort={} transferTag={} experimentId={} activityId={} classifier={} fireflyDestination={} collector={}", + "scitags event=marker state={} protocol={} source={} sourcePort={} destination={} destinationPort={} direction={} transferTag={} experimentId={} activityId={} classifier={} fireflyDestination={} collector={}", event, protocolInfo.getProtocol().toLowerCase(), formatAddress(src), formatPort(src), formatAddress(dst), formatPort(dst), + direction.name().toLowerCase(), formatTransferTag(protocolInfo), experimentId, activityId, @@ -461,13 +510,14 @@ private void logMarkerEvent(String event, InetSocketAddress src, InetSocketAddre formatCollectorDestination()); } else { SCITAGS_LOGGER.debug( - "scitags event=marker state={} protocol={} source={} sourcePort={} destination={} destinationPort={} transferTag={} experimentId={} activityId={} classifier={} bytesRead={} bytesWritten={} fireflyDestination={} collector={}", + "scitags event=marker state={} protocol={} source={} sourcePort={} destination={} destinationPort={} direction={} transferTag={} experimentId={} activityId={} classifier={} bytesRead={} bytesWritten={} fireflyDestination={} collector={}", event, protocolInfo.getProtocol().toLowerCase(), formatAddress(src), formatPort(src), formatAddress(dst), formatPort(dst), + direction.name().toLowerCase(), formatTransferTag(protocolInfo), experimentId, activityId, diff --git a/modules/dcache/src/test/java/org/dcache/net/FlowMarkerTest.java b/modules/dcache/src/test/java/org/dcache/net/FlowMarkerTest.java index 01f3a00c83d..400000f80ea 100644 --- a/modules/dcache/src/test/java/org/dcache/net/FlowMarkerTest.java +++ b/modules/dcache/src/test/java/org/dcache/net/FlowMarkerTest.java @@ -1,5 +1,6 @@ package org.dcache.net; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -10,6 +11,7 @@ import java.time.Instant; import java.util.stream.Collectors; import org.dcache.net.FlowMarker.FlowMarkerBuilder; +import org.dcache.util.Version; import org.everit.json.schema.Schema; import org.everit.json.schema.ValidationException; import org.everit.json.schema.loader.SchemaLoader; @@ -105,6 +107,24 @@ public void shouldAcceptFullyBuildOngoingMarker() { assertFirefly(data); } + @Test + public void shouldCarryDcacheVersion() { + var data = new FlowMarkerBuilder() + .withStartedAt(Instant.now()) + .withExperimentId(2) + .withActivityId(1) + .wittApplication("curl") + .withProtocol("tcp") + .withAFI("ipv4") + .withDestination(new InetSocketAddress(InetAddress.getLoopbackAddress(), 1234)) + .withSource(new InetSocketAddress(InetAddress.getLoopbackAddress(), 5678)) + .build("start"); + + assertFirefly(data); + var json = new JSONObject(data.substring(data.indexOf("{"))); + assertEquals(Version.of(FlowMarker.class).getVersion(), json.getString("dcache-version")); + } + @Test(expected = AssertionError.class) public void shouldFailWithEmptyMarker() { var data = new FlowMarkerBuilder() diff --git a/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java b/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java index 3238b0279b5..c1c30b45efc 100644 --- a/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java +++ b/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java @@ -3,6 +3,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -13,10 +14,13 @@ import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.nio.channels.CompletionHandler; +import java.nio.file.StandardOpenOption; +import java.util.Set; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import javax.security.auth.Subject; +import org.dcache.pool.movers.TransferLifeCycle.DataDirection; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -83,7 +87,18 @@ public void shouldSendStartMarkerForTheConnectionTheClientMadeToThePool() { channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); verify(transferLifeCycle).onStart(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION, - protocolInfo, subject); + DataDirection.OUTBOUND, protocolInfo, subject); + } + + @Test + public void shouldReportUploadAsInbound() { + doReturn(Set.of(StandardOpenOption.READ, StandardOpenOption.WRITE)) + .when(mover).getIoMode(); + + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + + verify(transferLifeCycle).onStart(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION, + DataDirection.INBOUND, protocolInfo, subject); } @Test @@ -96,7 +111,7 @@ public void shouldRecordEndpointsOfDataConnectionOnMoverForEndMarker() { @Test public void shouldNotSendStartMarkerBeforeClientConnects() { - verify(transferLifeCycle, never()).onStart(any(), any(), any(), any()); + verify(transferLifeCycle, never()).onStart(any(), any(), any(), any(), any()); verify(mover, never()).setRemoteEndpoint(any()); } @@ -106,7 +121,7 @@ public void shouldSendOnlyOneStartMarkerWhenFurtherConnectionsAttach() { channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); channel.connected(new InetSocketAddress("203.0.113.20", 50000), POOL_DATA_CONNECTION); - verify(transferLifeCycle, times(1)).onStart(any(), any(), any(), any()); + verify(transferLifeCycle, times(1)).onStart(any(), any(), any(), any(), any()); verify(mover, times(1)).setRemoteEndpoint(CLIENT_DATA_CONNECTION); } @@ -114,7 +129,7 @@ public void shouldSendOnlyOneStartMarkerWhenFurtherConnectionsAttach() { public void shouldIgnoreConnectionsThatAreNotIpSockets() { channel.connected(new LocalAddress("client"), new LocalAddress("pool")); - verify(transferLifeCycle, never()).onStart(any(), any(), any(), any()); + verify(transferLifeCycle, never()).onStart(any(), any(), any(), any(), any()); verify(mover, never()).setRemoteEndpoint(any()); } diff --git a/modules/dcache/src/test/java/org/dcache/pool/movers/TransferLifeCycleTest.java b/modules/dcache/src/test/java/org/dcache/pool/movers/TransferLifeCycleTest.java index 81d064ae02b..abbf3054c5b 100644 --- a/modules/dcache/src/test/java/org/dcache/pool/movers/TransferLifeCycleTest.java +++ b/modules/dcache/src/test/java/org/dcache/pool/movers/TransferLifeCycleTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import diskCacheV111.util.PnfsId; import diskCacheV111.vehicles.HttpProtocolInfo; @@ -23,6 +24,7 @@ import java.util.OptionalInt; import javax.security.auth.Subject; import org.dcache.auth.FQANPrincipal; +import org.dcache.pool.movers.TransferLifeCycle.DataDirection; import org.dcache.util.NetworkUtils; import org.json.JSONObject; import org.junit.Before; @@ -110,6 +112,7 @@ public void shouldSendEndMarkerForTransferWithoutMover() throws Exception { lifecycle.onEnd( new InetSocketAddress("203.0.113.20", 443), new InetSocketAddress("198.51.100.55", 42000), + DataDirection.INBOUND, new TestProtocolInfo("remotehttpsdatatransfer", "129"), new Subject(), startedAt, 2500, 10, 2048); @@ -131,8 +134,9 @@ public void shouldSendEndMarkerForTransferWithoutMover() throws Exception { assertEquals(443, flow.getInt("src-port")); assertEquals("198.51.100.55", flow.getString("dst-ip")); assertEquals(42000, flow.getInt("dst-port")); - assertEquals(10, usage.getLong("sent")); - assertEquals(2048, usage.getLong("received")); + // usage is counted from the side of the source, here the remote server + assertEquals(2048, usage.getLong("sent")); + assertEquals(10, usage.getLong("received")); assertFalse(payload.has("storage-stats")); } } @@ -199,6 +203,7 @@ public void shouldUsePoolHostNameAsSyslogHostnameInStartMarker() throws Exceptio lifecycle.onStart( new InetSocketAddress("2001:db8:3e::92", 8443), new InetSocketAddress("2001:db8:62::62", 35048), + DataDirection.INBOUND, new TestProtocolInfo("remotehttpsdatatransfer", "129"), new Subject()); @@ -207,23 +212,105 @@ public void shouldUsePoolHostNameAsSyslogHostnameInStartMarker() throws Exceptio } @Test - public void shouldUsePoolHostNameAsSyslogHostnameWhenDataEntersPool() throws Exception { - // e.g. client upload or HTTP-TPC pull: bytes are written to the pool + public void shouldNameRemoteAsSourceWhenDataEntersPool() throws Exception { + // e.g. HTTP-TPC pull or an upload: the remote party supplies the data String payload = endMarker("remotehttpsdatatransfer", 0, 1024); assertSyslogHostnameIsPoolHost(payload); - assertEquals(1024, usage(payload).getLong("received")); - assertEquals(0, usage(payload).getLong("sent")); + assertEquals(REMOTE_IP, flowId(payload).getString("src-ip")); + assertEquals(42000, flowId(payload).getInt("src-port")); + assertEquals(LOCAL_IP, flowId(payload).getString("dst-ip")); + assertEquals(2880, flowId(payload).getInt("dst-port")); + assertEquals(1024, usage(payload).getLong("sent")); + assertEquals(0, usage(payload).getLong("received")); } @Test - public void shouldUsePoolHostNameAsSyslogHostnameWhenDataLeavesPool() throws Exception { - // e.g. client download or HTTP-TPC push: bytes are read from the pool + public void shouldNamePoolAsSourceWhenDataLeavesPool() throws Exception { + // e.g. HTTP-TPC push or a download: the pool supplies the data String payload = endMarker("remotehttpsdatatransfer", 1024, 0); assertSyslogHostnameIsPoolHost(payload); - assertEquals(0, usage(payload).getLong("received")); + assertEquals(LOCAL_IP, flowId(payload).getString("src-ip")); + assertEquals(2880, flowId(payload).getInt("src-port")); + assertEquals(REMOTE_IP, flowId(payload).getString("dst-ip")); + assertEquals(42000, flowId(payload).getInt("dst-port")); assertEquals(1024, usage(payload).getLong("sent")); + assertEquals(0, usage(payload).getLong("received")); + } + + @Test + public void shouldNameRemoteAsSourceInStartMarkerOfInboundTransfer() throws Exception { + JSONObject flow = flowId(startMarker(DataDirection.INBOUND)); + + assertEquals(REMOTE_IP, flow.getString("src-ip")); + assertEquals(42000, flow.getInt("src-port")); + assertEquals(LOCAL_IP, flow.getString("dst-ip")); + assertEquals(2880, flow.getInt("dst-port")); + } + + @Test + public void shouldNameLocalNodeAsSourceInStartMarkerOfOutboundTransfer() throws Exception { + JSONObject flow = flowId(startMarker(DataDirection.OUTBOUND)); + + assertEquals(LOCAL_IP, flow.getString("src-ip")); + assertEquals(2880, flow.getInt("src-port")); + assertEquals(REMOTE_IP, flow.getString("dst-ip")); + assertEquals(42000, flow.getInt("dst-port")); + } + + @Test + public void shouldApplyExcludesWhateverTheDirection() throws Exception { + for (DataDirection direction : DataDirection.values()) { + try (DatagramSocket socket = + new DatagramSocket(0, InetAddress.getByName("127.0.0.1"))) { + socket.setSoTimeout(300); + + TransferLifeCycle lifecycle = new TransferLifeCycle(); + lifecycle.setEnabled(true); + lifecycle.setVoMapping("atlas:2"); + lifecycle.setExcludes(new String[]{"203.0.113.0/24", "198.51.100.0/24"}); + lifecycle.setFireflyDestination("127.0.0.1:" + socket.getLocalPort()); + + lifecycle.onStart( + new InetSocketAddress(REMOTE_IP, 42000), + new InetSocketAddress(LOCAL_IP, 2880), + direction, new TestProtocolInfo("xrootd", "129"), new Subject()); + + try { + socket.receive(new DatagramPacket(new byte[4096], 4096)); + fail("marker sent for an excluded " + direction + " transfer"); + } catch (SocketTimeoutException expected) { + } + } + } + } + + @Test + public void shouldCountUsageFromTheSideOfTheSourceForOutboundTransferWithoutMover() + throws Exception { + try (DatagramSocket socket = new DatagramSocket(0, InetAddress.getByName("127.0.0.1"))) { + socket.setSoTimeout(700); + + TransferLifeCycle lifecycle = new TransferLifeCycle(); + lifecycle.setEnabled(true); + lifecycle.setFireflyDestination("127.0.0.1:" + socket.getLocalPort()); + + // e.g. a download relayed by a door: 4096 bytes sent to the client + lifecycle.onEnd( + new InetSocketAddress(REMOTE_IP, 42000), + new InetSocketAddress(LOCAL_IP, 2880), + DataDirection.OUTBOUND, + new TestProtocolInfo("https", "129"), + new Subject(), Instant.parse("2026-11-11T10:00:00Z"), 2500, 4096, 0); + + String payload = receive(socket); + + assertEquals(LOCAL_IP, flowId(payload).getString("src-ip")); + assertEquals(REMOTE_IP, flowId(payload).getString("dst-ip")); + assertEquals(4096, usage(payload).getLong("sent")); + assertEquals(0, usage(payload).getLong("received")); + } } private static final String REMOTE_IP = "203.0.113.20"; @@ -246,6 +333,8 @@ private static String endMarker(String protocol, long bytesRead, long bytesWritt mover.setSubject(new Subject()); mover.setBytesRead(bytesRead); mover.setBytesWritten(bytesWritten); + // a mover that writes the file receives the data + mover.setFileCreated(bytesWritten > 0); lifecycle.onEnd( new InetSocketAddress(REMOTE_IP, 42000), @@ -256,19 +345,36 @@ private static String endMarker(String protocol, long bytesRead, long bytesWritt } } + private static String startMarker(DataDirection direction) throws Exception { + try (DatagramSocket socket = new DatagramSocket(0, InetAddress.getByName("127.0.0.1"))) { + socket.setSoTimeout(700); + + TransferLifeCycle lifecycle = new TransferLifeCycle(); + lifecycle.setEnabled(true); + lifecycle.setVoMapping("atlas:2"); + lifecycle.setFireflyDestination("127.0.0.1:" + socket.getLocalPort()); + + lifecycle.onStart( + new InetSocketAddress(REMOTE_IP, 42000), + new InetSocketAddress(LOCAL_IP, 2880), + direction, new TestProtocolInfo("xrootd", "129"), new Subject()); + + return receive(socket); + } + } + /** * The syslog HOSTNAME must name the emitting pool node, whatever the direction of the - * data, and the flow-id must be left untouched. + * data. */ private static void assertSyslogHostnameIsPoolHost(String payload) { String hostname = syslogHostname(payload); assertEquals(NetworkUtils.getCanonicalHostName(), hostname); assertNotEquals(REMOTE_IP, hostname); + } - JSONObject flowId = new JSONObject(payload.substring(payload.indexOf('{'))) - .getJSONObject("flow-id"); - assertEquals(REMOTE_IP, flowId.getString("src-ip")); - assertEquals(LOCAL_IP, flowId.getString("dst-ip")); + private static JSONObject flowId(String payload) { + return new JSONObject(payload.substring(payload.indexOf('{'))).getJSONObject("flow-id"); } private static JSONObject usage(String payload) { @@ -321,6 +427,7 @@ private boolean sendsStartMarker(String srcIp, String dstIp, String excludes, lifecycle.onStart( new InetSocketAddress(srcIp, 40000), new InetSocketAddress(dstIp, 20066), + DataDirection.INBOUND, protocolInfo, new Subject());