From 6b55d9790ca11ec3585d519a86cb911049561659 Mon Sep 17 00:00:00 2001 From: Shawn McKee Date: Sat, 14 Nov 2026 17:38:48 -0500 Subject: [PATCH 1/2] pool: take firefly flow-id of client transfers from the data connection Motivation: A firefly ties a network flow to an experiment and an activity. The flow-id must therefore name the TCP connection that carries the data. For xrootd and http(s) transfers that a door redirects to a pool, it does not (#8307). NettyTransferService sends the start marker when the mover is created, before the client has connected to the pool, and takes the remote endpoint from the protocol info. That is the client's connection to the door. The client then connects to the pool from a different port, so the marker carries one port from each connection and matches no flow. The end marker repeats the same endpoint. On a production pool the client port was wrong in 54 of 55 transfers and right in none. Modification: The start marker is no longer sent from executeMover. NettyMoverChannel gains connected(remote, local), which the protocol handlers call with the addresses of the Netty channel once a client has attached to the mover: HttpPoolRequestHandler for GET and PUT, XrootdPoolRequestHandler when a file is opened for reading or writing. The first such connection is recorded on the mover and announced with the start marker. Mover gains getRemoteEndpoint() (as in #8306) and isPassive(). DefaultPostTransferService uses the recorded remote endpoint for the end marker, so both markers of a transfer carry the same flow-id. For a passive mover that no client connected to, no end marker is sent, as there was no flow and no start marker. Not covered: when a pool is the destination of an xrootd third-party copy, the file arrives over a connection that the pool's third-party client (TpcWriteDescriptor) opens to the source. The requesting client's connection carries no file data and is not reported, so no marker is sent for such transfers until the third-party client reports its own connection. Result: Fireflies for xrootd and http(s) client transfers name the connection that carries the data. The start marker is sent when the client has connected to the pool instead of before. Movers that no client connects to emit no markers. Testing: - New NettyTransferServiceFlowMarkerTest. - PR pipeline unit-test job run locally in maven:3.9.12-eclipse-temurin-21 (mvn -P code-coverage clean verify -DskipTests=false -Drun.slow.tests): 5178 tests, 0 failures. - Backported to 11.2.10 and run on a production pool. TCP connections to the mover ports were captured for 10 minutes and compared with the endpoints the pool logged. Before: client port right in 0 of 55 transfers. After: right in 65 of 65 (64 xrootd, 1 https). Start and end markers carried the same flow-id in 67 of 67 transfers. Fixes: #8307 Acked-by: Target: master Require-book: no Require-notes: yes Co-Authored-By: Claude Fable 5.2 Signed-off-by: Shawn McKee --- .../xrootd/pool/XrootdPoolRequestHandler.java | 13 ++ .../dcache/http/HttpPoolRequestHandler.java | 6 + .../classic/DefaultPostTransferService.java | 12 +- .../org/dcache/pool/movers/AbstractMover.java | 10 ++ .../java/org/dcache/pool/movers/Mover.java | 19 +++ .../org/dcache/pool/movers/NettyMover.java | 9 ++ .../pool/movers/NettyTransferService.java | 39 +++++- .../NettyTransferServiceFlowMarkerTest.java | 128 ++++++++++++++++++ 8 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java diff --git a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdPoolRequestHandler.java b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdPoolRequestHandler.java index 0d166a930e7..b0c443651f3 100644 --- a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdPoolRequestHandler.java +++ b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdPoolRequestHandler.java @@ -372,6 +372,17 @@ protected XrootdResponse doOnAuthentication(ChannelHandle return withOk(msg); } + /** + * Reports the client's connection as the network flow of the transfer. This is not done when + * the pool is the destination of a third-party copy: the file then arrives over a connection + * that the third-party client opens to the source, and the connection from the client that + * requested the copy carries no file data. + */ + private static void reportDataConnection(ChannelHandlerContext ctx, + NettyTransferService.NettyMoverChannel file) { + file.connected(ctx.channel().remoteAddress(), ctx.channel().localAddress()); + } + /** * Obtains the right mover channel using an opaque token in the request. The mover channel is * wrapped by a file descriptor. The file descriptor is stored for subsequent access. @@ -444,9 +455,11 @@ protected XrootdResponse doOnOpen(ChannelHandlerContext ctx, tlsSessionInfo); } else { descriptor = new WriteDescriptor(file, posc); + reportDataConnection(ctx, file); } } else { descriptor = new ReadDescriptor(file); + reportDataConnection(ctx, file); } FileStatus stat = msg.isRetStat() ? stat(file) : null; diff --git a/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java b/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java index 7b18f694e31..0612b2722fc 100644 --- a/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java +++ b/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java @@ -435,6 +435,9 @@ protected ChannelFuture doOnGet(ChannelHandlerContext context, "Resource is not open for reading"); } + // This connection carries the file: report it as the transfer's network flow. + file.connected(context.channel().remoteAddress(), context.channel().localAddress()); + fileSize = file.size(); ranges = parseHttpRange(request, 0, fileSize - 1); } catch (Redirect e) { @@ -547,6 +550,9 @@ protected ChannelFuture doOnPut(ChannelHandlerContext context, HttpRequest reque "Resource is not open for writing"); } + // This connection carries the file: report it as the transfer's network flow. + file.connected(context.channel().remoteAddress(), context.channel().localAddress()); + contentMd5Checksum(request).ifPresent(file::addChecksum); OptionalLong contentLength = contentLength(request); diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java index 7b0c2655f31..07257bd591b 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java @@ -237,8 +237,18 @@ public void sendFinished(Mover mover, MoverInfoMessage moverInfoMessage) { protocolName(mover), mover.getFileAttributes().getPnfsId(), transferTag(mover)); + } else if (mover.isPassive() && mover.getRemoteEndpoint().isEmpty()) { + // Nobody connected to the mover: there was no flow and no start marker was sent. + SCITAGS_LOGGER.debug( + "scitags lifecycle=end skip reason=no-data-connection protocol={} pnfsid={} transferTag={}", + protocolName(mover), + mover.getFileAttributes().getPnfsId(), + transferTag(mover)); } else { - InetSocketAddress remoteEndpoint = ipProtocolInfo.getSocketAddress(); + // Prefer the endpoint of the connection the mover made, as reported in the start + // marker; the address in the protocol info is the one the door resolved. + InetSocketAddress remoteEndpoint = mover.getRemoteEndpoint() + .orElseGet(ipProtocolInfo::getSocketAddress); if (remoteEndpoint == null) { SCITAGS_LOGGER.debug( "scitags lifecycle=end skip reason=no-remote-endpoint protocol={} pnfsid={} transferTag={}", diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java index 65ad1ab7ab9..b69727bc4d5 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java @@ -85,6 +85,7 @@ public abstract class AbstractMover

_channel = Optional.empty(); private volatile InetSocketAddress _localEndpoint; + private volatile InetSocketAddress _remoteEndpoint; public AbstractMover(ReplicaDescriptor handle, PoolIoFileMessage message, CellPath pathToDoor, TransferService transferService) { @@ -377,4 +378,13 @@ public void setLocalEndpoint(InetSocketAddress addr) { public Optional getLocalEndpoint() { return Optional.ofNullable(_localEndpoint); } + + public void setRemoteEndpoint(InetSocketAddress addr) { + _remoteEndpoint = addr; + } + + @Override + public Optional getRemoteEndpoint() { + return Optional.ofNullable(_remoteEndpoint); + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java index 883eed6ce89..8c679832be1 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java @@ -205,6 +205,15 @@ default List remoteConnections() { return null; } + /** + * Whether this mover waits for the remote party to connect to it. Such a mover has no data + * connection, and therefore no network flow, until {@link #getRemoteEndpoint()} returns a + * value. + */ + default boolean isPassive() { + return false; + } + /** * Provide the expected total number of bytes transferred for this transfer, if known. Returns * null if this value is unknown. @@ -219,4 +228,14 @@ default Long getBytesExpected() { * to access the mover. */ Optional getLocalEndpoint(); + + /** + * Returns the {@link Optional} containing {@link InetSocketAddress} of the remote endpoint of + * the connection that carries the data, if the mover established that connection itself + * and knows it. This may differ from the address in the protocol info, for example if the + * remote server redirected the request. + */ + default Optional getRemoteEndpoint() { + return Optional.empty(); + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyMover.java index d77ba750e8c..29bda9610d6 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyMover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyMover.java @@ -44,4 +44,13 @@ public NettyMover(ReplicaDescriptor handle, public UUID getUuid() { return uuid; } + + /** + * A Netty mover listens for the client: the data connection exists once a client has + * presented the UUID of this mover. + */ + @Override + public boolean isPassive() { + return true; + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java index 511c1d16c4e..73350f2815d 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java @@ -50,6 +50,7 @@ import java.io.PrintWriter; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.channels.CompletionHandler; import java.util.ArrayList; @@ -61,6 +62,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import org.dcache.cells.CellStub; import org.dcache.pool.classic.Cancellable; @@ -368,6 +370,7 @@ public void execute() UUID uuid = mover.getUuid(); NettyMoverChannel channel = autoclose(new NettyMoverChannel(uuid, + mover, mover.open(), connectTimeoutUnit.toMillis(connectTimeout), this, mover::addChecksumType, @@ -382,9 +385,12 @@ public void execute() InetSocketAddress localEndpoint = new InetSocketAddress(localIP, getServerAddress().getPort()); mover.setLocalEndpoint(localEndpoint); - transferLifeCycle.onStart(((IpProtocolInfo)mover.getProtocolInfo()).getSocketAddress(), - localEndpoint, mover.getProtocolInfo(), mover.getSubject()); + /* No flow marker is sent here: the client has not connected yet, so the + * endpoints of the data connection are unknown. The address in the protocol + * info is that of the client's connection to the door. The marker is sent by + * NettyMoverChannel#connected once the client has attached to the mover. + */ sendAddressToDoor(mover, localEndpoint); } @@ -443,8 +449,11 @@ public class NettyMoverChannel extends MoverChannelDecorator

implements Cance private final Consumer checksumCalculation; private final Consumer integrityChecker; private final UUID moverUuid; + private final NettyMover

mover; + private final AtomicBoolean isConnected = new AtomicBoolean(); public NettyMoverChannel(UUID moverUuid, + NettyMover

mover, MoverChannel

file, long connectTimeout, CompletionHandler completionHandler, @@ -452,6 +461,7 @@ public NettyMoverChannel(UUID moverUuid, Consumer integrityChecker) { super(file); this.moverUuid = moverUuid; + this.mover = mover; this.completionHandler = completionHandler; this.checksumCalculation = checksumCalculation; this.integrityChecker = integrityChecker; @@ -480,6 +490,31 @@ public UUID getMoverUuid() { return moverUuid; } + /** + * Reports the connection through which a client has attached to this mover. This is the + * connection that carries the data, so its endpoints, rather than those the door saw, + * identify the network flow. The first connection reported is recorded on the mover + * and announced with a start flow marker; the end marker reuses the same endpoints. + * Further connections to the same mover (keep-alive requests, reconnects) are ignored. + * + * @param remote the client's end of the connection + * @param local the pool's end of the connection + */ + public void connected(SocketAddress remote, SocketAddress local) { + if (remote instanceof InetSocketAddress remoteEndpoint + && local instanceof InetSocketAddress localEndpoint + && isConnected.compareAndSet(false, true)) { + mover.setLocalEndpoint(localEndpoint); + mover.setRemoteEndpoint(remoteEndpoint); + if (transferLifeCycle != null) { + try (CDC ignored = cdc.restore()) { + transferLifeCycle.onStart(remoteEndpoint, localEndpoint, + mover.getProtocolInfo(), mover.getSubject()); + } + } + } + } + @Override public int read(ByteBuffer dst) throws IOException { checkState(sync.isExclusive()); diff --git a/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java b/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java new file mode 100644 index 00000000000..3238b0279b5 --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java @@ -0,0 +1,128 @@ +package org.dcache.pool.movers; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import diskCacheV111.vehicles.ProtocolInfo; +import io.netty.channel.local.LocalAddress; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.nio.channels.CompletionHandler; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import javax.security.auth.Subject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests that flow markers of a Netty mover describe the connection the client makes to the pool, + * not the client's connection to the door. + */ +public class NettyTransferServiceFlowMarkerTest { + + private static final InetSocketAddress CLIENT_DATA_CONNECTION = + new InetSocketAddress("203.0.113.20", 47742); + private static final InetSocketAddress POOL_DATA_CONNECTION = + new InetSocketAddress("198.51.100.55", 47548); + + private ScheduledExecutorService scheduler; + private TransferLifeCycle transferLifeCycle; + private NettyMover mover; + private ProtocolInfo protocolInfo; + private Subject subject; + private NettyTransferService.NettyMoverChannel channel; + + @Before + @SuppressWarnings("unchecked") + public void setup() throws Exception { + scheduler = Executors.newSingleThreadScheduledExecutor(); + transferLifeCycle = mock(TransferLifeCycle.class); + protocolInfo = mock(ProtocolInfo.class); + subject = new Subject(); + + mover = mock(NettyMover.class); + given(mover.getProtocolInfo()).willReturn(protocolInfo); + given(mover.getSubject()).willReturn(subject); + + NettyTransferService service = new NettyTransferService<>("test") { + @Override + protected void sendAddressToDoor(NettyMover mover, + InetSocketAddress localEndpoint) { + } + + @Override + protected UUID createUuid(ProtocolInfo protocolInfo) { + return UUID.randomUUID(); + } + }; + service.setTransferLifeCycle(transferLifeCycle); + + Field timeoutScheduler = NettyTransferService.class.getDeclaredField("timeoutScheduler"); + timeoutScheduler.setAccessible(true); + timeoutScheduler.set(service, scheduler); + + channel = service.new NettyMoverChannel(UUID.randomUUID(), mover, + mock(MoverChannel.class), 60_000, mock(CompletionHandler.class), + type -> {}, checksum -> {}); + } + + @After + public void tearDown() { + scheduler.shutdownNow(); + } + + @Test + public void shouldSendStartMarkerForTheConnectionTheClientMadeToThePool() { + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + + verify(transferLifeCycle).onStart(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION, + protocolInfo, subject); + } + + @Test + public void shouldRecordEndpointsOfDataConnectionOnMoverForEndMarker() { + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + + verify(mover).setRemoteEndpoint(CLIENT_DATA_CONNECTION); + verify(mover).setLocalEndpoint(POOL_DATA_CONNECTION); + } + + @Test + public void shouldNotSendStartMarkerBeforeClientConnects() { + verify(transferLifeCycle, never()).onStart(any(), any(), any(), any()); + verify(mover, never()).setRemoteEndpoint(any()); + } + + @Test + public void shouldSendOnlyOneStartMarkerWhenFurtherConnectionsAttach() { + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + channel.connected(new InetSocketAddress("203.0.113.20", 50000), POOL_DATA_CONNECTION); + + verify(transferLifeCycle, times(1)).onStart(any(), any(), any(), any()); + verify(mover, times(1)).setRemoteEndpoint(CLIENT_DATA_CONNECTION); + } + + @Test + public void shouldIgnoreConnectionsThatAreNotIpSockets() { + channel.connected(new LocalAddress("client"), new LocalAddress("pool")); + + verify(transferLifeCycle, never()).onStart(any(), any(), any(), any()); + verify(mover, never()).setRemoteEndpoint(any()); + } + + @Test + public void nettyMoverShouldBePassive() { + NettyMover realMover = mock(NettyMover.class, + org.mockito.Mockito.CALLS_REAL_METHODS); + + assertTrue(realMover.isPassive()); + } +} From eafe9b26340c6f315cf30e99b1026ddf4e6e0ca8 Mon Sep 17 00:00:00 2001 From: Shawn McKee Date: Sun, 15 Nov 2026 06:15:13 -0500 Subject: [PATCH 2/2] xrootd: take firefly flow-id of third-party copies from the source connection Motivation: When a pool is the destination of an xrootd third-party copy, the file arrives over a connection that the pool's third-party client opens to the source. The client that requested the copy connects to the pool too, but that connection carries no file data. Since the change for #8307, the requesting client's connection is not reported to the mover, so such transfers emit no flow markers. Before that change, the start marker named the requesting client's connection to the door, and the end marker named the source host and port from the request. Neither is the connection that carries the data, and the source may redirect the third-party client to another server. Modification: TpcWriteDescriptor reports the third-party client's connection to the mover (NettyMoverChannel#connected) when the first data arrives from the source. By then any redirect has been followed, so the connection reported is the one that carries the file. The start marker is sent then, and the end marker reuses the same endpoints. Result: Fireflies for xrootd third-party copies into dCache name the connection from the pool to the server that sends the file. A copy of a zero-length file, for which no data arrives, emits no markers. Testing: - New TpcWriteDescriptorFlowMarkerTest (5 tests): the connection is reported when the first data arrives, only once, and for the client that receives the data after a redirect. Three of the tests fail without the change. - PR pipeline unit-test job run locally in maven:3.9.12-eclipse-temurin-21: 5183 tests, 0 failures. - Backported to 11.2.10 and run on a production pool as the destination of `xrdcp --tpc delegate only` from a dCache source (MWT2), with the pool's connections captured. The source door redirected the pool to a source pool; both markers named that data connection (port for port) and the end marker reported the full file size as received. The requesting client appeared in no marker. Acked-by: Target: master Require-book: no Require-notes: yes Co-Authored-By: Claude Opus 5.5 Signed-off-by: Shawn McKee --- .../dcache/xrootd/tpc/TpcWriteDescriptor.java | 29 ++++ .../tpc/TpcWriteDescriptorFlowMarkerTest.java | 137 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 modules/dcache-xrootd/src/test/java/org/dcache/xrootd/tpc/TpcWriteDescriptorFlowMarkerTest.java diff --git a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/tpc/TpcWriteDescriptor.java b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/tpc/TpcWriteDescriptor.java index 0c8f8ef6b85..f26861a3e0a 100644 --- a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/tpc/TpcWriteDescriptor.java +++ b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/tpc/TpcWriteDescriptor.java @@ -129,6 +129,11 @@ public final class TpcWriteDescriptor extends WriteDescriptor private Integer transferStatus; private int clientChunkSize; + /* + * Whether the connection that carries the file has been reported to the mover. + */ + private volatile boolean isDataConnectionReported; + public TpcWriteDescriptor(NettyTransferService.NettyMoverChannel channel, boolean posc, ChannelHandlerContext ctx, @@ -328,9 +333,33 @@ public synchronized XrootdResponse sync(SyncRequest syncRequest) @Override public void write(InboundReadResponse inboundReadResponse) throws IOException { + reportDataConnection(); write((ByteBuffersProvider) inboundReadResponse); } + /** + * Reports the connection over which the file arrives as the network flow of the transfer. + * The connection from the client that requested the copy carries no file data; the file + * arrives over the connection the third-party client has made to the source. That + * connection is only known for certain once data arrives, since the source may redirect + * the client to another server first. + */ + private void reportDataConnection() { + if (isDataConnectionReported) { + return; + } + isDataConnectionReported = true; + + XrootdTpcClient current = client; + ChannelFuture future = current == null ? null : current.getChannelFuture(); + Channel channel = future == null ? null : future.channel(); + if (channel == null) { + LOGGER.debug("Third-party client has no channel; data connection not reported."); + return; + } + getChannel().connected(channel.remoteAddress(), channel.localAddress()); + } + @Override public void close() { if (client == null) { diff --git a/modules/dcache-xrootd/src/test/java/org/dcache/xrootd/tpc/TpcWriteDescriptorFlowMarkerTest.java b/modules/dcache-xrootd/src/test/java/org/dcache/xrootd/tpc/TpcWriteDescriptorFlowMarkerTest.java new file mode 100644 index 00000000000..92143a2a879 --- /dev/null +++ b/modules/dcache-xrootd/src/test/java/org/dcache/xrootd/tpc/TpcWriteDescriptorFlowMarkerTest.java @@ -0,0 +1,137 @@ +package org.dcache.xrootd.tpc; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.dcache.pool.movers.NettyTransferService; +import org.dcache.vehicles.XrootdProtocolInfo; +import org.dcache.xrootd.pool.XrootdTransferService; +import org.dcache.xrootd.tpc.protocol.messages.InboundReadResponse; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests that, when the pool is the destination of an xrootd third-party copy, the flow marker + * describes the connection over which the file arrives from the source. + */ +public class TpcWriteDescriptorFlowMarkerTest { + + private static final InetSocketAddress SOURCE = + new InetSocketAddress("203.0.113.30", 1094); + private static final InetSocketAddress REDIRECTED_SOURCE = + new InetSocketAddress("203.0.113.31", 21468); + private static final InetSocketAddress POOL = + new InetSocketAddress("198.51.100.55", 44034); + + private ScheduledExecutorService executor; + private NettyTransferService.NettyMoverChannel moverChannel; + private TpcWriteDescriptor descriptor; + + @Before + @SuppressWarnings("unchecked") + public void setup() { + executor = Executors.newSingleThreadScheduledExecutor(); + + moverChannel = mock(NettyTransferService.NettyMoverChannel.class); + given(moverChannel.getProtocolInfo()).willReturn(mock(XrootdProtocolInfo.class)); + + XrootdTransferService service = mock(XrootdTransferService.class); + given(service.getThirdPartyShutdownExecutor()).willReturn(executor); + given(service.getTpcClientChunkSize()).willReturn(1 << 20); + given(service.getTpcClientPlugins()).willReturn(Collections.emptyList()); + + XrootdTpcInfo info = mock(XrootdTpcInfo.class); + given(info.getSrcHost()).willReturn("127.0.0.1"); + given(info.getSrcPort()).willReturn(1094); + + descriptor = new TpcWriteDescriptor(moverChannel, false, + mock(ChannelHandlerContext.class), service, "user", info, null); + } + + @After + public void tearDown() { + executor.shutdownNow(); + } + + @Test + public void shouldNotReportAnyConnectionBeforeDataArrives() { + verify(moverChannel, never()).connected(any(), any()); + } + + @Test + public void shouldReportConnectionToSourceWhenFirstDataArrives() throws Exception { + givenThirdPartyClientConnectedTo(SOURCE); + + descriptor.write(someData()); + + verify(moverChannel).connected(SOURCE, POOL); + } + + @Test + public void shouldReportConnectionOnlyOnce() throws Exception { + givenThirdPartyClientConnectedTo(SOURCE); + + descriptor.write(someData()); + descriptor.write(someData()); + descriptor.write(someData()); + + verify(moverChannel, times(1)).connected(any(), any()); + } + + @Test + public void shouldReportConnectionOfClientThatReceivesDataAfterRedirect() throws Exception { + givenThirdPartyClientConnectedTo(SOURCE); + givenThirdPartyClientConnectedTo(REDIRECTED_SOURCE); + + descriptor.write(someData()); + + verify(moverChannel).connected(REDIRECTED_SOURCE, POOL); + verify(moverChannel, never()).connected(SOURCE, POOL); + } + + @Test + public void shouldNotFailWhenClientHasNoChannel() throws Exception { + setClient(mock(XrootdTpcClient.class)); + + descriptor.write(someData()); + + verify(moverChannel, never()).connected(any(), any()); + } + + private void givenThirdPartyClientConnectedTo(InetSocketAddress remote) throws Exception { + Channel channel = mock(Channel.class); + given(channel.remoteAddress()).willReturn(remote); + given(channel.localAddress()).willReturn(POOL); + ChannelFuture future = mock(ChannelFuture.class); + given(future.channel()).willReturn(channel); + XrootdTpcClient client = mock(XrootdTpcClient.class); + given(client.getChannelFuture()).willReturn(future); + setClient(client); + } + + private void setClient(XrootdTpcClient client) throws Exception { + Field field = TpcWriteDescriptor.class.getDeclaredField("client"); + field.setAccessible(true); + field.set(descriptor, client); + } + + private static InboundReadResponse someData() { + InboundReadResponse response = mock(InboundReadResponse.class); + given(response.getWriteOffset()).willReturn(0L); + given(response.toByteBuffers()).willReturn(new java.nio.ByteBuffer[0]); + return response; + } +}