diff --git a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdPoolRequestHandler.java b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdPoolRequestHandler.java index 0d166a930e7..b0c443651f3 100644 --- a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdPoolRequestHandler.java +++ b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdPoolRequestHandler.java @@ -372,6 +372,17 @@ protected XrootdResponse doOnAuthentication(ChannelHandle return withOk(msg); } + /** + * Reports the client's connection as the network flow of the transfer. This is not done when + * the pool is the destination of a third-party copy: the file then arrives over a connection + * that the third-party client opens to the source, and the connection from the client that + * requested the copy carries no file data. + */ + private static void reportDataConnection(ChannelHandlerContext ctx, + NettyTransferService.NettyMoverChannel file) { + file.connected(ctx.channel().remoteAddress(), ctx.channel().localAddress()); + } + /** * Obtains the right mover channel using an opaque token in the request. The mover channel is * wrapped by a file descriptor. The file descriptor is stored for subsequent access. @@ -444,9 +455,11 @@ protected XrootdResponse doOnOpen(ChannelHandlerContext ctx, tlsSessionInfo); } else { descriptor = new WriteDescriptor(file, posc); + reportDataConnection(ctx, file); } } else { descriptor = new ReadDescriptor(file); + reportDataConnection(ctx, file); } FileStatus stat = msg.isRetStat() ? stat(file) : null; diff --git a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/tpc/TpcWriteDescriptor.java b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/tpc/TpcWriteDescriptor.java index 0c8f8ef6b85..f26861a3e0a 100644 --- a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/tpc/TpcWriteDescriptor.java +++ b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/tpc/TpcWriteDescriptor.java @@ -129,6 +129,11 @@ public final class TpcWriteDescriptor extends WriteDescriptor private Integer transferStatus; private int clientChunkSize; + /* + * Whether the connection that carries the file has been reported to the mover. + */ + private volatile boolean isDataConnectionReported; + public TpcWriteDescriptor(NettyTransferService.NettyMoverChannel channel, boolean posc, ChannelHandlerContext ctx, @@ -328,9 +333,33 @@ public synchronized XrootdResponse sync(SyncRequest syncRequest) @Override public void write(InboundReadResponse inboundReadResponse) throws IOException { + reportDataConnection(); write((ByteBuffersProvider) inboundReadResponse); } + /** + * Reports the connection over which the file arrives as the network flow of the transfer. + * The connection from the client that requested the copy carries no file data; the file + * arrives over the connection the third-party client has made to the source. That + * connection is only known for certain once data arrives, since the source may redirect + * the client to another server first. + */ + private void reportDataConnection() { + if (isDataConnectionReported) { + return; + } + isDataConnectionReported = true; + + XrootdTpcClient current = client; + ChannelFuture future = current == null ? null : current.getChannelFuture(); + Channel channel = future == null ? null : future.channel(); + if (channel == null) { + LOGGER.debug("Third-party client has no channel; data connection not reported."); + return; + } + getChannel().connected(channel.remoteAddress(), channel.localAddress()); + } + @Override public void close() { if (client == null) { diff --git a/modules/dcache-xrootd/src/test/java/org/dcache/xrootd/tpc/TpcWriteDescriptorFlowMarkerTest.java b/modules/dcache-xrootd/src/test/java/org/dcache/xrootd/tpc/TpcWriteDescriptorFlowMarkerTest.java new file mode 100644 index 00000000000..92143a2a879 --- /dev/null +++ b/modules/dcache-xrootd/src/test/java/org/dcache/xrootd/tpc/TpcWriteDescriptorFlowMarkerTest.java @@ -0,0 +1,137 @@ +package org.dcache.xrootd.tpc; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.dcache.pool.movers.NettyTransferService; +import org.dcache.vehicles.XrootdProtocolInfo; +import org.dcache.xrootd.pool.XrootdTransferService; +import org.dcache.xrootd.tpc.protocol.messages.InboundReadResponse; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests that, when the pool is the destination of an xrootd third-party copy, the flow marker + * describes the connection over which the file arrives from the source. + */ +public class TpcWriteDescriptorFlowMarkerTest { + + private static final InetSocketAddress SOURCE = + new InetSocketAddress("203.0.113.30", 1094); + private static final InetSocketAddress REDIRECTED_SOURCE = + new InetSocketAddress("203.0.113.31", 21468); + private static final InetSocketAddress POOL = + new InetSocketAddress("198.51.100.55", 44034); + + private ScheduledExecutorService executor; + private NettyTransferService.NettyMoverChannel moverChannel; + private TpcWriteDescriptor descriptor; + + @Before + @SuppressWarnings("unchecked") + public void setup() { + executor = Executors.newSingleThreadScheduledExecutor(); + + moverChannel = mock(NettyTransferService.NettyMoverChannel.class); + given(moverChannel.getProtocolInfo()).willReturn(mock(XrootdProtocolInfo.class)); + + XrootdTransferService service = mock(XrootdTransferService.class); + given(service.getThirdPartyShutdownExecutor()).willReturn(executor); + given(service.getTpcClientChunkSize()).willReturn(1 << 20); + given(service.getTpcClientPlugins()).willReturn(Collections.emptyList()); + + XrootdTpcInfo info = mock(XrootdTpcInfo.class); + given(info.getSrcHost()).willReturn("127.0.0.1"); + given(info.getSrcPort()).willReturn(1094); + + descriptor = new TpcWriteDescriptor(moverChannel, false, + mock(ChannelHandlerContext.class), service, "user", info, null); + } + + @After + public void tearDown() { + executor.shutdownNow(); + } + + @Test + public void shouldNotReportAnyConnectionBeforeDataArrives() { + verify(moverChannel, never()).connected(any(), any()); + } + + @Test + public void shouldReportConnectionToSourceWhenFirstDataArrives() throws Exception { + givenThirdPartyClientConnectedTo(SOURCE); + + descriptor.write(someData()); + + verify(moverChannel).connected(SOURCE, POOL); + } + + @Test + public void shouldReportConnectionOnlyOnce() throws Exception { + givenThirdPartyClientConnectedTo(SOURCE); + + descriptor.write(someData()); + descriptor.write(someData()); + descriptor.write(someData()); + + verify(moverChannel, times(1)).connected(any(), any()); + } + + @Test + public void shouldReportConnectionOfClientThatReceivesDataAfterRedirect() throws Exception { + givenThirdPartyClientConnectedTo(SOURCE); + givenThirdPartyClientConnectedTo(REDIRECTED_SOURCE); + + descriptor.write(someData()); + + verify(moverChannel).connected(REDIRECTED_SOURCE, POOL); + verify(moverChannel, never()).connected(SOURCE, POOL); + } + + @Test + public void shouldNotFailWhenClientHasNoChannel() throws Exception { + setClient(mock(XrootdTpcClient.class)); + + descriptor.write(someData()); + + verify(moverChannel, never()).connected(any(), any()); + } + + private void givenThirdPartyClientConnectedTo(InetSocketAddress remote) throws Exception { + Channel channel = mock(Channel.class); + given(channel.remoteAddress()).willReturn(remote); + given(channel.localAddress()).willReturn(POOL); + ChannelFuture future = mock(ChannelFuture.class); + given(future.channel()).willReturn(channel); + XrootdTpcClient client = mock(XrootdTpcClient.class); + given(client.getChannelFuture()).willReturn(future); + setClient(client); + } + + private void setClient(XrootdTpcClient client) throws Exception { + Field field = TpcWriteDescriptor.class.getDeclaredField("client"); + field.setAccessible(true); + field.set(descriptor, client); + } + + private static InboundReadResponse someData() { + InboundReadResponse response = mock(InboundReadResponse.class); + given(response.getWriteOffset()).willReturn(0L); + given(response.toByteBuffers()).willReturn(new java.nio.ByteBuffer[0]); + return response; + } +} diff --git a/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java b/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java index 7b18f694e31..0612b2722fc 100644 --- a/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java +++ b/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java @@ -435,6 +435,9 @@ protected ChannelFuture doOnGet(ChannelHandlerContext context, "Resource is not open for reading"); } + // This connection carries the file: report it as the transfer's network flow. + file.connected(context.channel().remoteAddress(), context.channel().localAddress()); + fileSize = file.size(); ranges = parseHttpRange(request, 0, fileSize - 1); } catch (Redirect e) { @@ -547,6 +550,9 @@ protected ChannelFuture doOnPut(ChannelHandlerContext context, HttpRequest reque "Resource is not open for writing"); } + // This connection carries the file: report it as the transfer's network flow. + file.connected(context.channel().remoteAddress(), context.channel().localAddress()); + contentMd5Checksum(request).ifPresent(file::addChecksum); OptionalLong contentLength = contentLength(request); diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java index 7b0c2655f31..07257bd591b 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/DefaultPostTransferService.java @@ -237,8 +237,18 @@ public void sendFinished(Mover mover, MoverInfoMessage moverInfoMessage) { protocolName(mover), mover.getFileAttributes().getPnfsId(), transferTag(mover)); + } else if (mover.isPassive() && mover.getRemoteEndpoint().isEmpty()) { + // Nobody connected to the mover: there was no flow and no start marker was sent. + SCITAGS_LOGGER.debug( + "scitags lifecycle=end skip reason=no-data-connection protocol={} pnfsid={} transferTag={}", + protocolName(mover), + mover.getFileAttributes().getPnfsId(), + transferTag(mover)); } else { - InetSocketAddress remoteEndpoint = ipProtocolInfo.getSocketAddress(); + // Prefer the endpoint of the connection the mover made, as reported in the start + // marker; the address in the protocol info is the one the door resolved. + InetSocketAddress remoteEndpoint = mover.getRemoteEndpoint() + .orElseGet(ipProtocolInfo::getSocketAddress); if (remoteEndpoint == null) { SCITAGS_LOGGER.debug( "scitags lifecycle=end skip reason=no-remote-endpoint protocol={} pnfsid={} transferTag={}", diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java index 65ad1ab7ab9..b69727bc4d5 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/AbstractMover.java @@ -85,6 +85,7 @@ public abstract class AbstractMover

_channel = Optional.empty(); private volatile InetSocketAddress _localEndpoint; + private volatile InetSocketAddress _remoteEndpoint; public AbstractMover(ReplicaDescriptor handle, PoolIoFileMessage message, CellPath pathToDoor, TransferService transferService) { @@ -377,4 +378,13 @@ public void setLocalEndpoint(InetSocketAddress addr) { public Optional getLocalEndpoint() { return Optional.ofNullable(_localEndpoint); } + + public void setRemoteEndpoint(InetSocketAddress addr) { + _remoteEndpoint = addr; + } + + @Override + public Optional getRemoteEndpoint() { + return Optional.ofNullable(_remoteEndpoint); + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java index 883eed6ce89..8c679832be1 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java @@ -205,6 +205,15 @@ default List remoteConnections() { return null; } + /** + * Whether this mover waits for the remote party to connect to it. Such a mover has no data + * connection, and therefore no network flow, until {@link #getRemoteEndpoint()} returns a + * value. + */ + default boolean isPassive() { + return false; + } + /** * Provide the expected total number of bytes transferred for this transfer, if known. Returns * null if this value is unknown. @@ -219,4 +228,14 @@ default Long getBytesExpected() { * to access the mover. */ Optional getLocalEndpoint(); + + /** + * Returns the {@link Optional} containing {@link InetSocketAddress} of the remote endpoint of + * the connection that carries the data, if the mover established that connection itself + * and knows it. This may differ from the address in the protocol info, for example if the + * remote server redirected the request. + */ + default Optional getRemoteEndpoint() { + return Optional.empty(); + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyMover.java index d77ba750e8c..29bda9610d6 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyMover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyMover.java @@ -44,4 +44,13 @@ public NettyMover(ReplicaDescriptor handle, public UUID getUuid() { return uuid; } + + /** + * A Netty mover listens for the client: the data connection exists once a client has + * presented the UUID of this mover. + */ + @Override + public boolean isPassive() { + return true; + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java index 511c1d16c4e..73350f2815d 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/NettyTransferService.java @@ -50,6 +50,7 @@ import java.io.PrintWriter; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.channels.CompletionHandler; import java.util.ArrayList; @@ -61,6 +62,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import org.dcache.cells.CellStub; import org.dcache.pool.classic.Cancellable; @@ -368,6 +370,7 @@ public void execute() UUID uuid = mover.getUuid(); NettyMoverChannel channel = autoclose(new NettyMoverChannel(uuid, + mover, mover.open(), connectTimeoutUnit.toMillis(connectTimeout), this, mover::addChecksumType, @@ -382,9 +385,12 @@ public void execute() InetSocketAddress localEndpoint = new InetSocketAddress(localIP, getServerAddress().getPort()); mover.setLocalEndpoint(localEndpoint); - transferLifeCycle.onStart(((IpProtocolInfo)mover.getProtocolInfo()).getSocketAddress(), - localEndpoint, mover.getProtocolInfo(), mover.getSubject()); + /* No flow marker is sent here: the client has not connected yet, so the + * endpoints of the data connection are unknown. The address in the protocol + * info is that of the client's connection to the door. The marker is sent by + * NettyMoverChannel#connected once the client has attached to the mover. + */ sendAddressToDoor(mover, localEndpoint); } @@ -443,8 +449,11 @@ public class NettyMoverChannel extends MoverChannelDecorator

implements Cance private final Consumer checksumCalculation; private final Consumer integrityChecker; private final UUID moverUuid; + private final NettyMover

mover; + private final AtomicBoolean isConnected = new AtomicBoolean(); public NettyMoverChannel(UUID moverUuid, + NettyMover

mover, MoverChannel

file, long connectTimeout, CompletionHandler completionHandler, @@ -452,6 +461,7 @@ public NettyMoverChannel(UUID moverUuid, Consumer integrityChecker) { super(file); this.moverUuid = moverUuid; + this.mover = mover; this.completionHandler = completionHandler; this.checksumCalculation = checksumCalculation; this.integrityChecker = integrityChecker; @@ -480,6 +490,31 @@ public UUID getMoverUuid() { return moverUuid; } + /** + * Reports the connection through which a client has attached to this mover. This is the + * connection that carries the data, so its endpoints, rather than those the door saw, + * identify the network flow. The first connection reported is recorded on the mover + * and announced with a start flow marker; the end marker reuses the same endpoints. + * Further connections to the same mover (keep-alive requests, reconnects) are ignored. + * + * @param remote the client's end of the connection + * @param local the pool's end of the connection + */ + public void connected(SocketAddress remote, SocketAddress local) { + if (remote instanceof InetSocketAddress remoteEndpoint + && local instanceof InetSocketAddress localEndpoint + && isConnected.compareAndSet(false, true)) { + mover.setLocalEndpoint(localEndpoint); + mover.setRemoteEndpoint(remoteEndpoint); + if (transferLifeCycle != null) { + try (CDC ignored = cdc.restore()) { + transferLifeCycle.onStart(remoteEndpoint, localEndpoint, + mover.getProtocolInfo(), mover.getSubject()); + } + } + } + } + @Override public int read(ByteBuffer dst) throws IOException { checkState(sync.isExclusive()); diff --git a/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java b/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java new file mode 100644 index 00000000000..3238b0279b5 --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/pool/movers/NettyTransferServiceFlowMarkerTest.java @@ -0,0 +1,128 @@ +package org.dcache.pool.movers; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import diskCacheV111.vehicles.ProtocolInfo; +import io.netty.channel.local.LocalAddress; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.nio.channels.CompletionHandler; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import javax.security.auth.Subject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests that flow markers of a Netty mover describe the connection the client makes to the pool, + * not the client's connection to the door. + */ +public class NettyTransferServiceFlowMarkerTest { + + private static final InetSocketAddress CLIENT_DATA_CONNECTION = + new InetSocketAddress("203.0.113.20", 47742); + private static final InetSocketAddress POOL_DATA_CONNECTION = + new InetSocketAddress("198.51.100.55", 47548); + + private ScheduledExecutorService scheduler; + private TransferLifeCycle transferLifeCycle; + private NettyMover mover; + private ProtocolInfo protocolInfo; + private Subject subject; + private NettyTransferService.NettyMoverChannel channel; + + @Before + @SuppressWarnings("unchecked") + public void setup() throws Exception { + scheduler = Executors.newSingleThreadScheduledExecutor(); + transferLifeCycle = mock(TransferLifeCycle.class); + protocolInfo = mock(ProtocolInfo.class); + subject = new Subject(); + + mover = mock(NettyMover.class); + given(mover.getProtocolInfo()).willReturn(protocolInfo); + given(mover.getSubject()).willReturn(subject); + + NettyTransferService service = new NettyTransferService<>("test") { + @Override + protected void sendAddressToDoor(NettyMover mover, + InetSocketAddress localEndpoint) { + } + + @Override + protected UUID createUuid(ProtocolInfo protocolInfo) { + return UUID.randomUUID(); + } + }; + service.setTransferLifeCycle(transferLifeCycle); + + Field timeoutScheduler = NettyTransferService.class.getDeclaredField("timeoutScheduler"); + timeoutScheduler.setAccessible(true); + timeoutScheduler.set(service, scheduler); + + channel = service.new NettyMoverChannel(UUID.randomUUID(), mover, + mock(MoverChannel.class), 60_000, mock(CompletionHandler.class), + type -> {}, checksum -> {}); + } + + @After + public void tearDown() { + scheduler.shutdownNow(); + } + + @Test + public void shouldSendStartMarkerForTheConnectionTheClientMadeToThePool() { + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + + verify(transferLifeCycle).onStart(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION, + protocolInfo, subject); + } + + @Test + public void shouldRecordEndpointsOfDataConnectionOnMoverForEndMarker() { + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + + verify(mover).setRemoteEndpoint(CLIENT_DATA_CONNECTION); + verify(mover).setLocalEndpoint(POOL_DATA_CONNECTION); + } + + @Test + public void shouldNotSendStartMarkerBeforeClientConnects() { + verify(transferLifeCycle, never()).onStart(any(), any(), any(), any()); + verify(mover, never()).setRemoteEndpoint(any()); + } + + @Test + public void shouldSendOnlyOneStartMarkerWhenFurtherConnectionsAttach() { + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + channel.connected(CLIENT_DATA_CONNECTION, POOL_DATA_CONNECTION); + channel.connected(new InetSocketAddress("203.0.113.20", 50000), POOL_DATA_CONNECTION); + + verify(transferLifeCycle, times(1)).onStart(any(), any(), any(), any()); + verify(mover, times(1)).setRemoteEndpoint(CLIENT_DATA_CONNECTION); + } + + @Test + public void shouldIgnoreConnectionsThatAreNotIpSockets() { + channel.connected(new LocalAddress("client"), new LocalAddress("pool")); + + verify(transferLifeCycle, never()).onStart(any(), any(), any(), any()); + verify(mover, never()).setRemoteEndpoint(any()); + } + + @Test + public void nettyMoverShouldBePassive() { + NettyMover realMover = mock(NettyMover.class, + org.mockito.Mockito.CALLS_REAL_METHODS); + + assertTrue(realMover.isPassive()); + } +}