diff --git a/modules/dcache/src/main/java/org/dcache/http/tpc/RemoteHttpTransferClient.java b/modules/dcache/src/main/java/org/dcache/http/tpc/RemoteHttpTransferClient.java index 06f6e0b9d50..8db098475f1 100644 --- a/modules/dcache/src/main/java/org/dcache/http/tpc/RemoteHttpTransferClient.java +++ b/modules/dcache/src/main/java/org/dcache/http/tpc/RemoteHttpTransferClient.java @@ -34,6 +34,7 @@ import diskCacheV111.util.ThirdPartyTransferFailedCacheException; import diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo; import java.io.IOException; +import java.io.OutputStream; import java.net.ConnectException; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -48,6 +49,7 @@ import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.regex.Pattern; @@ -70,6 +72,7 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.entity.HttpEntityWrapper; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.conn.ConnectionShutdownException; import org.apache.http.protocol.HttpContext; @@ -135,6 +138,7 @@ private enum HeaderFlags { = EnumSet.noneOf(HeaderFlags.class); private static final Logger LOGGER = LoggerFactory.getLogger(RemoteHttpTransferClient.class); + private static final Logger SCITAGS_LOGGER = LoggerFactory.getLogger("org.dcache.scitags"); /** * Maximum time to wait when establishing a connection. @@ -202,6 +206,7 @@ private enum HeaderFlags { private final CloseableHttpClient _client; private final RemoteHttpDataTransferProtocolInfo _info; private final Runnable _onConnected; + private final AtomicBoolean _connectionReported = new AtomicBoolean(); @GuardedBy("this") private HttpClientContext _context; @@ -217,8 +222,14 @@ private enum HeaderFlags { /** * @param client the HTTP client used to make all requests * @param info describes the remote resource, credentials and headers - * @param onConnected called after the first response headers of a request have been received, - * when the connection's endpoints are known + * @param onConnected called once, when the connection that carries the file's data is + * known: for a download, after the response headers of the last request + * have been received; for an upload, just before the first byte of the + * file is sent. At that time {@link #remoteAddress()} and + * {@link #localAddress()} describe that connection. An upload request + * that is redirected before any data is sent is not reported. If the + * file is sent again, because a server asked for a redirection only + * after it had received data, the later connection is not reported. */ public RemoteHttpTransferClient(CloseableHttpClient client, RemoteHttpDataTransferProtocolInfo info, Runnable onConnected) { @@ -368,6 +379,18 @@ private CloseableHttpResponse execute(HttpUriRequest request, HttpContext contex return _client.execute(request, context); } + private void reportConnection() { + if (_connectionReported.compareAndSet(false, true)) { + _onConnected.run(); + } else { + // The file is being sent again, on a connection other than the reported one. + SCITAGS_LOGGER.debug( + "scitags event=connection-unreported reason=data-sent-again remote={} local={}", + remoteAddress().map(Object::toString).orElse("-"), + localAddress().map(Object::toString).orElse("-")); + } + } + /** * The address of the remote endpoint of the most recent request, if known. */ @@ -551,8 +574,6 @@ private CloseableHttpResponse doGet(HttpContext context, long deadline) throws I HttpGet get = buildGetRequest(deadline); CloseableHttpResponse response = execute(get, context); - _onConnected.run(); - boolean isSuccessful = false; try { while (shouldRetry(response) && System.currentTimeMillis() < deadline) { @@ -560,9 +581,13 @@ private CloseableHttpResponse doGet(HttpContext context, long deadline) throws I response.close(); get = buildGetRequest(deadline); - response = execute(get, null); + // Use the same context, so that the endpoints describe this request's + // connection: the server may have closed the previous one. + response = execute(get, context); } + reportConnection(); + int statusCode = response.getStatusLine().getStatusCode(); String reason = response.getStatusLine().getReasonPhrase(); @@ -602,7 +627,6 @@ private void put(Supplier entity, Progress progress) redirectionCount > 0 ? REDIRECTED_REQUEST : INITIAL_REQUEST); try (CloseableHttpResponse response = execute(put, context)) { - _onConnected.run(); StatusLine status = response.getStatusLine(); switch (status.getStatusCode()) { case 200: /* OK (not actually a valid response from PUT) */ @@ -694,7 +718,16 @@ private HttpPut buildPutRequest(URI location, HttpEntity entity, Set mover, MoverInfoMessage moverInfoMessage) { protocolName(mover), mover.getFileAttributes().getPnfsId(), transferTag(mover)); - } else if (mover.isPassive() && mover.getRemoteEndpoint().isEmpty()) { - // Nobody connected to the mover: there was no flow and no start marker was sent. + } else if (hasNoDataConnection(mover)) { + // Nobody connected to the mover, or the mover failed before it connected to the + // remote party: there was no flow and no start marker was sent. SCITAGS_LOGGER.debug( "scitags lifecycle=end skip reason=no-data-connection protocol={} pnfsid={} transferTag={}", protocolName(mover), @@ -296,6 +297,16 @@ public void sendFinished(Mover mover, MoverInfoMessage moverInfoMessage) { _door.notify(mover.getPathToDoor(), finished); } + private static boolean hasNoDataConnection(Mover mover) { + if (mover.isPassive()) { + return mover.getRemoteEndpoint().isEmpty(); + } + if (mover.isActive()) { + return mover.getLocalEndpoint().isEmpty(); + } + return false; + } + private Optional deriveLocalEndpoint(InetSocketAddress remoteEndpoint) { if (remoteEndpoint == null) { return Optional.empty(); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java index 8c679832be1..7af43724267 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java @@ -214,6 +214,15 @@ default boolean isPassive() { return false; } + /** + * Whether this mover connects to the remote party itself and reports that connection through + * {@link #getLocalEndpoint()}. Such a mover has no data connection, and therefore no network + * flow, until {@link #getLocalEndpoint()} returns a value. + */ + default boolean isActive() { + return false; + } + /** * Provide the expected total number of bytes transferred for this transfer, if known. Returns * null if this value is unknown. diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java index d5e7b48cb11..e6a301753ea 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java @@ -71,4 +71,23 @@ default List remoteConnections() { default Optional getLocalEndpoint() { return Optional.empty(); } + + /** + * Returns the {@link Optional} containing {@link InetSocketAddress} of the remote endpoint of + * the connection that carries the data, if the mover established that connection itself + * and knows it. This may differ from the address in the protocol info, for example if the + * remote server redirected the request. + */ + default Optional getRemoteEndpoint() { + return Optional.empty(); + } + + /** + * Whether this mover connects to the remote party itself and reports that connection through + * {@link #getLocalEndpoint()}. Such a mover has no data connection, and therefore no network + * flow, until {@link #getLocalEndpoint()} returns a value. + */ + default boolean isActive() { + return false; + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocolMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocolMover.java index 27d02d8689e..d7a017c1448 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocolMover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocolMover.java @@ -83,4 +83,14 @@ public Long getBytesExpected() { public Optional getLocalEndpoint() { return _moverProtocol.getLocalEndpoint(); } + + @Override + public Optional getRemoteEndpoint() { + return _moverProtocol.getRemoteEndpoint(); + } + + @Override + public boolean isActive() { + return _moverProtocol.isActive(); + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java index f1c435da5c7..2bb3ce1daba 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java @@ -97,7 +97,8 @@ public class RemoteHttpDataTransferProtocol implements MoverProtocol, private volatile RemoteHttpTransferClient _transfer; - private InetSocketAddress _localEndpoint; + private volatile InetSocketAddress _localEndpoint; + private volatile InetSocketAddress _remoteEndpoint; private final TransferLifeCycle _transferLifeCycle; private Subject _subject; private boolean _startMarkerSent; @@ -164,8 +165,14 @@ public void runIO(FileAttributes attributes, RepositoryChannel channel, */ protected void afterTransfer() {} + /** + * Called once, when the connection that carries the file's data is known. The remote server + * may have redirected the request, and its name may resolve to several addresses, so both + * endpoints are taken from the connection rather than from the transfer's URL. + */ private void onConnected() { _localEndpoint = _transfer.localAddress().orElse(null); + _remoteEndpoint = _transfer.remoteAddress().orElse(null); startFlowMarker(); } @@ -239,7 +246,9 @@ private void startFlowMarker() { return; } - InetSocketAddress remoteEndpoint = ipInfo.getSocketAddress(); + InetSocketAddress remoteEndpoint = _remoteEndpoint != null + ? _remoteEndpoint + : ipInfo.getSocketAddress(); if (remoteEndpoint == null) { return; } @@ -253,4 +262,14 @@ private void startFlowMarker() { public Optional getLocalEndpoint() { return Optional.ofNullable(_localEndpoint); } + + @Override + public Optional getRemoteEndpoint() { + return Optional.ofNullable(_remoteEndpoint); + } + + @Override + public boolean isActive() { + return true; + } } diff --git a/modules/dcache/src/test/java/org/dcache/http/tpc/RemoteHttpTransferClientTest.java b/modules/dcache/src/test/java/org/dcache/http/tpc/RemoteHttpTransferClientTest.java index cfb2fe67bd0..8d8925068ed 100644 --- a/modules/dcache/src/test/java/org/dcache/http/tpc/RemoteHttpTransferClientTest.java +++ b/modules/dcache/src/test/java/org/dcache/http/tpc/RemoteHttpTransferClientTest.java @@ -25,16 +25,25 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import com.google.common.collect.ImmutableMap; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; import diskCacheV111.util.ThirdPartyTransferFailedCacheException; import diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.ArrayList; @@ -51,12 +60,14 @@ import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.protocol.HttpRequestExecutor; import org.dcache.util.Checksum; import org.dcache.util.ChecksumType; import org.junit.After; import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import org.slf4j.LoggerFactory; public class RemoteHttpTransferClientTest { @@ -270,6 +281,213 @@ public void shouldFollowRedirectOnUpload() throws Exception { assertThat(requests.get(1), equalTo("PUT /target")); } + @Test + public void shouldReportConnectionBeforeSendingUpload() throws Exception { + List> reported = Collections.synchronizedList(new ArrayList<>()); + AtomicInteger reportedWhenBodyReceived = new AtomicInteger(-1); + handle("/file", e -> { + if (e.getRequestMethod().equals("PUT")) { + e.getRequestBody().transferTo(uploaded); + reportedWhenBodyReceived.set(reported.size()); + respond(e, 201, null); + } else { + respond(e, 200, null, "Content-Length", String.valueOf(DATA.length)); + } + }); + AtomicReference transfer = new AtomicReference<>(); + transfer.set(new RemoteHttpTransferClient(client, info("/file", false), + () -> reported.add(transfer.get().remoteAddress()))); + + transfer.get().putAndVerify(RemoteHttpTransferClientTest::entity, NO_PROGRESS, + DATA.length, Optional.empty()); + + assertThat(uploaded.toByteArray(), equalTo(DATA)); + assertEquals("connection must be reported before the upload completes", + 1, reportedWhenBodyReceived.get()); + assertThat(reported.get(0).map(InetSocketAddress::getPort), + equalTo(Optional.of(server.getAddress().getPort()))); + } + + @Test + public void shouldReportOnlyDataConnectionWhenUploadIsRedirected() throws Exception { + handle("/target", e -> { + if (e.getRequestMethod().equals("PUT")) { + e.getRequestBody().transferTo(uploaded); + respond(e, 201, null); + } else { + respond(e, 200, null, "Content-Length", String.valueOf(DATA.length)); + } + }); + String target = "http://127.0.0.1:" + server.getAddress().getPort() + "/target"; + + // Like a dCache door: answers the request headers with a redirect, without sending + // "100 Continue", so the client never sends it any data. + try (ServerSocket door = new ServerSocket(0, 5, InetAddress.getLoopbackAddress())) { + Thread doorThread = new Thread(() -> redirectAllRequests(door, target)); + doorThread.setDaemon(true); + doorThread.start(); + + List> reported = + Collections.synchronizedList(new ArrayList<>()); + AtomicReference transfer = new AtomicReference<>(); + RemoteHttpDataTransferProtocolInfo info = new RemoteHttpDataTransferProtocolInfo( + "RemoteHttpDataTransfer", 1, 1, + new InetSocketAddress(InetAddress.getLoopbackAddress(), door.getLocalPort()), + "http://127.0.0.1:" + door.getLocalPort() + "/file", + false, ImmutableMap.of(), List.of()); + transfer.set(new RemoteHttpTransferClient(client, info, + () -> reported.add(transfer.get().remoteAddress()))); + + transfer.get().putAndVerify(RemoteHttpTransferClientTest::entity, NO_PROGRESS, + DATA.length, Optional.empty()); + + assertThat(uploaded.toByteArray(), equalTo(DATA)); + assertThat(reported.stream() + .map(a -> a.map(InetSocketAddress::getPort)).toList(), + contains(Optional.of(server.getAddress().getPort()))); + } + } + + @Test + public void shouldReportConnectionOfRetriedDownload() throws Exception { + // Like DPM while it calculates the checksum: "202 Accepted" until the file is ready. + // The first connection is closed, so the data arrives on another one. + List clientPorts = Collections.synchronizedList(new ArrayList<>()); + handle("/file", e -> { + clientPorts.add(e.getRemoteAddress().getPort()); + if (clientPorts.size() == 1) { + respond(e, 202, null, "Connection", "close"); + } else { + respond(e, 200, DATA); + } + }); + List> reported = Collections.synchronizedList(new ArrayList<>()); + AtomicReference transfer = new AtomicReference<>(); + transfer.set(new RemoteHttpTransferClient(client, info("/file", false), + () -> reported.add(transfer.get().localAddress()))); + ByteArrayOutputStream body = new ByteArrayOutputStream(); + + transfer.get().get(c -> {}, entity -> entity.writeTo(body), NO_PROGRESS, l -> {}); + + assertThat(body.toByteArray(), equalTo(DATA)); + assertEquals(2, clientPorts.size()); + Assume.assumeTrue("the retry used a new connection", + !clientPorts.get(0).equals(clientPorts.get(1))); + assertThat("only the connection that carries the data is reported", + reported.stream().map(a -> a.map(InetSocketAddress::getPort)).toList(), + contains(Optional.of(clientPorts.get(1)))); + } + + @Test + public void shouldReportConnectionOnlyOnceWhenUploadIsSentAgain() throws Exception { + handle("/target", e -> { + if (e.getRequestMethod().equals("PUT")) { + e.getRequestBody().transferTo(uploaded); + respond(e, 201, null); + } else { + respond(e, 200, null, "Content-Length", String.valueOf(DATA.length)); + } + }); + String target = "http://127.0.0.1:" + server.getAddress().getPort() + "/target"; + + // A server that does not answer "Expect: 100-continue": the client gives up waiting and + // sends the file, and only then learns that it has to send it somewhere else. + try (ServerSocket slowDoor = new ServerSocket(0, 5, InetAddress.getLoopbackAddress()); + CloseableHttpClient impatientClient = HttpClients.custom() + .setRequestExecutor(new HttpRequestExecutor(50)).build()) { + Thread doorThread = new Thread(() -> redirectAfterReceivingBody(slowDoor, target)); + doorThread.setDaemon(true); + doorThread.start(); + + List> reported = + Collections.synchronizedList(new ArrayList<>()); + AtomicInteger sent = new AtomicInteger(); + AtomicReference transfer = new AtomicReference<>(); + RemoteHttpDataTransferProtocolInfo info = new RemoteHttpDataTransferProtocolInfo( + "RemoteHttpDataTransfer", 1, 1, + new InetSocketAddress(InetAddress.getLoopbackAddress(), + slowDoor.getLocalPort()), + "http://127.0.0.1:" + slowDoor.getLocalPort() + "/file", + false, ImmutableMap.of(), List.of()); + transfer.set(new RemoteHttpTransferClient(impatientClient, info, + () -> reported.add(transfer.get().remoteAddress()))); + + Logger scitagsLogger = (Logger) LoggerFactory.getLogger("org.dcache.scitags"); + Level previousLevel = scitagsLogger.getLevel(); + ListAppender log = new ListAppender<>(); + log.start(); + scitagsLogger.addAppender(log); + scitagsLogger.setLevel(Level.DEBUG); + try { + transfer.get().putAndVerify(() -> new ByteArrayEntity(DATA) { + @Override + public void writeTo(OutputStream out) throws IOException { + sent.incrementAndGet(); + super.writeTo(out); + } + }, NO_PROGRESS, DATA.length, Optional.empty()); + } finally { + scitagsLogger.detachAppender(log); + scitagsLogger.setLevel(previousLevel); + } + + assertThat(uploaded.toByteArray(), equalTo(DATA)); + assertEquals("the file is sent to both servers", 2, sent.get()); + assertThat("the connection is reported once, when data is first sent", + reported.stream().map(a -> a.map(InetSocketAddress::getPort)).toList(), + contains(Optional.of(slowDoor.getLocalPort()))); + assertThat("the second connection is logged", + log.list.stream().map(ILoggingEvent::getFormattedMessage) + .filter(m -> m.contains("event=connection-unreported")).toList(), + contains(containsString("reason=data-sent-again remote=/127.0.0.1:" + + server.getAddress().getPort() + " "))); + } + } + + private static void redirectAfterReceivingBody(ServerSocket door, String location) { + while (!door.isClosed()) { + try (Socket socket = door.accept()) { + InputStream in = socket.getInputStream(); + StringBuilder headers = new StringBuilder(); + int c; + while (!headers.toString().endsWith("\r\n\r\n") && (c = in.read()) != -1) { + headers.append((char) c); + } + if (headers.toString().startsWith("PUT ")) { + // no "100 Continue": wait until the client sends the file anyway + in.readNBytes(DATA.length); + } + OutputStream out = socket.getOutputStream(); + out.write(("HTTP/1.1 307 Temporary Redirect\r\nLocation: " + location + + "\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.US_ASCII)); + out.flush(); + } catch (IOException e) { + // the test closed the socket, or the client went away + } + } + } + + private static void redirectAllRequests(ServerSocket door, String location) { + while (!door.isClosed()) { + try (Socket socket = door.accept()) { + BufferedReader in = new BufferedReader(new InputStreamReader( + socket.getInputStream(), StandardCharsets.US_ASCII)); + String line; + while ((line = in.readLine()) != null && !line.isEmpty()) { + // skip the request line and headers + } + OutputStream out = socket.getOutputStream(); + out.write(("HTTP/1.1 307 Temporary Redirect\r\nLocation: " + location + + "\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.US_ASCII)); + out.flush(); + } catch (IOException e) { + // the test closed the socket, or the client went away + } + } + } + @Test public void shouldDeleteRemoteFileWhenChecksumMismatches() throws Exception { handle("/file", e -> { diff --git a/modules/dcache/src/test/java/org/dcache/pool/classic/DefaultPostTransferServiceTest.java b/modules/dcache/src/test/java/org/dcache/pool/classic/DefaultPostTransferServiceTest.java new file mode 100644 index 00000000000..1577b267b32 --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/pool/classic/DefaultPostTransferServiceTest.java @@ -0,0 +1,103 @@ +package org.dcache.pool.classic; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import diskCacheV111.util.PnfsId; +import diskCacheV111.vehicles.IpProtocolInfo; +import diskCacheV111.vehicles.MoverInfoMessage; +import dmg.cells.nucleus.CellAddressCore; +import dmg.cells.nucleus.CellEndpoint; +import dmg.cells.nucleus.CellPath; +import java.net.InetSocketAddress; +import java.util.Optional; +import org.dcache.pool.movers.Mover; +import org.dcache.pool.movers.TransferLifeCycle; +import org.dcache.vehicles.FileAttributes; +import org.junit.Before; +import org.junit.Test; + +public class DefaultPostTransferServiceTest { + + private static final PnfsId PNFSID = new PnfsId("000000000000000000000000000000000001"); + private static final InetSocketAddress REMOTE_FROM_DOOR = + new InetSocketAddress("192.0.2.10", 443); + private static final InetSocketAddress REMOTE_CONNECTED = + new InetSocketAddress("192.0.2.20", 24100); + private static final InetSocketAddress LOCAL = new InetSocketAddress("198.51.100.5", 40000); + + private DefaultPostTransferService service; + private TransferLifeCycle transferLifeCycle; + private Mover mover; + private MoverInfoMessage moverInfo; + + @Before + @SuppressWarnings("unchecked") + public void setUp() { + transferLifeCycle = mock(TransferLifeCycle.class); + + service = new DefaultPostTransferService(); + service.setPoolName("pool_a"); + service.setTransferLifeCycle(transferLifeCycle); + service.setCellEndpoint(mock(CellEndpoint.class)); + service.init(); + + IpProtocolInfo protocolInfo = mock(IpProtocolInfo.class); + when(protocolInfo.getProtocol()).thenReturn("RemoteHttpsDataTransfer"); + when(protocolInfo.getSocketAddress()).thenReturn(REMOTE_FROM_DOOR); + + mover = mock(Mover.class); + when(mover.getProtocolInfo()).thenReturn(protocolInfo); + when(mover.getFileAttributes()).thenReturn(FileAttributes.ofPnfsId(PNFSID)); + when(mover.getPathToDoor()).thenReturn(new CellPath("door")); + when(mover.getLocalEndpoint()).thenReturn(Optional.empty()); + when(mover.getRemoteEndpoint()).thenReturn(Optional.empty()); + + moverInfo = new MoverInfoMessage(new CellAddressCore("pool_a", "poolDomain"), PNFSID); + } + + @Test + public void shouldSendNoEndMarkerForActiveMoverThatNeverConnected() { + // e.g. an HTTP-TPC pull that failed on the DNS lookup of the source + when(mover.isActive()).thenReturn(true); + + service.sendFinished(mover, moverInfo); + + verify(transferLifeCycle, never()).onEnd(any(), any(), any()); + } + + @Test + public void shouldSendEndMarkerForConnectionOfActiveMover() { + when(mover.isActive()).thenReturn(true); + when(mover.getLocalEndpoint()).thenReturn(Optional.of(LOCAL)); + when(mover.getRemoteEndpoint()).thenReturn(Optional.of(REMOTE_CONNECTED)); + + service.sendFinished(mover, moverInfo); + + verify(transferLifeCycle).onEnd(REMOTE_CONNECTED, LOCAL, moverInfo); + } + + @Test + public void shouldSendNoEndMarkerForPassiveMoverNobodyConnectedTo() { + when(mover.isPassive()).thenReturn(true); + when(mover.getLocalEndpoint()).thenReturn(Optional.of(LOCAL)); + + service.sendFinished(mover, moverInfo); + + verify(transferLifeCycle, never()).onEnd(any(), any(), any()); + } + + @Test + public void shouldSendEndMarkerForPassiveMoverWithClient() { + when(mover.isPassive()).thenReturn(true); + when(mover.getLocalEndpoint()).thenReturn(Optional.of(LOCAL)); + when(mover.getRemoteEndpoint()).thenReturn(Optional.of(REMOTE_CONNECTED)); + + service.sendFinished(mover, moverInfo); + + verify(transferLifeCycle).onEnd(REMOTE_CONNECTED, LOCAL, moverInfo); + } +}