From 5c37b8329d3efc81bfcfc5dc6dbb6ef254a94297 Mon Sep 17 00:00:00 2001 From: Shawn McKee Date: Fri, 13 Nov 2026 14:02:15 -0500 Subject: [PATCH] pool: report the data connection in firefly markers of HTTP-TPC transfers Motivation: For HTTP third-party copies the flow-id of the firefly markers does not describe the connection that carries the data (#8305): - The remote endpoint is the address the door resolved from the host name in the transfer URL. If the remote server redirects the request, or its name has several addresses, the marker names a host the data does not go to. The marker is also sent to that host. - For uploads, the transfer client reported a connection after each request had completed. A request that is redirected completes at once, so the start marker described the connection to the redirecting server, while the end marker described the data connection. If the request was not redirected, the start marker was only sent once all data had been sent. Observed on a pool: a push to a remote dCache produced markers naming :2881, with local port 55138 in the start marker and 44034 in the end marker, while the data went from local port 44034 to :21468. Of 13 pulls in production traffic, 12 had a marker whose remote endpoint differed from the connection. Modification: RemoteHttpTransferClient reports the connection of an upload just before the first byte of the entity is written, instead of after each request. With expect-continue the entity is not written to a server that redirects the request, so only the data connection is reported. For a download it reports the connection after the retry loop, and the retried request uses the same context: while a server answers 202 the connection may be closed, and the data then arrives on a new one. The connection is reported once. If a server asks for a redirection only after it has received the file, the file is sent again; the second connection is not reported, so that start and end markers agree. It is logged at debug level by the org.dcache.scitags logger (event=connection-unreported), so that a site can tell whether this happens. RemoteHttpDataTransferProtocol takes the remote endpoint of the start marker from that connection, as it already does for the local endpoint, and falls back to the address in the protocol info. Both endpoint fields are volatile. Add Mover#getRemoteEndpoint and MoverProtocol#getRemoteEndpoint. DefaultPostTransferService uses it for the end marker, so that start and end markers carry the same flow-id. Add Mover#isActive and MoverProtocol#isActive for a mover that connects to the remote party itself and reports that connection. DefaultPostTransferService sends no end marker for such a mover if it never connected, as it already does for a passive mover nobody connected to. Before, a pull that failed on the DNS lookup or the TLS handshake sent an end marker with local port 0 and no start marker. Result: The firefly markers of an HTTP-TPC transfer describe the TCP connection that carries the data, for pulls and pushes, also when the remote server redirects the request, makes the pool retry the request, or its name has several addresses. The start marker of a push is sent when the upload starts. A transfer that fails before the pool has a connection to report, for example on the DNS lookup or the TLS handshake, sends no markers. Fixes: #8305 Acked-by: Target: master Require-book: no Require-notes: yes Signed-off-by: Shawn McKee Co-Authored-By: Claude Fable 5.2 --- .../http/tpc/RemoteHttpTransferClient.java | 47 +++- .../classic/DefaultPostTransferService.java | 15 +- .../java/org/dcache/pool/movers/Mover.java | 9 + .../org/dcache/pool/movers/MoverProtocol.java | 19 ++ .../pool/movers/MoverProtocolMover.java | 10 + .../RemoteHttpDataTransferProtocol.java | 23 +- .../tpc/RemoteHttpTransferClientTest.java | 218 ++++++++++++++++++ .../DefaultPostTransferServiceTest.java | 103 +++++++++ 8 files changed, 433 insertions(+), 11 deletions(-) create mode 100644 modules/dcache/src/test/java/org/dcache/pool/classic/DefaultPostTransferServiceTest.java diff --git a/modules/dcache/src/main/java/org/dcache/http/tpc/RemoteHttpTransferClient.java b/modules/dcache/src/main/java/org/dcache/http/tpc/RemoteHttpTransferClient.java index 06f6e0b9d50..8db098475f1 100644 --- a/modules/dcache/src/main/java/org/dcache/http/tpc/RemoteHttpTransferClient.java +++ b/modules/dcache/src/main/java/org/dcache/http/tpc/RemoteHttpTransferClient.java @@ -34,6 +34,7 @@ import diskCacheV111.util.ThirdPartyTransferFailedCacheException; import diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo; import java.io.IOException; +import java.io.OutputStream; import java.net.ConnectException; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -48,6 +49,7 @@ import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.regex.Pattern; @@ -70,6 +72,7 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.entity.HttpEntityWrapper; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.conn.ConnectionShutdownException; import org.apache.http.protocol.HttpContext; @@ -135,6 +138,7 @@ private enum HeaderFlags { = EnumSet.noneOf(HeaderFlags.class); private static final Logger LOGGER = LoggerFactory.getLogger(RemoteHttpTransferClient.class); + private static final Logger SCITAGS_LOGGER = LoggerFactory.getLogger("org.dcache.scitags"); /** * Maximum time to wait when establishing a connection. @@ -202,6 +206,7 @@ private enum HeaderFlags { private final CloseableHttpClient _client; private final RemoteHttpDataTransferProtocolInfo _info; private final Runnable _onConnected; + private final AtomicBoolean _connectionReported = new AtomicBoolean(); @GuardedBy("this") private HttpClientContext _context; @@ -217,8 +222,14 @@ private enum HeaderFlags { /** * @param client the HTTP client used to make all requests * @param info describes the remote resource, credentials and headers - * @param onConnected called after the first response headers of a request have been received, - * when the connection's endpoints are known + * @param onConnected called once, when the connection that carries the file's data is + * known: for a download, after the response headers of the last request + * have been received; for an upload, just before the first byte of the + * file is sent. At that time {@link #remoteAddress()} and + * {@link #localAddress()} describe that connection. An upload request + * that is redirected before any data is sent is not reported. If the + * file is sent again, because a server asked for a redirection only + * after it had received data, the later connection is not reported. */ public RemoteHttpTransferClient(CloseableHttpClient client, RemoteHttpDataTransferProtocolInfo info, Runnable onConnected) { @@ -368,6 +379,18 @@ private CloseableHttpResponse execute(HttpUriRequest request, HttpContext contex return _client.execute(request, context); } + private void reportConnection() { + if (_connectionReported.compareAndSet(false, true)) { + _onConnected.run(); + } else { + // The file is being sent again, on a connection other than the reported one. + SCITAGS_LOGGER.debug( + "scitags event=connection-unreported reason=data-sent-again remote={} local={}", + remoteAddress().map(Object::toString).orElse("-"), + localAddress().map(Object::toString).orElse("-")); + } + } + /** * The address of the remote endpoint of the most recent request, if known. */ @@ -551,8 +574,6 @@ private CloseableHttpResponse doGet(HttpContext context, long deadline) throws I HttpGet get = buildGetRequest(deadline); CloseableHttpResponse response = execute(get, context); - _onConnected.run(); - boolean isSuccessful = false; try { while (shouldRetry(response) && System.currentTimeMillis() < deadline) { @@ -560,9 +581,13 @@ private CloseableHttpResponse doGet(HttpContext context, long deadline) throws I response.close(); get = buildGetRequest(deadline); - response = execute(get, null); + // Use the same context, so that the endpoints describe this request's + // connection: the server may have closed the previous one. + response = execute(get, context); } + reportConnection(); + int statusCode = response.getStatusLine().getStatusCode(); String reason = response.getStatusLine().getReasonPhrase(); @@ -602,7 +627,6 @@ private void put(Supplier entity, Progress progress) redirectionCount > 0 ? REDIRECTED_REQUEST : INITIAL_REQUEST); try (CloseableHttpResponse response = execute(put, context)) { - _onConnected.run(); StatusLine status = response.getStatusLine(); switch (status.getStatusCode()) { case 200: /* OK (not actually a valid response from PUT) */ @@ -694,7 +718,16 @@ private HttpPut buildPutRequest(URI location, HttpEntity entity, Set mover, MoverInfoMessage moverInfoMessage) { protocolName(mover), mover.getFileAttributes().getPnfsId(), transferTag(mover)); - } else if (mover.isPassive() && mover.getRemoteEndpoint().isEmpty()) { - // Nobody connected to the mover: there was no flow and no start marker was sent. + } else if (hasNoDataConnection(mover)) { + // Nobody connected to the mover, or the mover failed before it connected to the + // remote party: there was no flow and no start marker was sent. SCITAGS_LOGGER.debug( "scitags lifecycle=end skip reason=no-data-connection protocol={} pnfsid={} transferTag={}", protocolName(mover), @@ -296,6 +297,16 @@ public void sendFinished(Mover mover, MoverInfoMessage moverInfoMessage) { _door.notify(mover.getPathToDoor(), finished); } + private static boolean hasNoDataConnection(Mover mover) { + if (mover.isPassive()) { + return mover.getRemoteEndpoint().isEmpty(); + } + if (mover.isActive()) { + return mover.getLocalEndpoint().isEmpty(); + } + return false; + } + private Optional deriveLocalEndpoint(InetSocketAddress remoteEndpoint) { if (remoteEndpoint == null) { return Optional.empty(); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java index 8c679832be1..7af43724267 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/Mover.java @@ -214,6 +214,15 @@ default boolean isPassive() { return false; } + /** + * Whether this mover connects to the remote party itself and reports that connection through + * {@link #getLocalEndpoint()}. Such a mover has no data connection, and therefore no network + * flow, until {@link #getLocalEndpoint()} returns a value. + */ + default boolean isActive() { + return false; + } + /** * Provide the expected total number of bytes transferred for this transfer, if known. Returns * null if this value is unknown. diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java index d5e7b48cb11..e6a301753ea 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java @@ -71,4 +71,23 @@ default List remoteConnections() { default Optional getLocalEndpoint() { return Optional.empty(); } + + /** + * Returns the {@link Optional} containing {@link InetSocketAddress} of the remote endpoint of + * the connection that carries the data, if the mover established that connection itself + * and knows it. This may differ from the address in the protocol info, for example if the + * remote server redirected the request. + */ + default Optional getRemoteEndpoint() { + return Optional.empty(); + } + + /** + * Whether this mover connects to the remote party itself and reports that connection through + * {@link #getLocalEndpoint()}. Such a mover has no data connection, and therefore no network + * flow, until {@link #getLocalEndpoint()} returns a value. + */ + default boolean isActive() { + return false; + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocolMover.java b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocolMover.java index 27d02d8689e..d7a017c1448 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocolMover.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocolMover.java @@ -83,4 +83,14 @@ public Long getBytesExpected() { public Optional getLocalEndpoint() { return _moverProtocol.getLocalEndpoint(); } + + @Override + public Optional getRemoteEndpoint() { + return _moverProtocol.getRemoteEndpoint(); + } + + @Override + public boolean isActive() { + return _moverProtocol.isActive(); + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java index f1c435da5c7..2bb3ce1daba 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java @@ -97,7 +97,8 @@ public class RemoteHttpDataTransferProtocol implements MoverProtocol, private volatile RemoteHttpTransferClient _transfer; - private InetSocketAddress _localEndpoint; + private volatile InetSocketAddress _localEndpoint; + private volatile InetSocketAddress _remoteEndpoint; private final TransferLifeCycle _transferLifeCycle; private Subject _subject; private boolean _startMarkerSent; @@ -164,8 +165,14 @@ public void runIO(FileAttributes attributes, RepositoryChannel channel, */ protected void afterTransfer() {} + /** + * Called once, when the connection that carries the file's data is known. The remote server + * may have redirected the request, and its name may resolve to several addresses, so both + * endpoints are taken from the connection rather than from the transfer's URL. + */ private void onConnected() { _localEndpoint = _transfer.localAddress().orElse(null); + _remoteEndpoint = _transfer.remoteAddress().orElse(null); startFlowMarker(); } @@ -239,7 +246,9 @@ private void startFlowMarker() { return; } - InetSocketAddress remoteEndpoint = ipInfo.getSocketAddress(); + InetSocketAddress remoteEndpoint = _remoteEndpoint != null + ? _remoteEndpoint + : ipInfo.getSocketAddress(); if (remoteEndpoint == null) { return; } @@ -253,4 +262,14 @@ private void startFlowMarker() { public Optional getLocalEndpoint() { return Optional.ofNullable(_localEndpoint); } + + @Override + public Optional getRemoteEndpoint() { + return Optional.ofNullable(_remoteEndpoint); + } + + @Override + public boolean isActive() { + return true; + } } diff --git a/modules/dcache/src/test/java/org/dcache/http/tpc/RemoteHttpTransferClientTest.java b/modules/dcache/src/test/java/org/dcache/http/tpc/RemoteHttpTransferClientTest.java index cfb2fe67bd0..8d8925068ed 100644 --- a/modules/dcache/src/test/java/org/dcache/http/tpc/RemoteHttpTransferClientTest.java +++ b/modules/dcache/src/test/java/org/dcache/http/tpc/RemoteHttpTransferClientTest.java @@ -25,16 +25,25 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import com.google.common.collect.ImmutableMap; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; import diskCacheV111.util.ThirdPartyTransferFailedCacheException; import diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.ArrayList; @@ -51,12 +60,14 @@ import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.protocol.HttpRequestExecutor; import org.dcache.util.Checksum; import org.dcache.util.ChecksumType; import org.junit.After; import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import org.slf4j.LoggerFactory; public class RemoteHttpTransferClientTest { @@ -270,6 +281,213 @@ public void shouldFollowRedirectOnUpload() throws Exception { assertThat(requests.get(1), equalTo("PUT /target")); } + @Test + public void shouldReportConnectionBeforeSendingUpload() throws Exception { + List> reported = Collections.synchronizedList(new ArrayList<>()); + AtomicInteger reportedWhenBodyReceived = new AtomicInteger(-1); + handle("/file", e -> { + if (e.getRequestMethod().equals("PUT")) { + e.getRequestBody().transferTo(uploaded); + reportedWhenBodyReceived.set(reported.size()); + respond(e, 201, null); + } else { + respond(e, 200, null, "Content-Length", String.valueOf(DATA.length)); + } + }); + AtomicReference transfer = new AtomicReference<>(); + transfer.set(new RemoteHttpTransferClient(client, info("/file", false), + () -> reported.add(transfer.get().remoteAddress()))); + + transfer.get().putAndVerify(RemoteHttpTransferClientTest::entity, NO_PROGRESS, + DATA.length, Optional.empty()); + + assertThat(uploaded.toByteArray(), equalTo(DATA)); + assertEquals("connection must be reported before the upload completes", + 1, reportedWhenBodyReceived.get()); + assertThat(reported.get(0).map(InetSocketAddress::getPort), + equalTo(Optional.of(server.getAddress().getPort()))); + } + + @Test + public void shouldReportOnlyDataConnectionWhenUploadIsRedirected() throws Exception { + handle("/target", e -> { + if (e.getRequestMethod().equals("PUT")) { + e.getRequestBody().transferTo(uploaded); + respond(e, 201, null); + } else { + respond(e, 200, null, "Content-Length", String.valueOf(DATA.length)); + } + }); + String target = "http://127.0.0.1:" + server.getAddress().getPort() + "/target"; + + // Like a dCache door: answers the request headers with a redirect, without sending + // "100 Continue", so the client never sends it any data. + try (ServerSocket door = new ServerSocket(0, 5, InetAddress.getLoopbackAddress())) { + Thread doorThread = new Thread(() -> redirectAllRequests(door, target)); + doorThread.setDaemon(true); + doorThread.start(); + + List> reported = + Collections.synchronizedList(new ArrayList<>()); + AtomicReference transfer = new AtomicReference<>(); + RemoteHttpDataTransferProtocolInfo info = new RemoteHttpDataTransferProtocolInfo( + "RemoteHttpDataTransfer", 1, 1, + new InetSocketAddress(InetAddress.getLoopbackAddress(), door.getLocalPort()), + "http://127.0.0.1:" + door.getLocalPort() + "/file", + false, ImmutableMap.of(), List.of()); + transfer.set(new RemoteHttpTransferClient(client, info, + () -> reported.add(transfer.get().remoteAddress()))); + + transfer.get().putAndVerify(RemoteHttpTransferClientTest::entity, NO_PROGRESS, + DATA.length, Optional.empty()); + + assertThat(uploaded.toByteArray(), equalTo(DATA)); + assertThat(reported.stream() + .map(a -> a.map(InetSocketAddress::getPort)).toList(), + contains(Optional.of(server.getAddress().getPort()))); + } + } + + @Test + public void shouldReportConnectionOfRetriedDownload() throws Exception { + // Like DPM while it calculates the checksum: "202 Accepted" until the file is ready. + // The first connection is closed, so the data arrives on another one. + List clientPorts = Collections.synchronizedList(new ArrayList<>()); + handle("/file", e -> { + clientPorts.add(e.getRemoteAddress().getPort()); + if (clientPorts.size() == 1) { + respond(e, 202, null, "Connection", "close"); + } else { + respond(e, 200, DATA); + } + }); + List> reported = Collections.synchronizedList(new ArrayList<>()); + AtomicReference transfer = new AtomicReference<>(); + transfer.set(new RemoteHttpTransferClient(client, info("/file", false), + () -> reported.add(transfer.get().localAddress()))); + ByteArrayOutputStream body = new ByteArrayOutputStream(); + + transfer.get().get(c -> {}, entity -> entity.writeTo(body), NO_PROGRESS, l -> {}); + + assertThat(body.toByteArray(), equalTo(DATA)); + assertEquals(2, clientPorts.size()); + Assume.assumeTrue("the retry used a new connection", + !clientPorts.get(0).equals(clientPorts.get(1))); + assertThat("only the connection that carries the data is reported", + reported.stream().map(a -> a.map(InetSocketAddress::getPort)).toList(), + contains(Optional.of(clientPorts.get(1)))); + } + + @Test + public void shouldReportConnectionOnlyOnceWhenUploadIsSentAgain() throws Exception { + handle("/target", e -> { + if (e.getRequestMethod().equals("PUT")) { + e.getRequestBody().transferTo(uploaded); + respond(e, 201, null); + } else { + respond(e, 200, null, "Content-Length", String.valueOf(DATA.length)); + } + }); + String target = "http://127.0.0.1:" + server.getAddress().getPort() + "/target"; + + // A server that does not answer "Expect: 100-continue": the client gives up waiting and + // sends the file, and only then learns that it has to send it somewhere else. + try (ServerSocket slowDoor = new ServerSocket(0, 5, InetAddress.getLoopbackAddress()); + CloseableHttpClient impatientClient = HttpClients.custom() + .setRequestExecutor(new HttpRequestExecutor(50)).build()) { + Thread doorThread = new Thread(() -> redirectAfterReceivingBody(slowDoor, target)); + doorThread.setDaemon(true); + doorThread.start(); + + List> reported = + Collections.synchronizedList(new ArrayList<>()); + AtomicInteger sent = new AtomicInteger(); + AtomicReference transfer = new AtomicReference<>(); + RemoteHttpDataTransferProtocolInfo info = new RemoteHttpDataTransferProtocolInfo( + "RemoteHttpDataTransfer", 1, 1, + new InetSocketAddress(InetAddress.getLoopbackAddress(), + slowDoor.getLocalPort()), + "http://127.0.0.1:" + slowDoor.getLocalPort() + "/file", + false, ImmutableMap.of(), List.of()); + transfer.set(new RemoteHttpTransferClient(impatientClient, info, + () -> reported.add(transfer.get().remoteAddress()))); + + Logger scitagsLogger = (Logger) LoggerFactory.getLogger("org.dcache.scitags"); + Level previousLevel = scitagsLogger.getLevel(); + ListAppender log = new ListAppender<>(); + log.start(); + scitagsLogger.addAppender(log); + scitagsLogger.setLevel(Level.DEBUG); + try { + transfer.get().putAndVerify(() -> new ByteArrayEntity(DATA) { + @Override + public void writeTo(OutputStream out) throws IOException { + sent.incrementAndGet(); + super.writeTo(out); + } + }, NO_PROGRESS, DATA.length, Optional.empty()); + } finally { + scitagsLogger.detachAppender(log); + scitagsLogger.setLevel(previousLevel); + } + + assertThat(uploaded.toByteArray(), equalTo(DATA)); + assertEquals("the file is sent to both servers", 2, sent.get()); + assertThat("the connection is reported once, when data is first sent", + reported.stream().map(a -> a.map(InetSocketAddress::getPort)).toList(), + contains(Optional.of(slowDoor.getLocalPort()))); + assertThat("the second connection is logged", + log.list.stream().map(ILoggingEvent::getFormattedMessage) + .filter(m -> m.contains("event=connection-unreported")).toList(), + contains(containsString("reason=data-sent-again remote=/127.0.0.1:" + + server.getAddress().getPort() + " "))); + } + } + + private static void redirectAfterReceivingBody(ServerSocket door, String location) { + while (!door.isClosed()) { + try (Socket socket = door.accept()) { + InputStream in = socket.getInputStream(); + StringBuilder headers = new StringBuilder(); + int c; + while (!headers.toString().endsWith("\r\n\r\n") && (c = in.read()) != -1) { + headers.append((char) c); + } + if (headers.toString().startsWith("PUT ")) { + // no "100 Continue": wait until the client sends the file anyway + in.readNBytes(DATA.length); + } + OutputStream out = socket.getOutputStream(); + out.write(("HTTP/1.1 307 Temporary Redirect\r\nLocation: " + location + + "\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.US_ASCII)); + out.flush(); + } catch (IOException e) { + // the test closed the socket, or the client went away + } + } + } + + private static void redirectAllRequests(ServerSocket door, String location) { + while (!door.isClosed()) { + try (Socket socket = door.accept()) { + BufferedReader in = new BufferedReader(new InputStreamReader( + socket.getInputStream(), StandardCharsets.US_ASCII)); + String line; + while ((line = in.readLine()) != null && !line.isEmpty()) { + // skip the request line and headers + } + OutputStream out = socket.getOutputStream(); + out.write(("HTTP/1.1 307 Temporary Redirect\r\nLocation: " + location + + "\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.US_ASCII)); + out.flush(); + } catch (IOException e) { + // the test closed the socket, or the client went away + } + } + } + @Test public void shouldDeleteRemoteFileWhenChecksumMismatches() throws Exception { handle("/file", e -> { diff --git a/modules/dcache/src/test/java/org/dcache/pool/classic/DefaultPostTransferServiceTest.java b/modules/dcache/src/test/java/org/dcache/pool/classic/DefaultPostTransferServiceTest.java new file mode 100644 index 00000000000..1577b267b32 --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/pool/classic/DefaultPostTransferServiceTest.java @@ -0,0 +1,103 @@ +package org.dcache.pool.classic; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import diskCacheV111.util.PnfsId; +import diskCacheV111.vehicles.IpProtocolInfo; +import diskCacheV111.vehicles.MoverInfoMessage; +import dmg.cells.nucleus.CellAddressCore; +import dmg.cells.nucleus.CellEndpoint; +import dmg.cells.nucleus.CellPath; +import java.net.InetSocketAddress; +import java.util.Optional; +import org.dcache.pool.movers.Mover; +import org.dcache.pool.movers.TransferLifeCycle; +import org.dcache.vehicles.FileAttributes; +import org.junit.Before; +import org.junit.Test; + +public class DefaultPostTransferServiceTest { + + private static final PnfsId PNFSID = new PnfsId("000000000000000000000000000000000001"); + private static final InetSocketAddress REMOTE_FROM_DOOR = + new InetSocketAddress("192.0.2.10", 443); + private static final InetSocketAddress REMOTE_CONNECTED = + new InetSocketAddress("192.0.2.20", 24100); + private static final InetSocketAddress LOCAL = new InetSocketAddress("198.51.100.5", 40000); + + private DefaultPostTransferService service; + private TransferLifeCycle transferLifeCycle; + private Mover mover; + private MoverInfoMessage moverInfo; + + @Before + @SuppressWarnings("unchecked") + public void setUp() { + transferLifeCycle = mock(TransferLifeCycle.class); + + service = new DefaultPostTransferService(); + service.setPoolName("pool_a"); + service.setTransferLifeCycle(transferLifeCycle); + service.setCellEndpoint(mock(CellEndpoint.class)); + service.init(); + + IpProtocolInfo protocolInfo = mock(IpProtocolInfo.class); + when(protocolInfo.getProtocol()).thenReturn("RemoteHttpsDataTransfer"); + when(protocolInfo.getSocketAddress()).thenReturn(REMOTE_FROM_DOOR); + + mover = mock(Mover.class); + when(mover.getProtocolInfo()).thenReturn(protocolInfo); + when(mover.getFileAttributes()).thenReturn(FileAttributes.ofPnfsId(PNFSID)); + when(mover.getPathToDoor()).thenReturn(new CellPath("door")); + when(mover.getLocalEndpoint()).thenReturn(Optional.empty()); + when(mover.getRemoteEndpoint()).thenReturn(Optional.empty()); + + moverInfo = new MoverInfoMessage(new CellAddressCore("pool_a", "poolDomain"), PNFSID); + } + + @Test + public void shouldSendNoEndMarkerForActiveMoverThatNeverConnected() { + // e.g. an HTTP-TPC pull that failed on the DNS lookup of the source + when(mover.isActive()).thenReturn(true); + + service.sendFinished(mover, moverInfo); + + verify(transferLifeCycle, never()).onEnd(any(), any(), any()); + } + + @Test + public void shouldSendEndMarkerForConnectionOfActiveMover() { + when(mover.isActive()).thenReturn(true); + when(mover.getLocalEndpoint()).thenReturn(Optional.of(LOCAL)); + when(mover.getRemoteEndpoint()).thenReturn(Optional.of(REMOTE_CONNECTED)); + + service.sendFinished(mover, moverInfo); + + verify(transferLifeCycle).onEnd(REMOTE_CONNECTED, LOCAL, moverInfo); + } + + @Test + public void shouldSendNoEndMarkerForPassiveMoverNobodyConnectedTo() { + when(mover.isPassive()).thenReturn(true); + when(mover.getLocalEndpoint()).thenReturn(Optional.of(LOCAL)); + + service.sendFinished(mover, moverInfo); + + verify(transferLifeCycle, never()).onEnd(any(), any(), any()); + } + + @Test + public void shouldSendEndMarkerForPassiveMoverWithClient() { + when(mover.isPassive()).thenReturn(true); + when(mover.getLocalEndpoint()).thenReturn(Optional.of(LOCAL)); + when(mover.getRemoteEndpoint()).thenReturn(Optional.of(REMOTE_CONNECTED)); + + service.sendFinished(mover, moverInfo); + + verify(transferLifeCycle).onEnd(REMOTE_CONNECTED, LOCAL, moverInfo); + } +}