Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import diskCacheV111.util.ThirdPartyTransferFailedCacheException;
import diskCacheV111.vehicles.RemoteHttpDataTransferProtocolInfo;
import java.io.IOException;
import java.io.OutputStream;
import java.net.ConnectException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
Expand All @@ -48,6 +49,7 @@
import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.regex.Pattern;
Expand All @@ -70,6 +72,7 @@
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.conn.ConnectionShutdownException;
import org.apache.http.protocol.HttpContext;
Expand Down Expand Up @@ -135,6 +138,7 @@ private enum HeaderFlags {
= EnumSet.noneOf(HeaderFlags.class);

private static final Logger LOGGER = LoggerFactory.getLogger(RemoteHttpTransferClient.class);
private static final Logger SCITAGS_LOGGER = LoggerFactory.getLogger("org.dcache.scitags");

/**
* Maximum time to wait when establishing a connection.
Expand Down Expand Up @@ -202,6 +206,7 @@ private enum HeaderFlags {
private final CloseableHttpClient _client;
private final RemoteHttpDataTransferProtocolInfo _info;
private final Runnable _onConnected;
private final AtomicBoolean _connectionReported = new AtomicBoolean();

@GuardedBy("this")
private HttpClientContext _context;
Expand All @@ -217,8 +222,14 @@ private enum HeaderFlags {
/**
* @param client the HTTP client used to make all requests
* @param info describes the remote resource, credentials and headers
* @param onConnected called after the first response headers of a request have been received,
* when the connection's endpoints are known
* @param onConnected called once, when the connection that carries the file's data is
* known: for a download, after the response headers of the last request
* have been received; for an upload, just before the first byte of the
* file is sent. At that time {@link #remoteAddress()} and
* {@link #localAddress()} describe that connection. An upload request
* that is redirected before any data is sent is not reported. If the
* file is sent again, because a server asked for a redirection only
* after it had received data, the later connection is not reported.
*/
public RemoteHttpTransferClient(CloseableHttpClient client,
RemoteHttpDataTransferProtocolInfo info, Runnable onConnected) {
Expand Down Expand Up @@ -368,6 +379,18 @@ private CloseableHttpResponse execute(HttpUriRequest request, HttpContext contex
return _client.execute(request, context);
}

private void reportConnection() {
if (_connectionReported.compareAndSet(false, true)) {
_onConnected.run();
} else {
// The file is being sent again, on a connection other than the reported one.
SCITAGS_LOGGER.debug(
"scitags event=connection-unreported reason=data-sent-again remote={} local={}",
remoteAddress().map(Object::toString).orElse("-"),
localAddress().map(Object::toString).orElse("-"));
}
}

/**
* The address of the remote endpoint of the most recent request, if known.
*/
Expand Down Expand Up @@ -551,18 +574,20 @@ private CloseableHttpResponse doGet(HttpContext context, long deadline) throws I
HttpGet get = buildGetRequest(deadline);
CloseableHttpResponse response = execute(get, context);

_onConnected.run();

boolean isSuccessful = false;
try {
while (shouldRetry(response) && System.currentTimeMillis() < deadline) {
Thread.sleep(DELAY_BETWEEN_REQUESTS);

response.close();
get = buildGetRequest(deadline);
response = execute(get, null);
// Use the same context, so that the endpoints describe this request's
// connection: the server may have closed the previous one.
response = execute(get, context);
}

reportConnection();

int statusCode = response.getStatusLine().getStatusCode();
String reason = response.getStatusLine().getReasonPhrase();

Expand Down Expand Up @@ -602,7 +627,6 @@ private void put(Supplier<HttpEntity> entity, Progress progress)
redirectionCount > 0 ? REDIRECTED_REQUEST : INITIAL_REQUEST);

try (CloseableHttpResponse response = execute(put, context)) {
_onConnected.run();
StatusLine status = response.getStatusLine();
switch (status.getStatusCode()) {
case 200: /* OK (not actually a valid response from PUT) */
Expand Down Expand Up @@ -694,7 +718,16 @@ private HttpPut buildPutRequest(URI location, HttpEntity entity, Set<HeaderFlags
.setSocketTimeout(0)
.build());
addHeadersToRequest(put, flags);
put.setEntity(entity);
put.setEntity(new HttpEntityWrapper(entity) {
@Override
public void writeTo(OutputStream out) throws IOException {
// The request is only complete once all data is sent, which is too late to
// learn about the connection. With expect-continue, the entity is not
// written to a server that redirects the request.
reportConnection();
super.writeTo(out);
}
});

long size = entity.getContentLength();
if (size != -1) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,9 @@ public void sendFinished(Mover<?> mover, MoverInfoMessage moverInfoMessage) {
protocolName(mover),
mover.getFileAttributes().getPnfsId(),
transferTag(mover));
} else if (mover.isPassive() && mover.getRemoteEndpoint().isEmpty()) {
// Nobody connected to the mover: there was no flow and no start marker was sent.
} else if (hasNoDataConnection(mover)) {
// Nobody connected to the mover, or the mover failed before it connected to the
// remote party: there was no flow and no start marker was sent.
SCITAGS_LOGGER.debug(
"scitags lifecycle=end skip reason=no-data-connection protocol={} pnfsid={} transferTag={}",
protocolName(mover),
Expand Down Expand Up @@ -296,6 +297,16 @@ public void sendFinished(Mover<?> mover, MoverInfoMessage moverInfoMessage) {
_door.notify(mover.getPathToDoor(), finished);
}

private static boolean hasNoDataConnection(Mover<?> mover) {
if (mover.isPassive()) {
return mover.getRemoteEndpoint().isEmpty();
}
if (mover.isActive()) {
return mover.getLocalEndpoint().isEmpty();
}
return false;
}

private Optional<InetSocketAddress> deriveLocalEndpoint(InetSocketAddress remoteEndpoint) {
if (remoteEndpoint == null) {
return Optional.empty();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,15 @@ default boolean isPassive() {
return false;
}

/**
* Whether this mover connects to the remote party itself and reports that connection through
* {@link #getLocalEndpoint()}. Such a mover has no data connection, and therefore no network
* flow, until {@link #getLocalEndpoint()} returns a value.
*/
default boolean isActive() {
return false;
}

/**
* Provide the expected total number of bytes transferred for this transfer, if known. Returns
* null if this value is unknown.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,23 @@ default List<InetSocketAddress> remoteConnections() {
default Optional<InetSocketAddress> getLocalEndpoint() {
return Optional.empty();
}

/**
* Returns the {@link Optional} containing {@link InetSocketAddress} of the remote endpoint of
* the connection that carries the data, if the mover established that connection itself
* and knows it. This may differ from the address in the protocol info, for example if the
* remote server redirected the request.
*/
default Optional<InetSocketAddress> getRemoteEndpoint() {
return Optional.empty();
}

/**
* Whether this mover connects to the remote party itself and reports that connection through
* {@link #getLocalEndpoint()}. Such a mover has no data connection, and therefore no network
* flow, until {@link #getLocalEndpoint()} returns a value.
*/
default boolean isActive() {
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -83,4 +83,14 @@ public Long getBytesExpected() {
public Optional<InetSocketAddress> getLocalEndpoint() {
return _moverProtocol.getLocalEndpoint();
}

@Override
public Optional<InetSocketAddress> getRemoteEndpoint() {
return _moverProtocol.getRemoteEndpoint();
}

@Override
public boolean isActive() {
return _moverProtocol.isActive();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ public class RemoteHttpDataTransferProtocol implements MoverProtocol,

private volatile RemoteHttpTransferClient _transfer;

private InetSocketAddress _localEndpoint;
private volatile InetSocketAddress _localEndpoint;
private volatile InetSocketAddress _remoteEndpoint;
private final TransferLifeCycle _transferLifeCycle;
private Subject _subject;
private boolean _startMarkerSent;
Expand Down Expand Up @@ -164,8 +165,14 @@ public void runIO(FileAttributes attributes, RepositoryChannel channel,
*/
protected void afterTransfer() {}

/**
* Called once, when the connection that carries the file's data is known. The remote server
* may have redirected the request, and its name may resolve to several addresses, so both
* endpoints are taken from the connection rather than from the transfer's URL.
*/
private void onConnected() {
_localEndpoint = _transfer.localAddress().orElse(null);
_remoteEndpoint = _transfer.remoteAddress().orElse(null);
startFlowMarker();
}

Expand Down Expand Up @@ -239,7 +246,9 @@ private void startFlowMarker() {
return;
}

InetSocketAddress remoteEndpoint = ipInfo.getSocketAddress();
InetSocketAddress remoteEndpoint = _remoteEndpoint != null
? _remoteEndpoint
: ipInfo.getSocketAddress();
if (remoteEndpoint == null) {
return;
}
Expand All @@ -253,4 +262,14 @@ private void startFlowMarker() {
public Optional<InetSocketAddress> getLocalEndpoint() {
return Optional.ofNullable(_localEndpoint);
}

@Override
public Optional<InetSocketAddress> getRemoteEndpoint() {
return Optional.ofNullable(_remoteEndpoint);
}

@Override
public boolean isActive() {
return true;
}
}
Loading
Loading