diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index 0641be77e..49d74b363 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -96,6 +96,19 @@ public class Constants { */ static final int SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT = 5000; + /** + * The most time closing the Usage Sharing Element may take, from its + * start, in milliseconds. + */ + static final int SHARE_USAGE_CLOSE_LIMIT_MILLIS = 5000; + + /** + * The part of the close limit, in milliseconds, kept for what closing + * does after it stops waiting for the data to be sent: discarding the + * rest, logging the warning and freeing the resources. + */ + static final int SHARE_USAGE_CLOSE_RESERVE_MILLIS = 500; + /** * Default lost data count for share usage before it wraps */ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 48d28f657..4c13ed897 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -789,8 +789,31 @@ public DataFactory getDataFactory() { return null; } + /** + * The time closing may still wait for the data to be sent, so that the + * whole of closing, measured from its start, keeps within the close + * limit with the reserve left for what follows the wait. + * @param closeStartNanos {@link System#nanoTime()} when closing started + * @return milliseconds left to wait, never less than 0 + */ + private static long closeWaitLeftMillis(long closeStartNanos) { + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - closeStartNanos); + return Math.max(0, closeWaitMillis() - elapsedMillis); + } + + /** + * The longest closing waits for the data to be sent, from its start. + * @return milliseconds + */ + private static long closeWaitMillis() { + return Constants.SHARE_USAGE_CLOSE_LIMIT_MILLIS - + Constants.SHARE_USAGE_CLOSE_RESERVE_MILLIS; + } + @Override protected void managedResourcesCleanup() { + long closeStartNanos = System.nanoTime(); // Tell any consumer - one that is already running as well as the final // one queued below - that the queue has to be drained completely rather // than stopping once fewer than minEntriesPerMessage entries are left. @@ -809,9 +832,12 @@ protected void managedResourcesCleanup() { submitSendData(); try { executor.shutdown(); - if (!executor.awaitTermination(Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT, TimeUnit.MILLISECONDS)){ + if (!executor.awaitTermination( + closeWaitLeftMillis(closeStartNanos), + TimeUnit.MILLISECONDS)) { // Sending stops rather than hold up the process that closes - // the element, and the data left is discarded. + // the element, and the data left is discarded, all within + // the close limit. discarding = true; int discarded = evidenceCollection.drainTo(new ArrayList()); @@ -820,7 +846,7 @@ protected void managedResourcesCleanup() { "not finished {} milliseconds after usage sharing " + "started closing.", discarded, - Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT); + closeWaitMillis()); executor.shutdownNow(); } } catch (InterruptedException interruptedException) { diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java index cf6507034..a232bcd25 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java @@ -32,6 +32,7 @@ import java.io.ByteArrayOutputStream; import java.io.OutputStream; import java.lang.reflect.Proxy; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -45,11 +46,12 @@ import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Closing waits up to 5 seconds for the data waiting to be sent. If it has - * not all been sent by then, sending stops, the rest is discarded and a + * Closing finishes within 5 seconds of its start. If the data waiting has + * not all been sent in time, sending stops, the rest is discarded and a * warning gives the number discarded. See Cleanup in the usage sharing * element specification. */ @@ -65,6 +67,22 @@ public class ShareUsageCloseTimeoutTests { */ private static final int ENTRIES = 10; + /** + * The 5 seconds the specification bounds the whole of cleanup at. The + * usage + * sharing element specification says "Cleanup SHOULD complete within + * 5 seconds, so that closing the element does not hold up the process + * that closes it." + *

+ * The limit is the specification's, not a measurement. Closing must take + * less, holding the wait for the data to be sent, discarding the rest, + * the warning and freeing the resources. The element waits 4,500 ms, its + * 5,000 ms budget less a 500 ms reserve, then does the rest, so it + * closes in a little over 4,500 ms. Code that waits the full 5 seconds + * before doing the rest always takes at least this long, so it fails. + */ + private static final long CLOSE_LIMIT_MILLIS = 5000; + /** * An upload whose first answer does not come until the test releases * it, as with a remote service that has stopped answering. Interrupting @@ -107,23 +125,7 @@ public void ShareUsageElement_CloseTimesOut_StopsSendingAndDiscards() List warnings = new CopyOnWriteArrayList<>(); StalledUploader uploader = new StalledUploader(); Pipeline pipeline = new PipelineBuilder().build(); - ShareUsageElement element = new ShareUsageElement( - recordingLogger(warnings), - 1, - 1, - 100, - 100, - 100, - 1, - true, - "http://localhost/usage", - new ArrayList<>(), - new ArrayList<>(), - new ArrayList<>(), - DEFAULT_SESSION_COOKIE_NAME, - flowData -> true); - element.addPipeline(pipeline); - element.dataUploader = uploader; + ShareUsageElement element = element(warnings, pipeline, uploader); try { for (int i = 1; i <= ENTRIES; i++) { process(pipeline, element, "10.0.0." + i); @@ -158,6 +160,82 @@ public void ShareUsageElement_CloseTimesOut_StopsSendingAndDiscards() } } + /** + * Closing while the remote service has stopped answering finishes within + * 5 seconds of its start, the whole of cleanup included, while the + * upload in progress is still waiting for its answer. + */ + @Test + public void ShareUsageElement_ServiceStopsAnswering_CloseWithin5Seconds() + throws Exception { + StalledUploader uploader = new StalledUploader(); + Pipeline pipeline = new PipelineBuilder().build(); + ShareUsageElement element = element( + new CopyOnWriteArrayList<>(), + pipeline, + uploader); + try { + for (int i = 1; i <= ENTRIES; i++) { + process(pipeline, element, "10.0.0." + i); + } + assertTrue( + uploader.sending.await(WAIT_LIMIT_MILLIS, TimeUnit.MILLISECONDS), + "No send was started."); + + long[] tookMillis = new long[1]; + assertTimeoutPreemptively( + Duration.ofMillis(WAIT_LIMIT_MILLIS), + () -> { + long start = System.nanoTime(); + element.close(); + tookMillis[0] = TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - start); + }, + "Closing waited for the remote service to answer."); + assertTrue(tookMillis[0] < CLOSE_LIMIT_MILLIS, + "Closing took " + tookMillis[0] + " ms, not within the " + + CLOSE_LIMIT_MILLIS + " ms the specification puts on " + + "cleanup."); + assertEquals(1, uploader.release.getCount(), + "The upload in progress was answered before closing ended."); + } + finally { + uploader.release.countDown(); + if (element.executor.isShutdown() == false) { + element.close(); + } + pipeline.close(); + } + } + + /** + * An element that sends each entry on its own, through the uploader + * given, logging its warnings to the list given. + */ + private static ShareUsageElement element( + List warnings, + Pipeline pipeline, + DataUploader uploader) { + ShareUsageElement element = new ShareUsageElement( + recordingLogger(warnings), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + flowData -> true); + element.addPipeline(pipeline); + element.dataUploader = uploader; + return element; + } + private static void process( Pipeline pipeline, ShareUsageElement element,