From 97af6d57ae0f522d2ffb3002aa0b8cdf28b6e93a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:02:37 +0000 Subject: [PATCH 01/35] Test: a cache neither stores nor serves a cloud response with a top-level fodid caching.md line 144 at 706de1f0: "A cache MUST NOT store a result that holds such a value, and MUST NOT serve one that it finds." (CO-21) --- .../CloudRequestEngineFodIdCacheTests.java | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java new file mode 100644 index 000000000..13358278f --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java @@ -0,0 +1,176 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.caching.FlowCache; +import org.json.JSONArray; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * A response that holds a value created for the one request that asked for + * it, such as a 51Did, is neither stored in nor served from the cache. + * See the Values created for one request section of + * caching.md: + * "A cache MUST NOT store a result that holds such a value, and MUST NOT + * serve one that it finds." + */ +public class CloudRequestEngineFodIdCacheTests + extends CloudRequestEngineTestsBase { + + public CloudRequestEngineFodIdCacheTests() throws MalformedURLException { + super(); + } + + /** + * Process the same Evidence twice with an engine that has a cache, the + * service answering each request with the next response in the list. + * @param responses the responses, one a request + * @return the JSON each request's Flow Data holds, in order + */ + private List processTwice(String... responses) throws Exception { + evidenceKeysResponse = + new JSONArray(Arrays.asList("query.User-Agent")).toString(); + configureMockedClient(); + final List remaining = new ArrayList<>(Arrays.asList(responses)); + doAnswer(invocation -> { + java.net.HttpURLConnection connection = invocation.getArgument(0); + String path = connection.getURL().getPath(); + if (path.endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (path.endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + return remaining.remove(0); + }).when(httpClient).postData(any(), anyMap(), (byte[]) any()); + + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .build(); + List answers = new ArrayList<>(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (int i = 0; i < 2; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + answers.add(data.get(CloudRequestData.class) + .getJsonResponse()); + } + } + } + return answers; + } + + /** + * A response whose JSON has a top-level member named fodid, in any case, + * is not stored, so the second identical request goes to the cloud + * service and gets its own 51Did. + */ + @ParameterizedTest + @ValueSource(strings = { "fodid", "FodId", "FODID" }) + public void process_FodIdResponseIsNotStored(String name) throws Exception { + String first = "{'device':{'value':'1'},'" + name + "':'first'}"; + String second = "{'device':{'value':'1'},'" + name + "':'second'}"; + + List answers = processTwice(first, second); + + assertEquals(Arrays.asList(first, second), answers, + "A response with a 51Did was served from the cache."); + verify(httpClient, times(2)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + any()); + } + + /** + * Every other response is cached as before, including one whose fodid + * member is not at the top level. + */ + @Test + public void process_ResponseWithoutTopLevelFodIdIsCached() + throws Exception { + String first = "{'device':{'value':'1','fodid':'nested'}}"; + + List answers = processTwice(first); + + assertEquals(Arrays.asList(first, first), answers); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + any()); + } + + /** + * A result with a 51Did that the cache already holds, such as one an + * earlier version stored, is not served. The request goes to the cloud + * service instead. + */ + @Test + public void process_FodIdResultFoundInTheCacheIsNotServed() + throws Exception { + evidenceKeysResponse = + new JSONArray(Arrays.asList("query.User-Agent")).toString(); + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .build(); + FlowCache cache = mock(FlowCache.class); + engine.setCache(cache); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build(); + FlowData earlier = pipeline.createFlowData(); + FlowData data = pipeline.createFlowData()) { + CloudRequestData cached = new CloudRequestData( + loggerFactory.getLogger("test"), earlier, engine); + cached.setJsonResponse( + "{'device':{'value':'1'},'fodid':'stale'}"); + when(cache.get(any())).thenReturn(cached); + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + assertEquals(jsonResponse, + data.get(CloudRequestData.class).getJsonResponse(), + "A result with a 51Did found in the cache was served."); + } + verify(cache, never()).put(any(), any()); + } +} From d26dd968d645142ce42d56be72d475c014127b80 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:08:00 +0000 Subject: [PATCH 02/35] Test: only the result with a 51Did found in the cache is kept out of it The engine stores the fresh result it gets in its place. caching.md line 144 at 706de1f0 (CO-21). --- .../flowelements/CloudRequestEngineFodIdCacheTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java index 13358278f..8d17244a5 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java @@ -170,7 +170,7 @@ public void process_FodIdResultFoundInTheCacheIsNotServed() assertEquals(jsonResponse, data.get(CloudRequestData.class).getJsonResponse(), "A result with a 51Did found in the cache was served."); + verify(cache, never()).put(any(), same(cached)); } - verify(cache, never()).put(any(), any()); } } From 277e08da64069749448807013f9c024bf7fa517b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:11:28 +0000 Subject: [PATCH 03/35] Fix: keep a cloud response with a top-level fodid out of the result cache AspectEngineBase asks isCacheable before it stores a result and before it serves one it finds. The Cloud Request Engine answers false for a response whose JSON has a top-level member named fodid, in any case. caching.md line 144 at 706de1f0: "A cache MUST NOT store a result that holds such a value, and MUST NOT serve one that it finds." (CO-21) --- .../CloudRequestEngineDefault.java | 35 +++++++++++++++++++ .../flowelements/AspectEngineBase.java | 22 ++++++++++-- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index b95d7ec5d..4b68daac0 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -457,6 +457,41 @@ private void completeDiscovery() throws IOException { } } + /** + * A response whose JSON has a top-level member named fodid, in any case, + * holds a 51Did created for the one request, so it is neither stored in + * nor served from the cache, and the request goes to the cloud service + * instead. See the + * caching specification: + * "A cache MUST NOT store a result that holds such a value, and MUST NOT + * serve one that it finds." A response that cannot be read as a JSON + * object is not cached either, as it cannot be checked. + * @param data the result to check + * @return false if the response has a top-level fodid member + */ + @Override + protected boolean isCacheable(CloudRequestData data) { + String response = data.getJsonResponse(); + if (response == null) { + return false; + } + try { + for (String key : new JSONObject(response).keySet()) { + if (FODID.equalsIgnoreCase(key)) { + return false; + } + } + return true; + } catch (JSONException e) { + return false; + } + } + + /** + * The top-level member of a cloud response that holds a 51Did. + */ + private static final String FODID = "fodid"; + @Override protected void processEngine(FlowData data, CloudRequestData aspectData) throws IOException { ((CloudRequestDataInternal)aspectData).setProcessStarted(true); diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java index 89a545be1..e95167c6e 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java @@ -135,6 +135,23 @@ protected final void processInternal(FlowData flowData) throws Exception { processWithCache(flowData); } + /** + * Whether a result may be stored in the cache, and served when it is + * found there. A result that holds a value created for the one request + * that asked for it, such as a 51Did, must be neither. See the + * caching specification: + * "A cache MUST NOT store a result that holds such a value, and MUST NOT + * serve one that it finds." + *

+ * With lazy loading the result is checked before it has been processed. + * @param data the result to check + * @return true, unless an engine overrides this to keep a result out of + * the cache + */ + protected boolean isCacheable(TData data) { + return true; + } + /** * Private method that checks if the result is already in the cache or not. * If it is then the result is added to 'data', if not then @@ -151,7 +168,8 @@ private void processWithCache(final FlowData flowData) throws Exception { Object cacheResultsObject = cache.get(flowData); if (cacheResultsObject != null && getTypedDataKey().getType().isAssignableFrom( - cacheResultsObject.getClass())) { + cacheResultsObject.getClass()) && + isCacheable((TData)cacheResultsObject)) { // This is checked. //noinspection unchecked cacheResult = (TData)cacheResultsObject; @@ -190,7 +208,7 @@ public Void call() throws Exception { } // If there is a cache then add the result // of processing to the cache. - if (cache != null) { + if (cache != null && isCacheable(aspectData)) { cache.put(flowData, flowData.getFromElement(this)); } } else { From 2acc1b9bd55234e62a805d19c4de0cffd00f2c0f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:12:21 +0000 Subject: [PATCH 04/35] Test: the Pipeline interface exposes isSuppressProcessExceptions exception-handling.md line 42 at 706de1f0: "This property MUST be exposed as a public property as part of the `Pipeline` public interface as `IsSuppressProcessExceptions` - for Pipeline wrappers such as `WebPipeline` to be able to use it as well to make a decision on whether the exceptions should be suppressed and logged during processing." (CO-26) --- ...ipelineSuppressProcessExceptionsTests.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineSuppressProcessExceptionsTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineSuppressProcessExceptionsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineSuppressProcessExceptionsTests.java new file mode 100644 index 000000000..756802f18 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineSuppressProcessExceptionsTests.java @@ -0,0 +1,77 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The suppress process exceptions setting can be read through the public + * Pipeline interface, so that a wrapper such as the web integration can + * decide from it. See + * exception-handling.md: + * "This property MUST be exposed as a public property as part of the + * `Pipeline` public interface as `IsSuppressProcessExceptions`". + */ +public class PipelineSuppressProcessExceptionsTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * The Pipeline interface declares a public isSuppressProcessExceptions + * that returns a boolean. + */ + @Test + public void pipelineInterface_DeclaresIsSuppressProcessExceptions() + throws Exception { + Method method = Pipeline.class.getMethod("isSuppressProcessExceptions"); + assertEquals(boolean.class, method.getReturnType()); + assertTrue(Modifier.isPublic(method.getModifiers())); + } + + /** + * The value read through the interface is the one the builder set, for + * both settings. + */ + @Test + public void isSuppressProcessExceptions_GivesTheBuilderSetting() + throws Exception { + for (boolean suppress : new boolean[] { true, false }) { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(suppress) + .build()) { + Method method = + Pipeline.class.getMethod("isSuppressProcessExceptions"); + assertEquals(suppress, method.invoke(pipeline)); + } + } + } +} From 7d911838b7578a8b029e9a427e619873fe87fe8b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:14:30 +0000 Subject: [PATCH 05/35] Fix: expose isSuppressProcessExceptions on the Pipeline interface PipelineDefault gives the setting the builder made. The interface's default method gives the builder's default, so an implementation of Pipeline written before this still compiles. exception-handling.md line 42 at 706de1f0: "This property MUST be exposed as a public property as part of the `Pipeline` public interface as `IsSuppressProcessExceptions`" (CO-26) --- .../pipeline/core/flowelements/Pipeline.java | 18 ++++++++++++++++++ .../core/flowelements/PipelineDefault.java | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/Pipeline.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/Pipeline.java index 098f54fee..f36061939 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/Pipeline.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/Pipeline.java @@ -61,6 +61,24 @@ public interface Pipeline extends AutoCloseable { */ boolean isConcurrent(); + /** + * True if exceptions that happen while processing are recorded in + * {@link FlowData#getErrors()} and logged, rather than thrown once + * processing is complete. A wrapper of the Pipeline, such as the web + * integration, can read it to decide whether to suppress and log an + * exception of its own. See the + * exception handling specification. + *

+ * The default gives the builder's default, so that an implementation + * written before this method was added still compiles. Override it to + * give the real setting. + * @return true if exceptions during processing are suppressed + */ + default boolean isSuppressProcessExceptions() { + return fiftyone.pipeline.core.Constants + .DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS; + } + /** * Indicates whether the Pipeline has been closed using the {@link #close()} * method, either explicitly or as a result of a 'try with resource'. diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java index 302b51d94..5f0eb1a26 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java @@ -212,6 +212,11 @@ public boolean isConcurrent() { return isConcurrent; } + @Override + public boolean isSuppressProcessExceptions() { + return suppressProcessExceptions; + } + @Override public boolean isClosed() { return isClosed; From 4486b4f8d0269d5065fc27a51b60be2df7eae4f1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:19:09 +0000 Subject: [PATCH 06/35] Test: only one thread at a time reads the Device Detection native results Covers getValues, the property getters, the match metrics and the device id, each started while another read is inside the native results. thread-safety.md line 57 at 706de1f0: "Reading an Element Data instance once the Flow Element that writes it has finished MUST therefore be safe from several threads at once." (CO-48) thread-safety.md line 61 at 706de1f0: "Such Element Data MUST let only one thread at a time read from its results, for example by holding a lock for each read" (CO-49) --- .../DeviceDataHashThreadSafetyTests.java | 242 ++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashThreadSafetyTests.java diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashThreadSafetyTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashThreadSafetyTests.java new file mode 100644 index 000000000..f56b54751 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashThreadSafetyTests.java @@ -0,0 +1,242 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ResultsHashSwig; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.MissingPropertyService; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.Logger; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Reads of finished Device Detection on-premise results are safe from + * several threads, because only one thread at a time reads from the native + * results. See + * thread-safety.md line 57: + * "Reading an Element Data instance once the Flow Element that writes it + * has finished MUST therefore be safe from several threads at once." and + * line 61: + * "Such Element Data MUST let only one thread at a time read from its + * results". + *

+ * The native results are a mock. The first thread's read is held inside a + * native call until the test lets it go. The second thread must then wait + * to enter the lock, which the test sees from the thread's state, and must + * not reach the native results. Nothing here depends on how long anything + * takes; the deadline only stops a broken run from hanging. + */ +public class DeviceDataHashThreadSafetyTests { + + private static final long DEADLINE_MILLIS = 60_000; + + /** + * Native results whose every read counts the threads inside it, and + * holds the first thread's first read until released. + */ + private static class Gate implements Answer { + final AtomicInteger inside = new AtomicInteger(); + final AtomicInteger most = new AtomicInteger(); + final CountDownLatch firstInside = new CountDownLatch(1); + final CountDownLatch release = new CountDownLatch(1); + volatile Thread first; + + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + String name = invocation.getMethod().getName(); + if (name.equals("close") || name.equals("delete") || + name.equals("finalize")) { + return null; + } + int now = inside.incrementAndGet(); + most.accumulateAndGet(now, Math::max); + try { + if (Thread.currentThread() == first && + firstInside.getCount() > 0) { + firstInside.countDown(); + assertTrue(release.await( + DEADLINE_MILLIS, TimeUnit.MILLISECONDS)); + } + return valueFor(invocation); + } finally { + inside.decrementAndGet(); + } + } + + private Object valueFor(InvocationOnMock invocation) { + Class type = invocation.getMethod().getReturnType(); + if (type == boolean.class) { + Object arg = invocation.getArguments().length > 0 ? + invocation.getArgument(0) : null; + String property = arg instanceof byte[] ? + new String((byte[]) arg, StandardCharsets.UTF_8) : + String.valueOf(arg); + return property.toLowerCase().contains("ismobile"); + } + if (type == int.class) { + return 0; + } + if (type == String.class) { + return "0-0-0-0"; + } + if (type == void.class) { + return null; + } + // A native value holder, such as VectorStringValuesSwig, with + // no value. + return mock(type); + } + } + + private DeviceDataHashDefault newData(Gate gate) { + FlowData flowData = mock(FlowData.class); + when(flowData.getPipeline()).thenReturn(mock(Pipeline.class)); + DeviceDataHashDefault data = new DeviceDataHashDefault( + mock(Logger.class), + flowData, + mock(DeviceDetectionHashEngine.class), + mock(MissingPropertyService.class)); + data.setResults(mock(ResultsHashSwig.class, gate)); + return data; + } + + /** + * Start the first read and hold it inside the native results, then start + * the second read and wait until it either waits for the lock or reaches + * the native results. Then let both finish. + */ + private void assertOneReaderAtATime( + Consumer firstRead, + Consumer secondRead) throws Exception { + Gate gate = new Gate(); + DeviceDataHashDefault data = newData(gate); + AtomicReference failure = new AtomicReference<>(); + + Thread first = new Thread(() -> run(firstRead, data, failure)); + gate.first = first; + first.start(); + assertTrue("The first read did not reach the native results.", + gate.firstInside.await(DEADLINE_MILLIS, TimeUnit.MILLISECONDS)); + + Thread second = new Thread(() -> run(secondRead, data, failure)); + second.start(); + long deadline = System.currentTimeMillis() + DEADLINE_MILLIS; + while (gate.most.get() < 2 && + second.isAlive() && + waitingForTheLock(second) == false) { + assertTrue("The second read neither waited nor read.", + System.currentTimeMillis() < deadline); + Thread.sleep(1); + } + + gate.release.countDown(); + first.join(DEADLINE_MILLIS); + second.join(DEADLINE_MILLIS); + assertNull("A read failed: " + failure.get(), failure.get()); + assertEquals( + "Two threads read from the native results at once.", + 1, + gate.most.get()); + } + + private static void run( + Consumer read, + DeviceDataHashDefault data, + AtomicReference failure) { + try { + read.accept(data); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + } + + /** + * True if the thread is blocked entering a monitor in the Device + * Detection Element Data, rather than anywhere else. + */ + private static boolean waitingForTheLock(Thread thread) { + if (thread.getState() != Thread.State.BLOCKED) { + return false; + } + StackTraceElement[] stack = thread.getStackTrace(); + return stack.length > 0 && + stack[0].getClassName().startsWith("fiftyone.devicedetection."); + } + + @Test + public void getValues_WaitsForAnotherGetValues() throws Exception { + assertOneReaderAtATime( + d -> d.getValues("IsMobile"), + d -> d.getValues("IsMobile")); + } + + @Test + public void getValues_WaitsForAPropertyGetter() throws Exception { + assertOneReaderAtATime( + d -> d.getIsMobile(), + d -> d.getValues("IsMobile")); + } + + @Test + public void propertyGetter_WaitsForGetValues() throws Exception { + assertOneReaderAtATime( + d -> d.getValues("IsMobile"), + d -> d.getIsMobile()); + } + + @Test + public void matchMetric_WaitsForGetValues() throws Exception { + assertOneReaderAtATime( + d -> d.getValues("IsMobile"), + d -> d.getDifference()); + } + + @Test + public void deviceId_WaitsForAPropertyGetter() throws Exception { + assertOneReaderAtATime( + d -> d.getIsMobile(), + d -> d.getDeviceId()); + } + + @Test + public void getValues_WaitsForAMatchMetric() throws Exception { + assertOneReaderAtATime( + d -> d.getDrift(), + d -> d.getValues("IsMobile")); + } +} From 1366d1522e60ad200beb6f12d78ae938c4198c0b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:21:32 +0000 Subject: [PATCH 07/35] Fix: hold the results lock for every read of the Device Detection native results DeviceDataBaseOnPremise gives subclasses its lock as getResultsLock and holds it while it asks whether a property is available. DeviceDataHashDefault holds it in propertyIsAvailable, in the public getValues and while it reads the match metrics and the device id. The lock is reentrant, so a read that calls another read is unchanged. thread-safety.md line 57 at 706de1f0: "Reading an Element Data instance once the Flow Element that writes it has finished MUST therefore be safe from several threads at once." (CO-48) thread-safety.md line 61 at 706de1f0: "Such Element Data MUST let only one thread at a time read from its results, for example by holding a lock for each read" (CO-49) --- .../flowelements/DeviceDataHashDefault.java | 96 +++++++++++-------- .../shared/DeviceDataBaseOnPremise.java | 21 +++- 2 files changed, 76 insertions(+), 41 deletions(-) diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java index 691761125..8bc0cdfb6 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java @@ -259,33 +259,45 @@ private AspectPropertyValue> getUserAgentsInternal() { @Override protected boolean propertyIsAvailable(String propertyName) { checkState(); - for (ResultsHashSwig results : resultsList) { - if (results.containsProperty(Swig.asBytes(propertyName))) { - return true; + synchronized (getResultsLock()) { + for (ResultsHashSwig results : resultsList) { + if (results.containsProperty(Swig.asBytes(propertyName))) { + return true; + } } + return false; } - return false; } + /** + * Get the values of a property as a list. This is public, so it holds + * the results lock itself, as every read from the native results does. + * See the + * thread safety specification. + * @param propertyName name of the property to get values for + * @return values as a list + */ @Override public AspectPropertyValue> getValues(String propertyName) { checkState(); AspectPropertyValue> result = new AspectPropertyValueDefault<>(); - ResultsHashSwig results = getSingleResults(propertyName); - if (results != null) { - Integer index = indexOf(propertyName); - try (VectorStringValuesSwig value = index == null ? - results.getValues(Swig.asBytes(propertyName)) : - results.getValues(index)) { - if (value.hasValue()) { - try (VectorStringSwig vector = value.getValue()) { - result.setValue(Collections.unmodifiableList( - Swig.asList(vector))); + synchronized (getResultsLock()) { + ResultsHashSwig results = getSingleResults(propertyName); + if (results != null) { + Integer index = indexOf(propertyName); + try (VectorStringValuesSwig value = index == null ? + results.getValues(Swig.asBytes(propertyName)) : + results.getValues(index)) { + if (value.hasValue()) { + try (VectorStringSwig vector = value.getValue()) { + result.setValue(Collections.unmodifiableList( + Swig.asList(vector))); + } + } + else { + result.setNoValueMessage(value.getNoValueMessage()); } - } - else { - result.setNoValueMessage(value.getNoValueMessage()); } } } @@ -407,29 +419,33 @@ protected TryGetResult tryGetValue( if (result.hasValue() == false) { boolean objSet = false; Object obj = null; - if (key.equalsIgnoreCase("DeviceId")) { - obj = getDeviceIdInternal(); - objSet = true; - } else if (key.equalsIgnoreCase("Difference")) { - obj = getDifferenceInternal(); - objSet = true; - } else if (key.equalsIgnoreCase("UserAgents")) { - obj = getUserAgentsInternal(); - objSet = true; - } else if (key.equalsIgnoreCase("Drift")) { - obj = getDriftInternal(); - objSet = true; - } else if (key.equalsIgnoreCase("Iterations")) { - obj = getIterationsInternal(); - objSet = true; - } - else if (key.equalsIgnoreCase("MatchedNodes")) { - obj = getMatchedNodesInternal(); - objSet = true; - } - else if (key.equalsIgnoreCase("Method")){ - obj = getMethodInternal(); - objSet = true; + // The match metrics read the native results, so they hold the + // results lock as every other read does. + synchronized (getResultsLock()) { + if (key.equalsIgnoreCase("DeviceId")) { + obj = getDeviceIdInternal(); + objSet = true; + } else if (key.equalsIgnoreCase("Difference")) { + obj = getDifferenceInternal(); + objSet = true; + } else if (key.equalsIgnoreCase("UserAgents")) { + obj = getUserAgentsInternal(); + objSet = true; + } else if (key.equalsIgnoreCase("Drift")) { + obj = getDriftInternal(); + objSet = true; + } else if (key.equalsIgnoreCase("Iterations")) { + obj = getIterationsInternal(); + objSet = true; + } + else if (key.equalsIgnoreCase("MatchedNodes")) { + obj = getMatchedNodesInternal(); + objSet = true; + } + else if (key.equalsIgnoreCase("Method")){ + obj = getMethodInternal(); + objSet = true; + } } if (objSet == true) { try { diff --git a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/DeviceDataBaseOnPremise.java b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/DeviceDataBaseOnPremise.java index fcf900ce6..e53f980ea 100644 --- a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/DeviceDataBaseOnPremise.java +++ b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/DeviceDataBaseOnPremise.java @@ -67,6 +67,21 @@ protected DeviceDataBaseOnPremise( primitiveTypes = getPrimitiveTypeMap(); } + /** + * The lock that every read from the underlying results holds, so that + * only one thread at a time reads from them. A subclass holds it in each + * method that reads the results and can be called without going through + * {@link #tryGetValue(String, Class, Class[])}. The lock is reentrant, so + * a read that calls another read holds it once more. See the + * thread safety specification: + * "Such Element Data MUST let only one thread at a time read from its + * results, for example by holding a lock for each read". + * @return the lock for reads from the results + */ + protected final Object getResultsLock() { + return getLock; + } + /** * Determine whether a property is available to return values from the * underlying results. @@ -199,7 +214,11 @@ protected TryGetResult tryGetValue( // If the complete set of values has not been populated // then we don't want to retrieve values for all // properties so just get the one we want. - if (propertyIsAvailable(key)) { + boolean available; + synchronized (getLock) { + available = propertyIsAvailable(key); + } + if (available) { if (type.equals(Object.class)) { type = (Class)AspectPropertyValue.class; parameterisedTypes = new Class[]{ getPropertyType(key) }; From 7802440f26b0ff162961edf706d0e9c335e1f34d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:26:23 +0000 Subject: [PATCH 08/35] Test: Latitude and Longitude are given rounded to 3 decimal places Reads the weighted doubles, the weighted floats and the single number. The free Lite file has neither property, so the test needs a paid IP Intelligence data file and is skipped without one. value-types.md line 99 at 706de1f0: "A value converted from Azimuth or Declination MUST be given to the caller rounded to 3 decimal places, whether it is given as a number, as a string or as a point in a WKT shape." (CO-58) --- .../flowelements/AngleRoundingTests.java | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AngleRoundingTests.java diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AngleRoundingTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AngleRoundingTests.java new file mode 100644 index 000000000..3d26cf1ae --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AngleRoundingTests.java @@ -0,0 +1,188 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.ipintelligence.shared.IPIntelligenceDataBaseOnPremise; +import fiftyone.ipintelligence.shared.testhelpers.FileUtils; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +/** + * Latitude and Longitude are stored as Declination and Azimuth, so every + * form they are given in is rounded to 3 decimal places. See + * value-types.md line 99: + * "A value converted from Azimuth or Declination MUST be given to the + * caller rounded to 3 decimal places, whether it is given as a number, as a + * string or as a point in a WKT shape." + *

+ * The free Lite file has no Latitude or Longitude, so these tests need a + * paid data file and are skipped without one. + */ +public class AngleRoundingTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private static final List ANGLES = + Arrays.asList("Latitude", "Longitude"); + + /** + * Addresses in different parts of the world, so that the values are + * not all whole numbers. + */ + private static final List ADDRESSES = Arrays.asList( + "185.28.167.77", + "116.154.188.222", + "45.236.48.61", + "2001:4860:4860::8888"); + + private interface Check { + void accept(IPIntelligenceData data) throws Exception; + } + + private static void forEachAddress(Check check) throws Exception { + File file = FileUtils.getHashFile(); + assumeTrue("No IP Intelligence data file was found.", file != null); + try (IPIntelligenceOnPremiseEngine engine = + new IPIntelligenceOnPremiseEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.MaxPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false) + .build(file.toString(), false); + Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + for (String angle : ANGLES) { + assumeTrue( + "The data file has no " + angle + ".", + engine.getProperties().stream().anyMatch( + p -> p.getName().equalsIgnoreCase(angle))); + } + for (String address : ADDRESSES) { + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.client-ip", address); + flowData.process(); + check.accept(flowData.get(IPIntelligenceData.class)); + } + } + } + } + + private static void assertRounded(String angle, double value) { + double rounded = Math.round(value * 1000d) / 1000d; + assertEquals( + angle + " " + value + " is not rounded to 3 decimal places.", + rounded, + value, + 0d); + } + + private static void assertRounded(String angle, float value) { + float rounded = (float) (Math.round(value * 1000d) / 1000d); + assertEquals( + angle + " " + value + " is not rounded to 3 decimal places.", + rounded, + value, + 0f); + } + + /** + * A weighted list of doubles gives each value rounded. + */ + @Test + public void weightedDoubles_AreRounded() throws Exception { + forEachAddress(data -> { + for (String angle : ANGLES) { + for (Object value : weighted(data, angle, Double.class)) { + assertRounded(angle, (double) (Double) value); + } + } + }); + } + + /** + * The values of a weighted list, read through the public getValues. + */ + private static List weighted( + IPIntelligenceData data, String angle, Class type) { + AspectPropertyValue> values = + ((IPIntelligenceDataBaseOnPremise) data).getValues( + angle, + new Class[] { List.class, IWeightedValue.class, type }); + List result = new java.util.ArrayList<>(); + if (values != null && values.hasValue()) { + for (Object value : values.getValue()) { + result.add(((IWeightedValue) value).getValue()); + } + } + return result; + } + + /** + * A weighted list of floats gives each value rounded. + */ + @Test + public void weightedFloats_AreRounded() throws Exception { + forEachAddress(data -> { + for (String angle : ANGLES) { + for (Object value : weighted(data, angle, Float.class)) { + assertRounded(angle, (float) (Float) value); + } + } + }); + } + + /** + * The single number is rounded. The text the native results give is + * not reachable through a public accessor, so it is not checked here. + */ + @Test + public void number_IsRounded() throws Exception { + forEachAddress(data -> { + AspectPropertyValue latitude = data.getLatitude(); + if (latitude.hasValue()) { + assertRounded("Latitude", latitude.getValue()); + } + AspectPropertyValue longitude = data.getLongitude(); + if (longitude.hasValue()) { + assertRounded("Longitude", longitude.getValue()); + } + }); + } +} From 1a4236fdc34c2bb76d222e7ce792b9a0e023c2e0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:26:49 +0000 Subject: [PATCH 09/35] Fix: move cxx to 178371ac, which rounds Azimuth and Declination numbers From 7992af70. 51degrees-cxx #177 (a1c0504d) gives a number stored as an angle rounded to the decimal places of its text, 3 by default, so the weighted Latitude and Longitude lists come rounded. The headers the wrappers use lose only private members in this range. cxx no longer has its device-detection-data and ip-intelligence-data folders, as the README expected. value-types.md line 99 at 706de1f0: "A value converted from Azimuth or Declination MUST be given to the caller rounded to 3 decimal places, whether it is given as a number, as a string or as a point in a WKT shape." (CO-58) --- cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cxx b/cxx index 7992af701..178371ac8 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 7992af7010b7654c115510938a2f7626204bf936 +Subproject commit 178371ac8d5dec71e625fcdcb833d1dcd1ce3480 From bec88e96e89a023bb77471a76566b475ade110a7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:28:27 +0000 Subject: [PATCH 10/35] Test: a built Pipeline's elements cannot be changed, even through its builder conceptual-overview.md line 118 at 706de1f0: "A Pipeline instance MUST be immutable. In other words, once created, it cannot be changed by adding new elements, removing old ones, etc." (CO-12) --- .../flowelements/PipelineImmutableTests.java | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineImmutableTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineImmutableTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineImmutableTests.java new file mode 100644 index 000000000..a63e24b46 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineImmutableTests.java @@ -0,0 +1,109 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByTenElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +/** + * Once built, a Pipeline's elements cannot be changed. See + * conceptual-overview.md line 118: + * "A Pipeline instance MUST be immutable. In other words, once created, it + * cannot be changed by adding new elements, removing old ones, etc." + */ +public class PipelineImmutableTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * The list of elements a Pipeline gives cannot be changed. + */ + @Test + public void getFlowElements_CannotBeChanged() throws Exception { + MultiplyByFiveElement five = new MultiplyByFiveElement( + mock(Logger.class)); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(five) + .build()) { + try { + pipeline.getFlowElements().add( + new MultiplyByTenElement(mock(Logger.class))); + fail("An element was added to a built Pipeline."); + } catch (UnsupportedOperationException e) { + // Expected. + } + try { + pipeline.getFlowElements().clear(); + fail("The elements of a built Pipeline were removed."); + } catch (UnsupportedOperationException e) { + // Expected. + } + assertEquals(Collections.singletonList(five), + pipeline.getFlowElements()); + } + } + + /** + * An element added to the builder after it has built a Pipeline is not + * part of that Pipeline, and does not run when it processes. + */ + @Test + public void builderReused_DoesNotChangeABuiltPipeline() throws Exception { + MultiplyByFiveElement five = new MultiplyByFiveElement( + mock(Logger.class)); + MultiplyByTenElement ten = new MultiplyByTenElement( + mock(Logger.class)); + PipelineBuilder builder = new PipelineBuilder(loggerFactory) + .addFlowElement(five); + try (Pipeline pipeline = builder.build()) { + builder.addFlowElement(ten); + + assertEquals(1, pipeline.getFlowElements().size()); + assertSame(five, pipeline.getFlowElements().get(0)); + assertEquals(null, pipeline.getElement(MultiplyByTenElement.class)); + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(five.evidenceKeys.get(0), 2); + flowData.process(); + assertEquals(10, flowData.getFromElement(five).getResult()); + assertFalse( + "An element added to the builder later ran.", + flowData.getDataKeys().stream().anyMatch( + k -> k.equalsIgnoreCase(ten.getElementDataKey()))); + } + } + } +} From e72bbb74dd9e8d56eec9d00aa5e386240ef45ce4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:33:44 +0000 Subject: [PATCH 11/35] Fix: give each Pipeline its own copy of the builder's elements PipelineDefault kept the builder's list, so an element added to the builder after build joined the Pipeline already built. It now keeps an unmodifiable copy. conceptual-overview.md line 118 at 706de1f0: "A Pipeline instance MUST be immutable. In other words, once created, it cannot be changed by adding new elements, removing old ones, etc." (CO-12) --- .../pipeline/core/flowelements/PipelineDefault.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java index 5f0eb1a26..93a8d4030 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java @@ -150,7 +150,11 @@ class PipelineDefault implements PipelineInternal { boolean suppressProcessExceptions) { this.logger = logger; this.flowDataFactory = flowDataFactory; - this.flowElements = flowElements; + // A copy, so that the builder adding to its own list later cannot + // change this Pipeline, which is immutable. See + // https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/conceptual-overview.md#L118 + this.flowElements = Collections.unmodifiableList( + new ArrayList<>(flowElements)); this.services = new ArrayList<>(); this.autoDisposeElements = autoDisposeElements; this.suppressProcessExceptions = suppressProcessExceptions; From 99763c046c00b175e8a6db13e3bd99b5dbab1ebd Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:38:15 +0000 Subject: [PATCH 12/35] Test: cloud and on-premise engines give results under one key and one interface For Device Detection and IP Intelligence. Passes as the code is, so no fix follows. README.md line 95 at 706de1f0: "Where both Cloud and On-Premise variants of an Engine are available, they MUST arrange that the Properties and values produced are compatible with each other so that the Engines can be substituted in the Pipeline without alteration to the consuming application." (CO-4) --- .../CloudOnPremiseSubstitutionTests.java | 101 ++++++++++++++++++ .../CloudOnPremiseSubstitutionTests.java | 86 +++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudOnPremiseSubstitutionTests.java create mode 100644 ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudOnPremiseSubstitutionTests.java diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudOnPremiseSubstitutionTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudOnPremiseSubstitutionTests.java new file mode 100644 index 000000000..e2c3b95f3 --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudOnPremiseSubstitutionTests.java @@ -0,0 +1,101 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngine; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import org.junit.Test; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; + +/** + * The cloud and on-premise Device Detection engines put their results in + * Flow Data under the same key and as the same interface, so a consuming + * application reads them in the same way whichever engine is in the + * Pipeline. See + * README.md line 95: + * "Where both Cloud and On-Premise variants of an Engine are available, + * they MUST arrange that the Properties and values produced are compatible + * with each other so that the Engines can be substituted in the Pipeline + * without alteration to the consuming application." + *

+ * Comparing the values each engine gives needs a resource key and the + * native engine with a data file, so it is left to the example tests. + */ +public class CloudOnPremiseSubstitutionTests { + + /** + * The engines are created without running a constructor, as only the + * data key, which is a constant, is read from them. + */ + @Test + public void elementDataKey_IsTheSame() { + FlowElement cloud = + mock(DeviceDetectionCloudEngine.class, CALLS_REAL_METHODS); + FlowElement onPremise = + mock(DeviceDetectionHashEngine.class, CALLS_REAL_METHODS); + assertEquals( + onPremise.getElementDataKey(), + cloud.getElementDataKey()); + } + + /** + * Each engine's Element Data is a {@link DeviceData}, which the + * consuming application asks Flow Data for. + */ + @Test + public void elementData_IsDeviceData() { + assertTrue(DeviceData.class.isAssignableFrom( + elementDataType(DeviceDetectionCloudEngine.class))); + assertTrue(DeviceData.class.isAssignableFrom( + elementDataType(DeviceDetectionHashEngine.class))); + } + + /** + * The first type argument of the nearest generic superclass, which is + * the engine's Element Data type. + */ + private static Class elementDataType(Class engine) { + for (Class c = engine; c != null; c = c.getSuperclass()) { + Type type = c.getGenericSuperclass(); + if (type instanceof ParameterizedType) { + Type first = + ((ParameterizedType) type).getActualTypeArguments()[0]; + if (first instanceof Class) { + return (Class) first; + } + } + } + assertNotNull("No Element Data type found for " + engine, null); + return null; + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudOnPremiseSubstitutionTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudOnPremiseSubstitutionTests.java new file mode 100644 index 000000000..352ebf27f --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudOnPremiseSubstitutionTests.java @@ -0,0 +1,86 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.cloud.flowelements.IPIntelligenceCloudEngine; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import org.junit.Test; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; + +/** + * The cloud and on-premise IP Intelligence engines put their results in + * Flow Data under the same key and as the same interface, so a consuming + * application reads them in the same way whichever engine is in the + * Pipeline. See + * README.md line 95: + * "Where both Cloud and On-Premise variants of an Engine are available, + * they MUST arrange that the Properties and values produced are compatible + * with each other so that the Engines can be substituted in the Pipeline + * without alteration to the consuming application." + */ +public class CloudOnPremiseSubstitutionTests { + + @Test + public void elementDataKey_IsTheSame() { + FlowElement cloud = + mock(IPIntelligenceCloudEngine.class, CALLS_REAL_METHODS); + FlowElement onPremise = + mock(IPIntelligenceOnPremiseEngine.class, CALLS_REAL_METHODS); + assertEquals( + onPremise.getElementDataKey(), + cloud.getElementDataKey()); + } + + @Test + public void elementData_IsIPIntelligenceData() { + assertTrue(IPIntelligenceData.class.isAssignableFrom( + elementDataType(IPIntelligenceCloudEngine.class))); + assertTrue(IPIntelligenceData.class.isAssignableFrom( + elementDataType(IPIntelligenceOnPremiseEngine.class))); + } + + private static Class elementDataType(Class engine) { + for (Class c = engine; c != null; c = c.getSuperclass()) { + Type type = c.getGenericSuperclass(); + if (type instanceof ParameterizedType) { + Type first = + ((ParameterizedType) type).getActualTypeArguments()[0]; + if (first instanceof Class) { + return (Class) first; + } + } + } + assertNotNull("No Element Data type found for " + engine, null); + return null; + } +} From f7bb90204b0d653c93ec6b0d89633899917f283a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:39:46 +0000 Subject: [PATCH 13/35] Test: one JSON Builder in two Pipelines uses each Pipeline's own configuration Passes as the code is, so no fix follows. multi-pipeline-elements.md line 17 at 706de1f0: "If this feature is enabled, then all Flow Elements that maintain any such state information MUST ensure that they have the means to keep track of isolated instances of this state for each Pipeline to which they are added and use the correct state information when processing." (CO-6) --- .../JsonBuilderMultiPipelineTests.java | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderMultiPipelineTests.java diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderMultiPipelineTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderMultiPipelineTests.java new file mode 100644 index 000000000..0b16dc2a6 --- /dev/null +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderMultiPipelineTests.java @@ -0,0 +1,150 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.jsonbuilder; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.data.PropertyMatcher; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.data.types.JavaScript; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.mockito.stubbing.Answer; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * One JSON Builder in two Pipelines keeps the configuration it reads from + * each Pipeline apart, and uses the right one for each. See + * multi-pipeline-elements.md line 17: + * "If this feature is enabled, then all Flow Elements that maintain any + * such state information MUST ensure that they have the means to keep + * track of isolated instances of this state for each Pipeline to which + * they are added and use the correct state information when processing." + */ +public class JsonBuilderMultiPipelineTests { + + /** + * A Pipeline whose "test" element has a JavaScript property, delayed or + * not. + */ + private static Pipeline pipeline(boolean delayExecution) { + FlowElement element = mock(FlowElement.class); + doReturn("test").when(element).getElementDataKey(); + ElementPropertyMetaData property = new ElementPropertyMetaDataDefault( + "jsproperty", + element, + "jsproperty", + JavaScript.class, + true, + null, + delayExecution, + null); + Map> metaData = + new HashMap<>(); + metaData.put("test", + Collections.singletonMap("jsproperty", property)); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElementAvailableProperties()).thenReturn(metaData); + return pipeline; + } + + /** + * Process Flow Data of the Pipeline with the element and return the + * JSON of the "test" element. + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static JSONObject process( + JsonBuilderElement element, + Pipeline pipeline) throws Exception { + ElementData data = mock(ElementData.class); + Map values = new HashMap<>(); + values.put("jsproperty", "var x = 1;"); + doReturn(values).when(data).asKeyMap(); + Map elementData = new HashMap<>(); + elementData.put("test", data); + + FlowData flowData = mock(FlowData.class); + when(flowData.elementDataAsMap()).thenReturn(elementData); + TryGetResult session = new TryGetResult<>(); + session.setValue("session"); + TryGetResult sequence = new TryGetResult<>(); + sequence.setValue(1); + when(flowData.tryGetEvidence(eq("query.session-id"), any(Class.class))) + .thenReturn(session); + when(flowData.tryGetEvidence(eq("query.sequence"), any(Class.class))) + .thenReturn(sequence); + when(flowData.getWhere(any(PropertyMatcher.class))) + .thenReturn(new HashMap<>()); + when(flowData.getPipeline()).thenReturn(pipeline); + when(flowData.getErrors()).thenReturn(new ArrayList()); + final JsonBuilderData[] result = { null }; + when(flowData.getOrAdd(anyString(), any(FlowElement.DataFactory.class))) + .thenAnswer((Answer) invocation -> { + FlowElement.DataFactory factory = invocation.getArgument(1); + result[0] = (JsonBuilderData) factory.create(flowData); + return result[0]; + }); + + element.process(flowData); + return new JSONObject(result[0].getJson()).getJSONObject("test"); + } + + @Test + public void onePipelinesConfiguration_IsNotUsedForAnother() + throws Exception { + JsonBuilderElement element = new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build(); + Pipeline delayed = pipeline(true); + Pipeline notDelayed = pipeline(false); + + assertTrue(process(element, delayed).has("jspropertydelayexecution"), + "The first Pipeline's property is delayed."); + assertFalse( + process(element, notDelayed).has("jspropertydelayexecution"), + "The second Pipeline's configuration came from the first."); + assertTrue(process(element, delayed).has("jspropertydelayexecution"), + "The first Pipeline's configuration came from the second."); + } +} From 075ea833c7ce277f507a79c9899baa911b16e02a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:42:16 +0000 Subject: [PATCH 14/35] Test: Flow Data takes writes from several threads, and locks only when it must Two threads adding Element Data under one key create it once, errors added from eight threads are all kept, and without parallel elements a write takes no lock. Passes as the code is, so no fix follows. parallel-processing.md line 22 at 706de1f0: "Flow Data MUST be capable of enforcing thread-safe writes to data structures while also maintaining performance when it is not needed." (CO-7) --- .../FlowDataConcurrentWriteTests.java | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataConcurrentWriteTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataConcurrentWriteTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataConcurrentWriteTests.java new file mode 100644 index 000000000..60b878968 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataConcurrentWriteTests.java @@ -0,0 +1,249 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByTenElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Flow Data of a Pipeline with parallel elements takes writes from several + * threads safely, and Flow Data of a Pipeline without them does not pay for + * it. See + * parallel-processing.md line 22: + * "Flow Data MUST be capable of enforcing thread-safe writes to data + * structures while also maintaining performance when it is not needed." + *

+ * The first writer is held inside its write until the test lets it go, + * and the second is checked from its thread's state, so nothing depends + * on how long anything takes. The deadline only stops a broken run from + * hanging. + */ +public class FlowDataConcurrentWriteTests { + + private static final long DEADLINE_MILLIS = 60_000; + + /** + * Loggers that write nothing, as the errors test logs thousands. + */ + private final ILoggerFactory loggerFactory = name -> mock(Logger.class); + + private Pipeline pipeline(boolean parallel) throws Exception { + PipelineBuilder builder = new PipelineBuilder(loggerFactory); + if (parallel) { + builder.addFlowElementsParallel(new FlowElement[] { + new MultiplyByFiveElement(mock(Logger.class)), + new MultiplyByTenElement(mock(Logger.class)) }); + } else { + builder.addFlowElement( + new MultiplyByFiveElement(mock(Logger.class))); + } + return builder.build(); + } + + private static boolean blockedInFlowData(Thread thread) { + if (thread.getState() != Thread.State.BLOCKED) { + return false; + } + StackTraceElement[] stack = thread.getStackTrace(); + return stack.length > 0 && + stack[0].getClassName().equals(FlowDataDefault.class.getName()); + } + + /** + * Two threads that add Element Data under the same key at once create + * it once, and both get the same instance. The second waits while the + * first creates it. + */ + @Test + public void getOrAdd_SameKeyFromTwoThreads_CreatesOnce() throws Exception { + try (Pipeline pipeline = pipeline(true); + FlowData flowData = pipeline.createFlowData()) { + assertTrue(pipeline.isConcurrent()); + AtomicInteger created = new AtomicInteger(); + CountDownLatch firstCreating = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + List results = new ArrayList<>(); + + FlowElement.DataFactory factory = data -> { + created.incrementAndGet(); + firstCreating.countDown(); + try { + release.await(DEADLINE_MILLIS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return mock(ElementData.class); + }; + Runnable add = () -> { + try { + ElementData result = flowData.getOrAdd("shared", factory); + synchronized (results) { + results.add(result); + } + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }; + + Thread first = new Thread(add); + first.start(); + assertTrue(firstCreating.await( + DEADLINE_MILLIS, TimeUnit.MILLISECONDS)); + Thread second = new Thread(add); + second.start(); + long deadline = System.currentTimeMillis() + DEADLINE_MILLIS; + while (created.get() < 2 && second.isAlive() && + blockedInFlowData(second) == false) { + assertTrue("The second writer neither waited nor wrote.", + System.currentTimeMillis() < deadline); + Thread.sleep(1); + } + release.countDown(); + first.join(DEADLINE_MILLIS); + second.join(DEADLINE_MILLIS); + + assertNull("A write failed: " + failure.get(), failure.get()); + assertEquals("The Element Data was created twice.", + 1, created.get()); + assertEquals(2, results.size()); + assertSame(results.get(0), results.get(1)); + } + } + + /** + * Errors added from several threads at once are all kept. + */ + @Test + public void addError_FromSeveralThreads_KeepsEveryError() + throws Exception { + final int threads = 8; + final int perThread = 1000; + try (Pipeline pipeline = pipeline(true); + FlowData flowData = pipeline.createFlowData()) { + CountDownLatch start = new CountDownLatch(1); + FlowElement element = mock(FlowElement.class); + AtomicReference failure = new AtomicReference<>(); + List workers = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + Thread worker = new Thread(() -> { + try { + start.await(); + } catch (InterruptedException e) { + return; + } + try { + for (int i = 0; i < perThread; i++) { + flowData.addError(new FlowError.Default( + new Exception("error"), element)); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } + }); + workers.add(worker); + worker.start(); + } + start.countDown(); + for (Thread worker : workers) { + worker.join(DEADLINE_MILLIS); + assertFalse(worker.isAlive()); + } + assertNull("A write failed: " + failure.get(), failure.get()); + assertEquals(threads * perThread, flowData.getErrors().size()); + } + } + + /** + * Without parallel elements, adding Element Data takes no lock, so a + * second writer for another key is not held up by the first. + */ + @Test + public void getOrAdd_WithoutParallelElements_TakesNoLock() + throws Exception { + try (Pipeline pipeline = pipeline(false); + FlowData flowData = pipeline.createFlowData()) { + assertFalse(pipeline.isConcurrent()); + CountDownLatch firstCreating = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + + Thread first = new Thread(() -> { + try { + flowData.getOrAdd("first", data -> { + firstCreating.countDown(); + try { + release.await( + DEADLINE_MILLIS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return mock(ElementData.class); + }); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }); + first.start(); + assertTrue(firstCreating.await( + DEADLINE_MILLIS, TimeUnit.MILLISECONDS)); + Thread second = new Thread(() -> { + try { + flowData.getOrAdd("second", + data -> mock(ElementData.class)); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }); + second.start(); + second.join(DEADLINE_MILLIS); + boolean secondFinished = second.isAlive() == false; + release.countDown(); + first.join(DEADLINE_MILLIS); + + assertNull("A write failed: " + failure.get(), failure.get()); + assertTrue("The second writer waited for the first.", + secondFinished); + } + } +} From bed28c7ec08ae5d65a791d74d2f3a316fa47832b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:43:42 +0000 Subject: [PATCH 15/35] Test: FlowElement requires the Evidence keys, the Properties and the Data Key Passes as the code is, so no fix follows. conceptual-overview.md line 73 at 706de1f0: "Each Element MUST:" (CO-11) --- .../FlowElementContractTests.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowElementContractTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowElementContractTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowElementContractTests.java new file mode 100644 index 000000000..dafb7e33f --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowElementContractTests.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Every Flow Element has to give the Evidence keys it uses, the Properties + * it populates and its Data Key, because the FlowElement interface declares + * all three without a default, so no element compiles without them. See + * conceptual-overview.md line 73: + * "Each Element MUST:" advertize the Evidence keys that it can make use of, + * publicize details of the Properties it will populate, and expose the + * string Data Key that is used to identify its Element Data within Flow + * Data. + */ +public class FlowElementContractTests { + + private static void assertRequired( + String name, Class returnType) throws Exception { + Method method = FlowElement.class.getMethod(name); + assertTrue(name + " is not public.", + Modifier.isPublic(method.getModifiers())); + assertTrue(name + " is not abstract, so an element need not give it.", + Modifier.isAbstract(method.getModifiers())); + assertFalse(name + " has a default.", method.isDefault()); + assertEquals(returnType, method.getReturnType()); + } + + @Test + public void evidenceKeys_AreRequired() throws Exception { + assertRequired("getEvidenceKeyFilter", EvidenceKeyFilter.class); + } + + @Test + public void properties_AreRequired() throws Exception { + assertRequired("getProperties", List.class); + } + + @Test + public void dataKey_IsRequired() throws Exception { + assertRequired("getElementDataKey", String.class); + } +} From 8a8f512a16d16941487ea7cc40037becc8e4884c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:44:44 +0000 Subject: [PATCH 16/35] Test: cache keys come from the advertised Evidence, with values compared by case Both pass as the code is, so no fix follows. caching.md line 59 at 706de1f0: "All Flow Elements MUST advertize the Evidence keys that they make use of." (CO-14) caching.md line 82 at 706de1f0: "Comparison of Evidence values MUST be case-sensitive." (CO-17) --- .../core/flowelements/CacheKeyTests.java | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/CacheKeyTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/CacheKeyTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/CacheKeyTests.java new file mode 100644 index 000000000..469dda135 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/CacheKeyTests.java @@ -0,0 +1,109 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.DataKey; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Mockito.mock; + +/** + * Cache keys are built from the Evidence that the element advertises, with + * values compared with their case. See the Generation of keys section of + * caching.md. + */ +public class CacheKeyTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private final MultiplyByFiveElement element = + new MultiplyByFiveElement(mock(Logger.class)); + + private DataKey keyFor(Map evidence) throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(element) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(evidence); + EvidenceKeyFilter filter = element.getEvidenceKeyFilter(); + return flowData.generateKey(filter); + } + } + + private static Map evidence(Object... pairs) { + Map map = new java.util.LinkedHashMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + map.put((String) pairs[i], pairs[i + 1]); + } + return map; + } + + /** + * caching.md line 59 at 706de1f0: "All Flow Elements MUST advertize the + * Evidence keys that they make use of." The key is built from those keys + * alone, so Evidence the element does not use does not change it, and + * Evidence it uses does. + */ + @Test + public void key_IsBuiltFromTheAdvertisedEvidenceOnly() throws Exception { + String used = element.evidenceKeys.get(0); + DataKey plain = keyFor(evidence(used, 2)); + DataKey withOther = keyFor(evidence(used, 2, "query.other", "x")); + DataKey otherValue = keyFor(evidence(used, 3)); + + assertEquals( + "Evidence the element does not use changed the key.", + plain, withOther); + assertEquals(plain.hashCode(), withOther.hashCode()); + assertNotEquals( + "Evidence the element uses did not change the key.", + plain, otherValue); + } + + /** + * caching.md line 82 at 706de1f0: "Comparison of Evidence values MUST + * be case-sensitive." + */ + @Test + public void values_AreComparedWithCase() throws Exception { + String used = element.evidenceKeys.get(0); + assertNotEquals( + "Two values that differ only in case gave the same key.", + keyFor(evidence(used, "abc")), + keyFor(evidence(used, "ABC"))); + assertEquals( + keyFor(evidence(used, "abc")), + keyFor(evidence(used, "abc"))); + } +} From f9b0f3b222bed96286cb1c9992c6c1533f7fcf66 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:46:26 +0000 Subject: [PATCH 17/35] Test: a cached engine takes Evidence keys in any case as the same key The Cloud Request Engine with a cache serves the second request from the cache when only the case of the key differs, and goes to the service when only the case of the value differs. Passes as the code is, so no fix follows. caching.md line 71 at 706de1f0: "Comparison of Evidence keys MUST be case-insensitive." (CO-16) --- .../CloudRequestEngineCacheKeyCaseTests.java | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCacheKeyCaseTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCacheKeyCaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCacheKeyCaseTests.java new file mode 100644 index 000000000..2f001f8a0 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCacheKeyCaseTests.java @@ -0,0 +1,121 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.json.JSONArray; +import org.junit.jupiter.api.Test; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * An engine with a cache treats two requests whose Evidence differs only + * in the case of a key as the same request, and two whose Evidence differs + * only in the case of a value as different ones. See the Generation of keys + * section of + * caching.md: + * "Comparison of Evidence keys MUST be case-insensitive." + */ +public class CloudRequestEngineCacheKeyCaseTests + extends CloudRequestEngineTestsBase { + + public CloudRequestEngineCacheKeyCaseTests() throws MalformedURLException { + super(); + } + + /** + * Process the two pieces of Evidence in turn with one engine that has a + * cache, and count the requests that reach the cloud service. + */ + private int requestsFor(String[] first, String[] second) throws Exception { + evidenceKeysResponse = + new JSONArray(Arrays.asList("query.User-Agent")).toString(); + configureMockedClient(); + final List answers = new ArrayList<>(Arrays.asList( + "{'device':{'value':'1'}}", + "{'device':{'value':'2'}}")); + doAnswer(invocation -> { + java.net.HttpURLConnection connection = invocation.getArgument(0); + String path = connection.getURL().getPath(); + if (path.endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (path.endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + return answers.remove(0); + }).when(httpClient).postData(any(), anyMap(), (byte[]) any()); + + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (String[] evidence : new String[][] { first, second }) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence[0], evidence[1]); + data.process(); + data.get(CloudRequestData.class).getJsonResponse(); + } + } + } + return 2 - answers.size(); + } + + @Test + public void keyInAnotherCase_IsServedFromTheCache() throws Exception { + assertEquals(1, requestsFor( + new String[] { "query.User-Agent", "iPhone" }, + new String[] { "query.user-agent", "iPhone" }), + "A key in another case was not taken as the same Evidence."); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false && + c.getURL().getPath().endsWith("evidencekeys") == false), + any(), + any()); + } + + @Test + public void valueInAnotherCase_IsNotServedFromTheCache() throws Exception { + assertEquals(2, requestsFor( + new String[] { "query.User-Agent", "iPhone" }, + new String[] { "query.User-Agent", "IPHONE" }), + "A value in another case was taken as the same Evidence."); + } +} From b4419d072e9cce2ebde2566c22794ebfc1a20c99 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:48:19 +0000 Subject: [PATCH 18/35] Test: an error in the web filter does not stop the request when exceptions are suppressed exception-handling.md line 73 at 706de1f0: "The implementor MUST ensure that exceptions/errors that happen in these areas are handled correctly in order to avoid the API taking down the customer's production website." (CO-28) --- .../web/PipelineFilterErrorTests.java | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterErrorTests.java diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterErrorTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterErrorTests.java new file mode 100644 index 000000000..d332210f1 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterErrorTests.java @@ -0,0 +1,189 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.web.services.PipelineResultServiceCore; +import fiftyone.pipeline.web.services.UACHServiceCore; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.stubbing.Answer; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.lang.reflect.Field; + +import static fiftyone.pipeline.web.Constants.HTTPCONTEXT_FLOWDATA; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * An error in the web integration does not stop the customer's request + * when the Pipeline suppresses process exceptions: it is logged and the + * request goes on down the filter chain. When the Pipeline does not + * suppress them, the error is thrown as before. See + * exception-handling.md line 73: + * "The implementor MUST ensure that exceptions/errors that happen in these + * areas are handled correctly in order to avoid the API taking down the + * customer's production website." and line 42, which gives + * IsSuppressProcessExceptions to wrappers "to make a decision on whether the + * exceptions should be suppressed and logged during processing." + */ +public class PipelineFilterErrorTests { + + private PipelineFilter filter; + private HttpServletRequest request; + private HttpServletResponse response; + private FilterChain chain; + private Object storedFlowData; + + @Before + public void init() throws Exception { + FilterConfig config = mock(FilterConfig.class); + when(config.getInitParameter(anyString())).thenAnswer( + (Answer) invocation -> + "config-file".equals(invocation.getArgument(0)) ? + "filename" : null); + ServletContext context = mock(ServletContext.class); + File file = new File(getClass().getClassLoader() + .getResource("config.xml").getFile()); + when(context.getRealPath(anyString())) + .thenReturn(file.getAbsolutePath()); + when(config.getServletContext()).thenReturn(context); + + request = mock(HttpServletRequest.class); + response = mock(HttpServletResponse.class); + chain = mock(FilterChain.class); + when(request.getScheme()).thenReturn(""); + when(request.getLocalAddr()).thenReturn(""); + when(request.getRequestURL()).thenReturn(new StringBuffer("")); + when(request.getRequestURI()).thenReturn(""); + when(request.getAttribute(eq(HTTPCONTEXT_FLOWDATA))) + .thenAnswer(invocation -> storedFlowData); + org.mockito.Mockito.doAnswer(invocation -> { + storedFlowData = invocation.getArgument(1); + return null; + }).when(request).setAttribute(eq(HTTPCONTEXT_FLOWDATA), any()); + + filter = new PipelineFilter(); + filter.init(config); + } + + @After + public void cleanUp() { + if (filter != null) { + filter.destroy(); + } + } + + private void set(String name, Object value) throws Exception { + Field field = PipelineFilter.class.getDeclaredField(name); + field.setAccessible(true); + field.set(filter, value); + } + + private Object get(String name) throws Exception { + Field field = PipelineFilter.class.getDeclaredField(name); + field.setAccessible(true); + return field.get(filter); + } + + /** + * Give the filter a Pipeline that reports the setting, keeping the + * real one to close. + */ + private void suppress(boolean value) throws Exception { + Pipeline real = (Pipeline) get("pipeline"); + Pipeline reporting = mock(Pipeline.class, + org.mockito.AdditionalAnswers.delegatesTo(real)); + when(reporting.isSuppressProcessExceptions()).thenReturn(value); + set("pipeline", reporting); + } + + private void failProcessing(RuntimeException failure) throws Exception { + PipelineResultServiceCore failing = + mock(PipelineResultServiceCore.class); + doThrow(failure).when(failing).process(any(HttpServletRequest.class)); + set("resultService", failing); + } + + @Test + public void processingFails_Suppressed_RequestGoesOn() throws Exception { + suppress(true); + failProcessing(new IllegalStateException("processing failed")); + + filter.doFilter(request, response, chain); + + verify(chain, times(1)).doFilter( + any(ServletRequest.class), any(ServletResponse.class)); + } + + @Test + public void headersFail_Suppressed_RequestGoesOn() throws Exception { + suppress(true); + UACHServiceCore failing = mock(UACHServiceCore.class); + doThrow(new IllegalStateException("headers failed")).when(failing) + .setResponseHeaders(any(FlowData.class), + any(HttpServletResponse.class)); + set("uachServiceCore", failing); + + filter.doFilter(request, response, chain); + + verify(chain, times(1)).doFilter( + any(ServletRequest.class), any(ServletResponse.class)); + } + + @Test + public void processingFails_NotSuppressed_IsThrown() throws Exception { + suppress(false); + IllegalStateException failure = + new IllegalStateException("processing failed"); + failProcessing(failure); + + try { + filter.doFilter(request, response, chain); + fail("The failure was not thrown."); + } catch (IllegalStateException e) { + assertSame(failure, e); + } catch (ServletException e) { + fail("The failure was wrapped: " + e); + } + } +} From d4e4bdd73c85fefa39aecb4bdd6ff1500a0fbd54 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:49:38 +0000 Subject: [PATCH 19/35] Fix: let the web filter suppress its own errors as the Pipeline is set to When the Pipeline suppresses process exceptions, an error in PipelineFilter's own work (the not modified check, processing, the UA-CH headers or serving the JavaScript and JSON) is logged and the request goes on down the filter chain without the results. Otherwise it is thrown, as before. The FlowData is closed either way, and only if processing stored one. exception-handling.md line 73 at 706de1f0: "The implementor MUST ensure that exceptions/errors that happen in these areas are handled correctly in order to avoid the API taking down the customer's production website." (CO-28) --- .../fiftyone/pipeline/web/PipelineFilter.java | 134 ++++++++++++++---- 1 file changed, 107 insertions(+), 27 deletions(-) diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java index 92fd7c546..84971d197 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java @@ -58,6 +58,9 @@ * @see Specification */ public class PipelineFilter implements Filter { + private static final org.slf4j.Logger logger = + LoggerFactory.getLogger(PipelineFilter.class); + private Pipeline pipeline; private WebRequestEvidenceServiceCore evidenceService; @@ -195,42 +198,119 @@ public void doFilter( ServletResponse response, FilterChain chain) throws IOException, ServletException { + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + boolean suppress = isSuppressProcessExceptions(); + + boolean served; + try { + served = processRequest(httpRequest, httpResponse); + } catch (RuntimeException e) { + // The Pipeline's setting decides whether an error in the web + // integration stops the request. When exceptions are suppressed + // it is logged and the request goes on without the results, so + // the API does not take down the website. See + // https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/features/exception-handling.md#L73 + closeFlowData(httpRequest, suppress, e); + if (suppress == false) { + throw e; + } + logger.error("The 51Degrees Pipeline could not process the " + + "request, which goes on without its results.", e); + served = false; + } + if (served) { + closeFlowData(httpRequest, suppress, null); + return; + } + + try { + chain.doFilter(request, response);//sends request to next resource + } finally { + // The rest of the filters in the chain have now been called, or + // one of them failed. Either way it is time to dispose of the + // FlowData instance. + closeFlowData(httpRequest, suppress, null); + } + } + + /** + * Whether the Pipeline suppresses process exceptions, which also + * decides whether an error in this filter stops the request. + * @return true if exceptions are suppressed and logged + */ + private boolean isSuppressProcessExceptions() { + return pipeline != null && pipeline.isSuppressProcessExceptions(); + } + + /** + * Process the request and set the response headers, and serve the + * 51Degrees JavaScript or JSON if that is what is asked for. + * @param request the request + * @param response the response + * @return true if the response has been served, so the request does not + * go on down the filter chain + * @throws IOException if the JavaScript or JSON could not be written + */ + private boolean processRequest( + HttpServletRequest request, + HttpServletResponse response) throws IOException { // A request for the 51Degrees JavaScript or JSON whose If-None-Match // matches the ETag of its Evidence is answered without processing, // because that Evidence alone decides the content. - if (jsService.serveNotModified( - (HttpServletRequest) request, - (HttpServletResponse) response, - resultService)) { - return; + if (jsService.serveNotModified(request, response, resultService)) { + return true; } // Populate the request properties and store against the // HttpContext. - resultService.process((HttpServletRequest)request); + resultService.process(request); + + // Get FlowData from the request. + FlowData flowData = flowDataProviderCore.getFlowData(request); + + // Set UACH response headers. + uachServiceCore.setResponseHeaders(flowData, response); + // If 51Degrees JavaScript or JSON is being requested then serve it. + // Otherwise the request continues down the filter chain. + return jsService.serveJS(request, response) || + jsService.serveJson(request, response); + } + + /** + * Close the request's FlowData, if processing put one there. + * @param request the request + * @param suppress true if a failure to close is logged rather than + * thrown + * @param failure the failure already being handled, which a failure to + * close is added to, or null + * @throws ServletException if the FlowData could not be closed and + * exceptions are not suppressed + */ + private void closeFlowData( + HttpServletRequest request, + boolean suppress, + RuntimeException failure) throws ServletException { + FlowData flowData; try { - // Get FlowData from the request. - FlowData flowData = flowDataProviderCore.getFlowData((HttpServletRequest)request); - - // Set UACH response headers. - uachServiceCore.setResponseHeaders( - flowData, (HttpServletResponse)response); - - // If 51Degrees JavaScript or JSON is being requested then serve it. - // Otherwise continue down the filter Pipeline. - if (jsService.serveJS((HttpServletRequest)request, (HttpServletResponse) response) == false && - jsService.serveJson((HttpServletRequest)request, (HttpServletResponse) response) == false) { - chain.doFilter(request, response);//sends request to next resource - } - } finally { - // The rest of the filters in the chain have now been called, or - // one of them failed. Either way it is time to dispose of the - // FlowData instance. - try { - flowDataProviderCore.getFlowData((HttpServletRequest) request).close(); - } catch (Exception e) { - throw new ServletException("FlowData could not be disposed of.", e); + flowData = flowDataProviderCore.getFlowData(request); + } catch (RuntimeException e) { + flowData = null; + } + if (flowData == null) { + return; + } + try { + flowData.close(); + } catch (Exception e) { + if (failure != null) { + failure.addSuppressed(e); + } else if (suppress) { + logger.error("FlowData could not be disposed of.", e); + } else { + throw new ServletException( + "FlowData could not be disposed of.", e); } } } From 0425ec7ca80ffa1c0f9eba41d481222e9853cb27 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:52:06 +0000 Subject: [PATCH 20/35] Test: an unexpected error in the background update check is logged and kept in the task exception-handling.md line 83 at 706de1f0: "When errors do occur, they will be logged and MUST NOT cause the process to crash." (CO-29) --- ...DataUpdateServiceBackgroundErrorTests.java | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceBackgroundErrorTests.java diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceBackgroundErrorTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceBackgroundErrorTests.java new file mode 100644 index 000000000..93863e58a --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceBackgroundErrorTests.java @@ -0,0 +1,148 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import ch.qos.logback.classic.Level; +import fiftyone.common.testhelpers.TestLogger; +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ScheduledFuture; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * An unexpected error in the background update check is logged and kept + * inside the task, so it can neither end the thread silently nor reach the + * process. See + * exception-handling.md line 83: + * "When errors do occur, they will be logged and MUST NOT cause the + * process to crash." + *

+ * The scheduled task is captured and run on the test's thread, so nothing + * depends on timing. + */ +public class DataUpdateServiceBackgroundErrorTests { + + private TestLogger logger; + private FutureFactory futureFactory; + private FileWrapperFactory fileWrapperFactory; + private DataUpdateServiceDefault dataUpdate; + + @Before + public void init() { + ch.qos.logback.classic.Logger real = + (ch.qos.logback.classic.Logger) LoggerFactory.getLogger( + getClass()); + real.setLevel(Level.INFO); + logger = new TestLogger("test", real); + futureFactory = mock(FutureFactory.class); + doReturn(mock(ScheduledFuture.class)).when(futureFactory) + .schedule(any(Runnable.class), anyLong()); + doReturn(mock(ScheduledFuture.class)).when(futureFactory) + .scheduleRepeating(any(Runnable.class), anyLong()); + fileWrapperFactory = mock(FileWrapperFactory.class); + dataUpdate = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory, + futureFactory); + } + + @SuppressWarnings("unchecked") + private AspectEngineDataFileDefault registerFile() { + DataFileConfigurationDefault config = + new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setFileSystemWatcherEnabled(false); + config.setDataFilePath("data.dat"); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(mock(OnPremiseAspectEngine.class)); + file.setConfiguration(config); + file.setTempDataFilePath("temp.dat"); + dataUpdate.registerDataFile(file); + return file; + } + + private Runnable scheduledCheck() { + ArgumentCaptor task = ArgumentCaptor.forClass(Runnable.class); + verify(futureFactory, atLeastOnce()).schedule(task.capture(), anyLong()); + return task.getValue(); + } + + @Test + public void scheduledCheck_UnexpectedError_IsLoggedAndKeptInTheTask() { + registerFile(); + Runnable check = scheduledCheck(); + when(fileWrapperFactory.getLastModified(anyString())) + .thenThrow(new IllegalStateException("disk went away")); + int errorsBefore = logger.errorsLogged.size(); + + try { + check.run(); + } catch (RuntimeException e) { + fail("The error left the background task: " + e); + } + + assertFalse("The error was not logged.", + logger.errorsLogged.size() == errorsBefore); + } + + @Test + public void scheduledCheck_ErrorInACallback_IsLoggedAndKeptInTheTask() { + dataUpdate.onUpdateComplete((sender, args) -> { + throw new IllegalStateException("callback failed"); + }); + registerFile(); + Runnable check = scheduledCheck(); + when(fileWrapperFactory.getLastModified(anyString())).thenReturn(0L); + int errorsBefore = logger.errorsLogged.size(); + + try { + check.run(); + } catch (RuntimeException e) { + fail("The error left the background task: " + e); + } + + assertFalse("The error was not logged.", + logger.errorsLogged.size() == errorsBefore); + } +} From 6dc6f2c56efae6f799a6fe59a47abceacdac70f9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:54:17 +0000 Subject: [PATCH 21/35] Fix: log an unexpected error in a background data update task The scheduled update checks and the file system watch run through logErrors, which logs an error the task did not expect and keeps it in the task. Before, the executor kept it in a future nobody read, so it was never logged. An error of the virtual machine itself is logged and thrown on. Rescheduling after such an error is left as it is. exception-handling.md line 83 at 706de1f0: "When errors do occur, they will be logged and MUST NOT cause the process to crash." (CO-29) --- .../services/DataUpdateServiceDefault.java | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index ba4631d33..b8026ac3e 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -223,6 +223,35 @@ public AutoUpdateStatus checkForUpdate(OnPremiseAspectEngineexception-handling.md line 83: + * "When errors do occur, they will be logged and MUST NOT cause the + * process to crash." An error of the virtual machine itself, such as + * running out of memory, is logged and then thrown on. + * @param what what the task does, for the log + * @param task the task + * @return the wrapped task + */ + private Runnable logErrors(final String what, final Runnable task) { + return new Runnable() { + @Override + public void run() { + try { + task.run(); + } catch (Throwable t) { + logger.error("Unexpected error in the background " + + what + ".", t); + if (t instanceof VirtualMachineError) { + throw (VirtualMachineError) t; + } + } + } + }; + } + private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { AutoUpdateStatus result = AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; @@ -296,12 +325,12 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { if (dataFile.getFuture() != null) { dataFile.getFuture().cancel(true); dataFile.setFuture(futureFactory.schedule( - new Runnable() { + logErrors("update check", new Runnable() { @Override public void run() { checkForUpdate(dataFile, false); } - }, + }), getIntervalMillis(dataFile.getConfiguration()))); } } @@ -543,7 +572,7 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { delayMillis); try { ScheduledFuture future = futureFactory.schedule( - new Runnable() { + logErrors("update check", new Runnable() { @Override public void run() { logger.debug("running checkForUpdate {}", @@ -551,7 +580,7 @@ public void run() { checkForUpdate(aspectDataFile, false); logger.debug("Returning from checkForUpdate"); } - }, + }), delayMillis); logger.debug("Update check scheduled for {} millis", future.getDelay(TimeUnit.MILLISECONDS)); @@ -603,7 +632,7 @@ public void run() { aspectDataFile.getConfiguration().getPollingIntervalSeconds() * 1000L; logger.debug("Setting poll future {} millis", pollingInterval); aspectDataFile.setPollFuture(futureFactory.scheduleRepeating( - new Runnable() { + logErrors("file system watch", new Runnable() { @Override public void run() { logger.debug("Watcher is running"); @@ -626,7 +655,7 @@ public void run() { // make sure to reset the watch key or we are not called again watchKey.reset(); } - }, + }), pollingInterval)); logger.debug("Poll check scheduled for {} mins", aspectDataFile.getPollFuture().getDelay(TimeUnit.MINUTES)); From 83eea043dfa698702b1cd11f5c9d88775391d257 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:55:43 +0000 Subject: [PATCH 22/35] Test: a Pipeline set not to close its elements leaves a shared element open Passes as the code is, so no fix follows. resource-cleanup.md line 28 at 706de1f0: "However, if the multi Pipeline Elements feature is implemented then there MUST be an option to disable this behavior." (CO-39) --- .../flowelements/PipelineAutoCloseTests.java | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineAutoCloseTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineAutoCloseTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineAutoCloseTests.java new file mode 100644 index 000000000..737a2a5da --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineAutoCloseTests.java @@ -0,0 +1,88 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Closing a Pipeline closes its elements only when it is set to, so that + * an element shared with another Pipeline can be kept open. See + * resource-cleanup.md line 28: + * "However, if the multi Pipeline Elements feature is implemented then + * there MUST be an option to disable this behavior." + */ +public class PipelineAutoCloseTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void autoCloseOff_SharedElementStaysOpen() throws Exception { + MultiplyByFiveElement shared = + new MultiplyByFiveElement(mock(Logger.class)); + try (Pipeline first = new PipelineBuilder(loggerFactory) + .setAutoCloseElements(false) + .addFlowElement(shared) + .build(); + Pipeline second = new PipelineBuilder(loggerFactory) + .setAutoCloseElements(false) + .addFlowElement(shared) + .build()) { + // Closing again at the end of the block does nothing. + first.close(); + + assertFalse("Closing one Pipeline closed the shared element.", + shared.isClosed()); + try (FlowData flowData = second.createFlowData()) { + flowData.addEvidence(shared.evidenceKeys.get(0), 2); + flowData.process(); + assertEquals(10, flowData.getFromElement(shared).getResult()); + } + } finally { + shared.close(); + } + } + + @Test + public void autoCloseOn_ClosesTheElement() throws Exception { + MultiplyByFiveElement element = + new MultiplyByFiveElement(mock(Logger.class)); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setAutoCloseElements(true) + .addFlowElement(element) + .build()) { + assertFalse(element.isClosed()); + } + assertTrue(element.isClosed()); + } +} From c334cd894e2afa99b60ec8435bd1fdb446fd3d15 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:56:49 +0000 Subject: [PATCH 23/35] Test: closing Flow Data leaves Element Data that needs no cleanup alone Passes as the code is, so no fix follows. resource-cleanup.md line 84 at 706de1f0: "Element Data instances that do not require cleanup MUST be left alone in order for result caching to work correctly." (CO-45) --- .../flowelements/FlowDataCleanupTests.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataCleanupTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataCleanupTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataCleanupTests.java new file mode 100644 index 000000000..fc2b13670 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataCleanupTests.java @@ -0,0 +1,68 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.withSettings; + +/** + * Closing Flow Data closes the Element Data that needs cleanup and does + * nothing at all to the rest, which a cache may still hold. See + * resource-cleanup.md line 84: + * "Element Data instances that do not require cleanup MUST be left alone + * in order for result caching to work correctly." + */ +public class FlowDataCleanupTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void close_LeavesElementDataWithoutCleanupAlone() throws Exception { + ElementData plain = mock(ElementData.class); + ElementData closeable = mock(ElementData.class, + withSettings().extraInterfaces(AutoCloseable.class)); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new MultiplyByFiveElement(mock(Logger.class))) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.getOrAdd("plain", data -> plain); + flowData.getOrAdd("closeable", data -> closeable); + } + + verifyNoInteractions(plain); + verify((AutoCloseable) closeable, times(1)).close(); + } +} From 616eb1520d2344181154708d5e0b2861bd47bafc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 21:59:39 +0000 Subject: [PATCH 24/35] Test: FlowData declares the exceptions its methods throw, and throws only those exception-handling.md line 17 at 706de1f0: "In all cases, standard language features SHOULD be used to let IDEs and users know what exceptions/errors a particular function can throw." (CO-24) --- .../FlowDataDeclaredExceptionsTests.java | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataDeclaredExceptionsTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataDeclaredExceptionsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataDeclaredExceptionsTests.java new file mode 100644 index 000000000..3f165ac37 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataDeclaredExceptionsTests.java @@ -0,0 +1,164 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.core.exceptions.PipelineDataException; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +/** + * The FlowData interface declares, in the throws clause that IDEs and + * reflection read, the exceptions its methods throw for a mistake by the + * caller, and the implementation throws one of those. See + * exception-handling.md line 17: + * "In all cases, standard language features SHOULD be used to let IDEs and + * users know what exceptions/errors a particular function can throw." + */ +public class FlowDataDeclaredExceptionsTests { + + private final ILoggerFactory loggerFactory = name -> mock(Logger.class); + + private interface Call { + void run(FlowData flowData) throws Exception; + } + + private static List> declared(String name, Class... types) + throws Exception { + Method method = FlowData.class.getMethod(name, types); + return Arrays.asList(method.getExceptionTypes()); + } + + private static void assertDeclares( + List> declared, Class type, String method) { + assertTrue(method + " does not declare " + type.getSimpleName() + + ", only " + declared + ".", + declared.contains(type)); + } + + /** + * Call the method on Flow Data that has not been processed, and check + * the exception is one the interface declares. + */ + private void assertThrowsDeclared( + List> declared, String method, boolean process, Call call) + throws Exception { + MultiplyByFiveElement element = + new MultiplyByFiveElement(mock(Logger.class)); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(element) + .build(); + FlowData flowData = pipeline.createFlowData()) { + if (process) { + flowData.addEvidence(element.evidenceKeys.get(0), 2); + flowData.process(); + } + try { + call.run(flowData); + fail(method + " did not throw."); + } catch (Exception e) { + boolean isDeclared = false; + for (Class type : declared) { + isDeclared |= type.isInstance(e); + } + assertTrue(method + " threw " + e.getClass().getName() + + ", which it does not declare, only " + declared + ".", + isDeclared); + } + } + } + + @Test + public void process_DeclaresAndThrowsIllegalState() throws Exception { + List> declared = declared("process"); + assertDeclares(declared, IllegalStateException.class, "process"); + assertThrowsDeclared(declared, "process", true, FlowData::process); + } + + @Test + @SuppressWarnings("unchecked") + public void getByTypedKey_DeclaresAndThrowsIllegalState() + throws Exception { + List> declared = declared("get", TypedKey.class); + assertDeclares(declared, IllegalStateException.class, "get(TypedKey)"); + assertDeclares(declared, IllegalArgumentException.class, + "get(TypedKey)"); + assertThrowsDeclared(declared, "get(TypedKey)", false, + d -> d.get(mock(TypedKey.class))); + } + + @Test + public void getByName_DeclaresAndThrowsIllegalState() throws Exception { + List> declared = declared("get", String.class); + assertDeclares(declared, IllegalStateException.class, "get(String)"); + assertThrowsDeclared(declared, "get(String)", false, + d -> d.get("muliplyByFive")); + } + + @Test + public void getFromElement_DeclaresAndThrowsIllegalState() + throws Exception { + List> declared = + declared("getFromElement", FlowElement.class); + assertDeclares(declared, IllegalStateException.class, + "getFromElement"); + assertDeclares(declared, IllegalArgumentException.class, + "getFromElement"); + assertThrowsDeclared(declared, "getFromElement", true, + d -> d.getFromElement(null)); + } + + @Test + public void getAs_DeclaresAndThrowsPipelineDataException() + throws Exception { + List> declared = declared("getAs", String.class, Class.class); + assertDeclares(declared, PipelineDataException.class, "getAs"); + assertDeclares(declared, ClassCastException.class, "getAs"); + assertThrowsDeclared(declared, "getAs", false, + d -> d.getAs("result", Integer.class)); + for (String name : new String[] { + "getAsString", "getAsBool", "getAsInt", "getAsLong", + "getAsFloat", "getAsDouble" }) { + assertDeclares(declared(name, String.class), + PipelineDataException.class, name); + } + } + + /** + * Keeps the ElementData import, which documents what get returns. + */ + @SuppressWarnings("unused") + private static ElementData unused; +} From 94971854928fdce11fb75443a4b26a3db920af23 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:01:27 +0000 Subject: [PATCH 25/35] Fix: declare on FlowData the exceptions its methods throw process, get, getFromElement, getAs and the getAs typed getters name in their throws clauses and @throws tags the unchecked exceptions they throw for a mistake by the caller. get(String) before processing now throws IllegalStateException, as get(TypedKey) does, rather than a plain RuntimeException. A caller that catches RuntimeException is unchanged. exception-handling.md line 17 at 706de1f0: "In all cases, standard language features SHOULD be used to let IDEs and users know what exceptions/errors a particular function can throw." (CO-24) --- .../fiftyone/pipeline/core/data/FlowData.java | 53 +++++++++++++++---- .../core/flowelements/FlowDataDefault.java | 2 +- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java index 763da2bb3..46adb7cf2 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.core.data; +import fiftyone.pipeline.core.exceptions.PipelineDataException; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.typed.TypedKey; @@ -40,8 +41,10 @@ public interface FlowData extends AutoCloseable { /** * Process this FlowData via the owning Pipeline. Once this method returns * the FlowData will not be changed further and cannot be processed again. + * @throws IllegalStateException if this FlowData has already been + * processed, or its Pipeline has been closed */ - void process(); + void process() throws IllegalStateException; /** * If this flag is set to true, future FlowElements will not process this @@ -129,15 +132,19 @@ public interface FlowData extends AutoCloseable { * @param key the aspect to get * @param type type of the aspect * @return aspect's {@link ElementData} or null if not present + * @throws IllegalStateException if this FlowData has not been processed + * @throws IllegalArgumentException if the key is null */ - T get(TypedKey key); + T get(TypedKey key) + throws IllegalStateException, IllegalArgumentException; /** * Get the {@link ElementData} for the requested data key. * @param key to the aspect to get * @return aspect's {@link ElementData} or null if not present + * @throws IllegalStateException if this FlowData has not been processed */ - ElementData get(String key); + ElementData get(String key) throws IllegalStateException; /** * Get the {@link ElementData} for the requested {@link ElementData} @@ -157,8 +164,11 @@ public interface FlowData extends AutoCloseable { * @param element {@link FlowElement} to get the data for * @param {@link ElementData} stored by the element * @return element data stored by the element + * @throws IllegalStateException if this FlowData has not been processed + * @throws IllegalArgumentException if the element is null */ - T getFromElement(FlowElement element); + T getFromElement(FlowElement element) + throws IllegalStateException, IllegalArgumentException; /** * Get the specified property as the specified type. @@ -166,8 +176,11 @@ public interface FlowData extends AutoCloseable { * @param key the name of the property to get * @param type the type to return the property value as * @return the property value + * @throws PipelineDataException if this FlowData has not been processed + * @throws ClassCastException if the value is not of the type asked for */ - T getAs(String key, Class type); + T getAs(String key, Class type) + throws PipelineDataException, ClassCastException; /** * Check if the flow data contains an item with the specified key name and @@ -183,43 +196,61 @@ public interface FlowData extends AutoCloseable { * Get the specified property as a string. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed + * @throws ClassCastException if the value is not a string */ - String getAsString(String propertyName); + String getAsString(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a boolean. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed + * @throws ClassCastException if the value is not a boolean */ - Boolean getAsBool(String propertyName); + Boolean getAsBool(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as an integer. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed + * @throws ClassCastException if the value is not an integer */ - Integer getAsInt(String propertyName); + Integer getAsInt(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a long. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed + * @throws ClassCastException if the value is not a long */ - Long getAsLong(String propertyName); + Long getAsLong(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a float. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed + * @throws ClassCastException if the value is not a float */ - Float getAsFloat(String propertyName); + Float getAsFloat(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a double. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed + * @throws ClassCastException if the value is not a double */ - Double getAsDouble(String propertyName); + Double getAsDouble(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get or add the specified element data to the internal map. diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java index f8758e126..1cb9dee48 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java @@ -191,7 +191,7 @@ public T get(TypedKey key) { @Override public ElementData get(String key) { if (processed.get() == false) { - throw new RuntimeException("This instance has not yet been processed"); + throw new IllegalStateException("This instance has not yet been processed"); } return data.get(new TypedKeyDefault(key, ElementData.class)); From e52c0adea7638febffd2a28cc50a7eba99b98a60 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:04:02 +0000 Subject: [PATCH 26/35] Test: getWhere leaves out data that is not there rather than throwing An element that did not run, or a property with no value, is left out of getWhere's map. Evidence that is missing or of another type gives no value from tryGetEvidence. exception-handling.md line 57 at 706de1f0: "The general guidance is that these functions and accessors SHOULD only throw errors/exceptions if the failure is likely due to something that the API user has done wrong." (CO-27) --- .../FlowDataAccessorErrorTests.java | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataAccessorErrorTests.java diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataAccessorErrorTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataAccessorErrorTests.java new file mode 100644 index 000000000..0da3dcc19 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataAccessorErrorTests.java @@ -0,0 +1,140 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.FlowData; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * FlowData's accessors throw only for something the caller did wrong. Data + * that is simply not there, because an element did not run or gave no + * value, is reported as absent rather than as an error. See + * exception-handling.md line 57: + * "The general guidance is that these functions and accessors SHOULD only + * throw errors/exceptions if the failure is likely due to something that + * the API user has done wrong." + */ +public class FlowDataAccessorErrorTests { + + private final ILoggerFactory loggerFactory = name -> mock(Logger.class); + + /** + * An element with one available property, "value". When it runs it + * adds Element Data holding the given value; when writes is false it + * adds nothing, as an element that did not run. + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static FlowElement element(boolean writes, Object value) + throws Exception { + FlowElement element = mock(FlowElement.class); + ElementPropertyMetaData property = new ElementPropertyMetaDataDefault( + "value", element, "test", String.class, true); + doReturn("props").when(element).getElementDataKey(); + doReturn(Collections.singletonList(property)) + .when(element).getProperties(); + when(element.hasLoadedProperties()).thenReturn(true); + doAnswer(invocation -> { + if (writes) { + FlowData flowData = invocation.getArgument(0); + ElementData data = mock(ElementData.class); + when(data.get(anyString())).thenReturn(value); + flowData.getOrAdd("props", d -> data); + } + return null; + }).when(element).process(any(FlowData.class)); + return element; + } + + private Map where(FlowElement element) + throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(element) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + return flowData.getWhere(p -> true); + } + } + + @Test + public void getWhere_ElementThatDidNotRun_LeavesItsPropertiesOut() + throws Exception { + assertTrue(where(element(false, null)).isEmpty()); + } + + @Test + public void getWhere_PropertyWithoutAValue_LeavesItOut() + throws Exception { + assertTrue(where(element(true, null)).isEmpty()); + } + + @Test + public void getWhere_PropertyWithAValue_GivesIt() throws Exception { + assertEquals(Collections.singletonMap("props.value", "x"), + where(element(true, "x"))); + } + + @Test + public void absentData_IsNullOrNoValue_NotAnError() throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(element(false, null)) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.number", "not a number"); + flowData.process(); + // get by key or by type throws for data that no element added, + // which FlowDataTests and the TypedKeyMap tests expect, as + // asking for it is taken as the caller's mistake. Evidence that + // is missing or of another type is not. + assertFalse(flowData.tryGetEvidence("query.number", Integer.class) + .hasValue()); + assertFalse(flowData.tryGetEvidence("query.missing", String.class) + .hasValue()); + } + } + + /** + * Keeps the List import, which the element's properties use. + */ + @SuppressWarnings("unused") + private static List unused; +} From cab1d748bec4069133cea1a1fe39d49e65b562a0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:08:36 +0000 Subject: [PATCH 27/35] Fix: leave out of getWhere the data that is not there, and say what get throws getWhere left a NullPointerException or NoSuchElementException to the caller when an element had not run or a property had no value, which is not the caller's mistake. It now leaves both out, and still throws IllegalStateException before processing. The Javadoc of get no longer promises null for missing data: get by key throws NoSuchElementException and get by type throws PipelineDataException, as FlowDataTests and the TypedKeyMap tests expect, and both are declared. exception-handling.md line 57 at 706de1f0: "The general guidance is that these functions and accessors SHOULD only throw errors/exceptions if the failure is likely due to something that the API user has done wrong." (CO-27) --- .../fiftyone/pipeline/core/data/FlowData.java | 31 +++++++++++++------ .../core/flowelements/FlowDataDefault.java | 25 ++++++++++++--- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java index 46adb7cf2..3bacd9da4 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java @@ -131,30 +131,37 @@ public interface FlowData extends AutoCloseable { * Get the {@link ElementData} for the requested aspect. * @param key the aspect to get * @param type type of the aspect - * @return aspect's {@link ElementData} or null if not present + * @return aspect's {@link ElementData} * @throws IllegalStateException if this FlowData has not been processed * @throws IllegalArgumentException if the key is null + * @throws java.util.NoSuchElementException if no element added data + * under the key */ T get(TypedKey key) - throws IllegalStateException, IllegalArgumentException; + throws IllegalStateException, IllegalArgumentException, + java.util.NoSuchElementException; /** * Get the {@link ElementData} for the requested data key. * @param key to the aspect to get - * @return aspect's {@link ElementData} or null if not present + * @return aspect's {@link ElementData} * @throws IllegalStateException if this FlowData has not been processed + * @throws java.util.NoSuchElementException if no element added data + * under the key */ - ElementData get(String key) throws IllegalStateException; + ElementData get(String key) + throws IllegalStateException, java.util.NoSuchElementException; /** * Get the {@link ElementData} for the requested {@link ElementData} - * implementation. If multiple instances of the type exist then an exception - * is thrown. + * implementation. * @param type the type of {@link ElementData} to return * @param the type of {@link ElementData} to return - * @return aspect's {@link ElementData} or null if the type is not present + * @return aspect's {@link ElementData} + * @throws PipelineDataException if no data of the type is present, or + * more than one instance is */ - T get(Class type); + T get(Class type) throws PipelineDataException; /** * Get the ElementData for the requested {@link FlowElement}. This method @@ -291,12 +298,16 @@ T getOrAdd( Iterable elementDataAsIterable(); /** - * Get all element data values that match the specified predicate. + * Get all element data values that match the specified predicate. The + * properties of an element that added no data, and properties with no + * value, are left out. * @param matcher if a property passed to this matcher returns true then it * will be included in the results * @return all the element data values that match the matcher + * @throws IllegalStateException if this FlowData has not been processed */ - Map getWhere(PropertyMatcher matcher); + Map getWhere(PropertyMatcher matcher) + throws IllegalStateException; /** * Generate a {@link DataKey} that contains the evidence data from this diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java index 1cb9dee48..7aa1b76d2 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java @@ -343,8 +343,21 @@ public Iterable elementDataAsIterable() { @Override @SuppressWarnings("rawtypes") public Map getWhere(PropertyMatcher matcher) { + if (processed.get() == false) { + throw new IllegalStateException( + "This instance has not yet been processed."); + } Map map = new HashMap<>(); for (FlowElement element : pipeline.getFlowElements()) { + // An element that did not run, for example because an earlier + // one stopped processing, has no data, and a property may have + // no value. Neither is the caller's mistake, so both are left + // out rather than thrown. See + // https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/features/exception-handling.md#L57 + if (data.containsKey(element.getElementDataKey()) == false) { + continue; + } + ElementData elementData = get(element.getElementDataKey()); // We know the property has extends ElementPropertyMetaData as it // is a constraint of the FlowElement interface, so don't check this // cast @@ -352,10 +365,14 @@ public Map getWhere(PropertyMatcher matcher) { ElementPropertyMetaData property = (ElementPropertyMetaData)propertyObject; if (property.isAvailable() && matcher.isMatch(property)) { - map.put( - element.getElementDataKey() + "." + property.getName(), - get(element.getElementDataKey()) - .get(property.getName().toLowerCase()).toString()); + Object value = + elementData.get(property.getName().toLowerCase()); + if (value != null) { + map.put( + element.getElementDataKey() + "." + + property.getName(), + value.toString()); + } } } } From 231737530d73669a915e4ca6ec486e777106765c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:11:29 +0000 Subject: [PATCH 28/35] Test: every builder setter in the pipeline packages gives its default value For the Pipeline builder, the engine builders and the data file configuration builder, the Cloud Request Engine builder, the usage sharing and 51Degrees Pipeline builders, and the translation engine builder. The JavaScript Builder's builder already gives every default. reference-implementation-notes.md line 79 at 706de1f0: "In all cases, there SHOULD be some mechanism to specify the default values alongside configuration property setters." (CO-65) --- .../BuilderDefaultValueTests.java | 68 +++++++++++++++++ .../BuilderDefaultValueTests.java | 73 +++++++++++++++++++ .../BuilderDefaultValueTests.java | 70 ++++++++++++++++++ .../BuilderDefaultValueTests.java | 73 +++++++++++++++++++ .../BuilderDefaultValueTests.java | 68 +++++++++++++++++ 5 files changed, 352 insertions(+) create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/BuilderDefaultValueTests.java create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/BuilderDefaultValueTests.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/BuilderDefaultValueTests.java create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/BuilderDefaultValueTests.java create mode 100644 pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/BuilderDefaultValueTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..bfde8d476 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,68 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Every configuration setter of the Cloud Request Engine builder gives its + * default value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void cloudRequestEngineBuilderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), + missingDefaults(CloudRequestEngineBuilder.class)); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..f9159208e --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the Pipeline builder gives its default + * value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + /** + * The public setters declared by the builders that do not carry a + * {@link DefaultValue}. + */ + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void pipelineBuilderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + PipelineBuilderBase.class, + PipelineBuilder.class)); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..8dab0cd98 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Every configuration setter of the usage sharing and 51Degrees Pipeline + * builders gives its default value beside it, with {@link DefaultValue}. + * See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void builderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + ShareUsageBuilderBase.class, + FiftyOnePipelineBuilder.class)); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..b29c14985 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationBuilderBase; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the engine builders gives its default + * value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void engineBuilderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + AspectEngineBuilderBase.class, + OnPremiseAspectEngineBuilderBase.class, + SingleFileAspectEngineBuilderBase.class, + CloudPipelineBuilderBase.class, + DataFileConfigurationBuilderBase.class)); + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..4d28b2448 --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,68 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Every configuration setter of the translation engine builder gives its + * default value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void translationEngineBuilderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), + missingDefaults(TranslationEngineBuilder.class)); + } +} From 5f5c8d2d2a06d0ede8d4673ee30cc2c41d536e60 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:18:50 +0000 Subject: [PATCH 29/35] Fix: give the default value beside every builder setter in the pipeline packages @DefaultValue on the 41 setters that had none, taken from the constant or field the builder or its configuration falls back to, or saying who sets it where it is not a value here. The randomisation setters in milliseconds gave the default in seconds, and three Cloud Request Engine endpoint defaults left out the slash after the base URL, so those are corrected. reference-implementation-notes.md line 79 at 706de1f0: "In all cases, there SHOULD be some mechanism to specify the default values alongside configuration property setters." (CO-65) --- .../flowelements/CloudRequestEngineBuilder.java | 7 ++++--- .../core/flowelements/PipelineBuilderBase.java | 3 +++ .../flowelements/FiftyOnePipelineBuilder.java | 2 ++ .../flowelements/ShareUsageBuilderBase.java | 2 ++ .../DataFileConfigurationBuilderBase.java | 16 ++++++++++++++++ .../flowelements/AspectEngineBuilderBase.java | 5 +++++ .../flowelements/CloudPipelineBuilderBase.java | 12 ++++++++++++ .../OnPremiseAspectEngineBuilderBase.java | 2 ++ .../SingleFileAspectEngineBuilderBase.java | 5 ++++- .../flowelements/TranslationEngineBuilder.java | 4 ++++ 10 files changed, 54 insertions(+), 4 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index db9f39443..1e57a166d 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -145,6 +145,7 @@ protected CloudRequestEngine newEngine(List properties) throws Exception * @param state the copy, or null to discover when building * @return this builder */ + @DefaultValue("None - discovered when the engine is built") public CloudRequestEngineBuilder setDiscoveryState( CloudDiscoveryState state) { this.discoveryState = state; @@ -271,7 +272,7 @@ public CloudRequestEngineBuilder setEndpoint(String uri) { * @param uri data endpoint * @return this builder */ - @DefaultValue(Constants.END_POINT_DEFAULT + "{resourcekey}.json") + @DefaultValue(Constants.END_POINT_DEFAULT + "/{resourcekey}.json") public CloudRequestEngineBuilder setDataEndpoint(String uri) { dataEndpoint = uri; return this; @@ -285,7 +286,7 @@ public CloudRequestEngineBuilder setDataEndpoint(String uri) { * @param uri properties endpoint * @return this builder */ - @DefaultValue(Constants.END_POINT_DEFAULT + "accessibleproperties") + @DefaultValue(Constants.END_POINT_DEFAULT + "/accessibleproperties") public CloudRequestEngineBuilder setPropertiesEndpoint(String uri) { propertiesEndpoint = uri; return this; @@ -299,7 +300,7 @@ public CloudRequestEngineBuilder setPropertiesEndpoint(String uri) { * @param uri evidence keys endpoint * @return this builder */ - @DefaultValue(Constants.END_POINT_DEFAULT + "evidencekeys") + @DefaultValue(Constants.END_POINT_DEFAULT + "/evidencekeys") public CloudRequestEngineBuilder setEvidenceKeysEndpoint(String uri) { evidenceKeysEndpoint = uri; return this; diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java index 147c7dc1a..778acb7ce 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.core.flowelements; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.core.Constants; import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.data.FlowData; @@ -186,6 +187,7 @@ protected void addServicesToPipeline(Pipeline pipeline) { * @return this builder */ @SuppressWarnings("unchecked") + @DefaultValue(booleanValue = Constants.DEFAULT_AUTO_CLOSE_ELEMENTS) public T setAutoCloseElements(boolean autoClose) { this.autoCloseElements = autoClose; return (T) this; @@ -201,6 +203,7 @@ public T setAutoCloseElements(boolean autoClose) { * @return this builder */ @SuppressWarnings("unchecked") + @DefaultValue(booleanValue = Constants.DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS) public T setSuppressProcessException(boolean suppress) { this.suppressProcessExceptions = suppress; return (T) this; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java index 83322e115..a111b1723 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.services.HttpClientDefault; @@ -57,6 +58,7 @@ public FiftyOnePipelineBuilder(ILoggerFactory loggerFactory) { * @param enabled true to enable usage sharing, false to disable * @return this builder */ + @DefaultValue(booleanValue = Constants.SHARE_USAGE_DEFAULT_ENABLED) public FiftyOnePipelineBuilder setShareUsage(boolean enabled) { this.shareUsageEnabled = enabled; return this; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index 1ab56e3f9..293f5704d 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -355,6 +355,7 @@ public int getMaximumQueueSize() { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.SHARE_USAGE_DEFAULT_ADD_TIMEOUT) public ShareUsageBuilderBase setAddTimeout(int milliseconds) { addTimeout = milliseconds; return this; @@ -385,6 +386,7 @@ public ShareUsageBuilderBase setAddTimeoutMillis(int milliseconds) { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.SHARE_USAGE_DEFAULT_TAKE_TIMEOUT) public ShareUsageBuilderBase setTakeTimeout(int milliseconds) { takeTimeout = milliseconds; return this; diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java index aeb9a1b29..9d7cb5818 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java @@ -22,6 +22,8 @@ package fiftyone.pipeline.engines.configuration; +import fiftyone.pipeline.annotations.DefaultValue; +import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.data.DataUpdateUrlFormatter; import fiftyone.pipeline.engines.services.DataUpdateService; @@ -71,6 +73,7 @@ public DataFileConfigurationBuilderBase() { * @param identifier the identifier to use * @return this builder instance */ + @DefaultValue("Default") @SuppressWarnings("unchecked") public TBuilder setDataFileIdentifier(String identifier) { this.identifier = identifier; @@ -83,6 +86,7 @@ public TBuilder setDataFileIdentifier(String identifier) { * @param url the URL to check for a new data file. * @return this builder instance. */ + @DefaultValue("51Degrees default, set by the engine's builder") @SuppressWarnings("unchecked") public TBuilder setDataUpdateUrl(String url) { dataUpdateUrlOverride = url; @@ -95,6 +99,7 @@ public TBuilder setDataUpdateUrl(String url) { * @param formatter the formatter to use * @return this builder instance */ + @DefaultValue("51Degrees default, set by the engine's builder") @SuppressWarnings("unchecked") public TBuilder setDataUpdateUrlFormatter( DataUpdateUrlFormatter formatter) { dataUpdateUrlFormatter = formatter; @@ -110,6 +115,7 @@ public TBuilder setDataUpdateUrlFormatter( DataUpdateUrlFormatter formatter) { * False otherwise * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_VERIFY_MD5) @SuppressWarnings("unchecked") public TBuilder setDataUpdateVerifyMd5(boolean verify) { dataUpdateVerifyMd5 = verify; @@ -123,6 +129,7 @@ public TBuilder setDataUpdateVerifyMd5(boolean verify) { * be decompressed. False otherwise * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_DECOMPRESS) @SuppressWarnings("unchecked") public TBuilder setDataUpdateDecompress(boolean decompress) { dataUpdateDecompress = decompress; @@ -137,6 +144,7 @@ public TBuilder setDataUpdateDecompress(boolean decompress) { * called on {@link DataUpdateService} * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_AUTOUPDATE_ENABLED) @SuppressWarnings("unchecked") public TBuilder setAutoUpdate(boolean enabled) { autoUpdateEnabled = enabled; @@ -155,6 +163,7 @@ public TBuilder setAutoUpdate(boolean enabled) { * @param enabled the cache configuration to use * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_WATCHER_ENABLED) @SuppressWarnings("unchecked") public TBuilder setDataFileSystemWatcher(boolean enabled) { dataFileSystemWatcherEnabled = enabled; @@ -175,6 +184,7 @@ public TBuilder setDataFileSystemWatcher(boolean enabled) { * @param pollingIntervalSeconds the number of seconds between checks * @return this builder instance */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT) @SuppressWarnings("unchecked") public TBuilder setUpdatePollingInterval(int pollingIntervalSeconds) { updatePollingIntervalSeconds = pollingIntervalSeconds; @@ -194,6 +204,7 @@ public TBuilder setUpdatePollingInterval(int pollingIntervalSeconds) { * @param pollingIntervalMillis the time between checks * @return this builder instance */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdatePollingInterval(long pollingIntervalMillis) { if (pollingIntervalMillis / 1000 > Integer.MAX_VALUE) { @@ -212,6 +223,7 @@ public TBuilder setUpdatePollingInterval(long pollingIntervalMillis) { * polling interval * @return this builder instance */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdateRandomisationMax(long maximumDeviationMillis){ if (maximumDeviationMillis / 1000 > Integer.MAX_VALUE) { @@ -228,6 +240,7 @@ public TBuilder setUpdateRandomisationMax(long maximumDeviationMillis){ * @param enabled whether to use the If-Modified-Since header * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_VERIFY_IF_MODIFIED_SINCE) @SuppressWarnings("unchecked") public TBuilder setVerifyIfModifiedSince(boolean enabled) { dataUpdateVerifyModifiedSince = enabled; @@ -241,6 +254,7 @@ public TBuilder setVerifyIfModifiedSince(boolean enabled) { * this file * @return this builder * */ + @DefaultValue("No license keys") @SuppressWarnings("unchecked") public TBuilder setDataUpdateLicenseKey(String key) { if (key == null) { @@ -265,6 +279,7 @@ public TBuilder setDataUpdateLicenseKey(String key) { * latest copy of the file * @return this builder */ + @DefaultValue(booleanValue = Constants.DEFAULT_UPDATE_ON_STARTUP) @SuppressWarnings("unchecked") public TBuilder setUpdateOnStartup(boolean enabled) { updateOnStartup = enabled; @@ -276,6 +291,7 @@ public TBuilder setUpdateOnStartup(boolean enabled) { * @param keys 51Degrees license keys * @return this builder * */ + @DefaultValue("No license keys") @SuppressWarnings("unchecked") public TBuilder setDataUpdateLicenseKeys(String[] keys) { Collections.addAll(licenseKeys, keys); diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java index aaba5f317..5fa9f0b3d 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java @@ -95,6 +95,7 @@ protected List getProperties() { * @param set The properties that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperties(Set set) { for (AspectPropertyMetaData property : set) { @@ -124,6 +125,7 @@ public TBuilder setProperty(String s) { * @param properties The properties that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperties(List properties) { for (String property : properties) { @@ -154,6 +156,7 @@ public TBuilder setProperties(String properties) { * @param aspectProperty the property that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperty(AspectPropertyMetaData aspectProperty) { tryAddProperty(aspectProperty.getName()); @@ -165,6 +168,7 @@ public TBuilder setProperty(AspectPropertyMetaData aspectProperty) { * @param configuration the configuration to use * @return this builder */ + @DefaultValue("No lazy loading") @SuppressWarnings("unchecked") public TBuilder setLazyLoading(LazyLoadingConfiguration configuration) { this.lazyLoadingConfig = configuration; @@ -177,6 +181,7 @@ public TBuilder setLazyLoading(LazyLoadingConfiguration configuration) { * @param cacheConfiguration the cache configuration to use * @return this builder */ + @DefaultValue("No results cache") @SuppressWarnings("unchecked") public TBuilder setCache(CacheConfiguration cacheConfiguration) { this.cacheConfig = cacheConfiguration; diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java index ffdf0e361..234f0099c 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.engines.flowelements; +import fiftyone.pipeline.annotations.DefaultValue; import org.slf4j.ILoggerFactory; import java.util.ArrayList; @@ -70,6 +71,8 @@ public CloudPipelineBuilderBase(ILoggerFactory loggerFactory) { * @param url endpoint URL * @return this builder */ + @DefaultValue("https://cloud.51degrees.com/api/v4/, the Cloud Request " + + "Engine's default") @SuppressWarnings("unchecked") public TBuilder setEndPoint(String url) { if (url.endsWith("/") == false) { @@ -84,6 +87,7 @@ public TBuilder setEndPoint(String url) { * @param url data URL * @return this builder */ + @DefaultValue("The endpoint followed by {resourcekey}.json") @SuppressWarnings("unchecked") public TBuilder setDataEndpoint(String url) { this.dataEndpoint = url; @@ -96,6 +100,7 @@ public TBuilder setDataEndpoint(String url) { * @param propertiesEndpoint properties URL * @return this builder */ + @DefaultValue("The endpoint followed by accessibleproperties") @SuppressWarnings("unchecked") public TBuilder setPropertiesEndpoint(String propertiesEndpoint) { this.propertiesEndpoint = propertiesEndpoint; @@ -108,6 +113,7 @@ public TBuilder setPropertiesEndpoint(String propertiesEndpoint) { * @param evidenceKeysEndpoint evidence keys URL * @return this builder */ + @DefaultValue("The endpoint followed by evidencekeys") @SuppressWarnings("unchecked") public TBuilder setEvidenceKeysEndpoint(String evidenceKeysEndpoint) { this.evidenceKeysEndpoint = evidenceKeysEndpoint; @@ -119,6 +125,8 @@ public TBuilder setEvidenceKeysEndpoint(String evidenceKeysEndpoint) { * @param key resource key * @return this builder */ + @DefaultValue("No default - a resource key or a license key must be " + + "supplied") @SuppressWarnings("unchecked") public TBuilder setResourceKey(String key) { this.resourceKey = key; @@ -134,6 +142,7 @@ public TBuilder setResourceKey(String key) { * @param key license key * @return this builder */ + @DefaultValue("No default") @SuppressWarnings("unchecked") public TBuilder setLicenseKey(String key) { this.licenseKey = key; @@ -148,6 +157,7 @@ public TBuilder setLicenseKey(String key) { * @param properties the fully qualified property names * @return this builder */ + @DefaultValue("None - required with a license key alone") @SuppressWarnings("unchecked") public TBuilder setRequestedProperties(List properties) { if (properties == null) { @@ -164,6 +174,7 @@ public TBuilder setRequestedProperties(List properties) { * @param properties comma separated fully qualified property names * @return this builder */ + @DefaultValue("None - required with a license key alone") public TBuilder setRequestedProperties(String properties) { if (properties == null) { throw new IllegalArgumentException( @@ -183,6 +194,7 @@ public TBuilder setRequestedProperties(String properties) { * @param cloudRequestOrigin the value to set the origin header to * @return this builder */ + @DefaultValue("None") @SuppressWarnings("unchecked") public TBuilder setCloudRequestOrigin(String cloudRequestOrigin) { this.cloudRequestOrigin = cloudRequestOrigin; diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java index 879f55f44..f743cd6a6 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java @@ -174,6 +174,8 @@ public TBuilder setTempDirPath(String dirPath) { * @param profile the performance profile for the engine to use * @return this engine builder instance */ + @DefaultValue("Set by each engine's builder, which gives it on its own " + + "setPerformanceProfile") public abstract TBuilder setPerformanceProfile(Constants.PerformanceProfiles profile); @Override diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java index 84df5a4f9..d389e1600 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java @@ -242,6 +242,7 @@ public TBuilder setDataFileSystemWatcher(boolean enabled) { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT) @SuppressWarnings("unchecked") public TBuilder setUpdatePollingInterval(int pollingIntervalSeconds) { dataFileBuilder.setUpdatePollingInterval(pollingIntervalSeconds); @@ -277,6 +278,7 @@ public TBuilder setUpdatePollingIntervalSeconds(int pollingIntervalSeconds) { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdatePollingInterval(long pollingIntervalMillis) { dataFileBuilder.setUpdatePollingInterval(pollingIntervalMillis); @@ -308,6 +310,7 @@ public TBuilder setUpdatePollingIntervalMillis(long pollingIntervalMillis) { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdateRandomisationMax(long maximumDeviationMillis) { dataFileBuilder.setUpdateRandomisationMax(maximumDeviationMillis); @@ -324,7 +327,7 @@ public TBuilder setUpdateRandomisationMax(long maximumDeviationMillis) { * polling interval * @return this builder */ - @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT) + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdateRandomisationMaxMillis(long maximumDeviationMillis) { dataFileBuilder.setUpdateRandomisationMax(maximumDeviationMillis); diff --git a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java index 3c460191d..39e157c75 100644 --- a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java +++ b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.translation.flowelements; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.annotations.ElementBuilder; import fiftyone.pipeline.translation.data.MissingTranslationBehavior; import fiftyone.pipeline.translation.data.TranslationData; @@ -80,6 +81,7 @@ public TranslationEngineBuilder(ILoggerFactory loggerFactory) { * @param sourceElementDataKey the source element data key * @return this builder */ + @DefaultValue("No default - the source element data key must be supplied") public TranslationEngineBuilder setSourceElementDataKey( String sourceElementDataKey) { this.sourceElementDataKey = sourceElementDataKey; @@ -93,6 +95,7 @@ public TranslationEngineBuilder setSourceElementDataKey( * @param language the language to translate to * @return this builder */ + @DefaultValue("None - the language is resolved from the evidence") public TranslationEngineBuilder setFixedLanguage(String language) { this.fixedLanguage = language; return this; @@ -103,6 +106,7 @@ public TranslationEngineBuilder setFixedLanguage(String language) { * @param behavior the missing-translation behaviour * @return this builder */ + @DefaultValue("ORIGINAL") public TranslationEngineBuilder setMissingTranslationBehavior( MissingTranslationBehavior behavior) { this.behavior = behavior; From 44eebc4542d608af87491ca9f260307d96fbbf7c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:20:05 +0000 Subject: [PATCH 30/35] Test: only one thread at a time reads the IP Intelligence native results The same check as for Device Detection, for getValues and the property getters of IPIntelligenceDataHashDefault. thread-safety.md line 61 at 706de1f0: "Such Element Data MUST let only one thread at a time read from its results, for example by holding a lock for each read" (CO-49, and CO-48 at line 57) --- ...IntelligenceDataHashThreadSafetyTests.java | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashThreadSafetyTests.java diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashThreadSafetyTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashThreadSafetyTests.java new file mode 100644 index 000000000..fd0350a5b --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashThreadSafetyTests.java @@ -0,0 +1,198 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.MissingPropertyService; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.Logger; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Reads of finished IP Intelligence on-premise results are safe from + * several threads, because only one thread at a time reads from the native + * results. See + * thread-safety.md line 57 + * and + * line 61: + * "Such Element Data MUST let only one thread at a time read from its + * results". + *

+ * As in the Device Detection test, the native results are a mock that + * holds the first thread's read, and the second thread is checked from its + * state, so nothing depends on timing. + */ +public class IPIntelligenceDataHashThreadSafetyTests { + + private static final long DEADLINE_MILLIS = 60_000; + + private static final Class[] WEIGHTED_STRINGS = + new Class[] { List.class, IWeightedValue.class, String.class }; + + private static class Gate implements Answer { + final AtomicInteger inside = new AtomicInteger(); + final AtomicInteger most = new AtomicInteger(); + final CountDownLatch firstInside = new CountDownLatch(1); + final CountDownLatch release = new CountDownLatch(1); + volatile Thread first; + + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + String name = invocation.getMethod().getName(); + if (name.equals("close") || name.equals("delete") || + name.equals("finalize")) { + return null; + } + int now = inside.incrementAndGet(); + most.accumulateAndGet(now, Math::max); + try { + if (Thread.currentThread() == first && + firstInside.getCount() > 0) { + firstInside.countDown(); + assertTrue(release.await( + DEADLINE_MILLIS, TimeUnit.MILLISECONDS)); + } + Class type = invocation.getMethod().getReturnType(); + if (type == boolean.class) { + return true; + } + if (type == int.class || type == long.class) { + return 0; + } + if (type == String.class) { + return ""; + } + if (type == void.class) { + return null; + } + return mock(type); + } finally { + inside.decrementAndGet(); + } + } + } + + private IPIntelligenceDataHashDefault newData(Gate gate) { + FlowData flowData = mock(FlowData.class); + when(flowData.getPipeline()).thenReturn(mock(Pipeline.class)); + IPIntelligenceDataHashDefault data = new IPIntelligenceDataHashDefault( + mock(Logger.class), + flowData, + mock(IPIntelligenceOnPremiseEngine.class), + mock(MissingPropertyService.class)); + data.setResults(mock(ResultsIpiSwig.class, gate)); + return data; + } + + private void assertOneReaderAtATime( + Consumer firstRead, + Consumer secondRead) throws Exception { + Gate gate = new Gate(); + IPIntelligenceDataHashDefault data = newData(gate); + AtomicReference failure = new AtomicReference<>(); + + Thread first = new Thread(() -> run(firstRead, data, failure)); + gate.first = first; + first.start(); + assertTrue("The first read did not reach the native results.", + gate.firstInside.await(DEADLINE_MILLIS, TimeUnit.MILLISECONDS)); + + Thread second = new Thread(() -> run(secondRead, data, failure)); + second.start(); + long deadline = System.currentTimeMillis() + DEADLINE_MILLIS; + while (gate.most.get() < 2 && + second.isAlive() && + waitingForTheLock(second) == false) { + assertTrue("The second read neither waited nor read.", + System.currentTimeMillis() < deadline); + Thread.sleep(1); + } + + gate.release.countDown(); + first.join(DEADLINE_MILLIS); + second.join(DEADLINE_MILLIS); + assertNull("A read failed: " + failure.get(), failure.get()); + assertEquals( + "Two threads read from the native results at once.", + 1, + gate.most.get()); + } + + private static void run( + Consumer read, + IPIntelligenceDataHashDefault data, + AtomicReference failure) { + try { + read.accept(data); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + } + + private static boolean waitingForTheLock(Thread thread) { + if (thread.getState() != Thread.State.BLOCKED) { + return false; + } + StackTraceElement[] stack = thread.getStackTrace(); + return stack.length > 0 && + stack[0].getClassName().startsWith("fiftyone.ipintelligence."); + } + + @Test + public void getValues_WaitsForAnotherGetValues() throws Exception { + assertOneReaderAtATime( + d -> d.getValues("RegisteredName", WEIGHTED_STRINGS), + d -> d.getValues("RegisteredName", WEIGHTED_STRINGS)); + } + + @Test + public void getValues_WaitsForAPropertyGetter() throws Exception { + assertOneReaderAtATime( + d -> d.getCountryCodesGeographical(), + d -> d.getValues("RegisteredName", WEIGHTED_STRINGS)); + } + + @Test + public void propertyGetter_WaitsForGetValues() throws Exception { + assertOneReaderAtATime( + d -> d.getValues("RegisteredName", WEIGHTED_STRINGS), + d -> d.getCountryCodesGeographical()); + } +} From 3e88f267782cb93bd429281f71af550e447ee3ca Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:21:12 +0000 Subject: [PATCH 31/35] Fix: hold the results lock for every read of the IP Intelligence native results IPIntelligenceDataBaseOnPremise gives subclasses its lock as getResultsLock and holds it while it asks whether a property is available. IPIntelligenceDataHashDefault holds it in propertyIsAvailable and in the public getValues, whose body moves to readValues unchanged. thread-safety.md line 61 at 706de1f0: "Such Element Data MUST let only one thread at a time read from its results, for example by holding a lock for each read" (CO-49, and CO-48 at line 57) --- .../IPIntelligenceDataHashDefault.java | 27 ++++++++++++++++--- .../IPIntelligenceDataBaseOnPremise.java | 21 ++++++++++++++- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java index 6d1300fd8..d1aa34fb5 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java @@ -142,12 +142,14 @@ private ResultsIpiSwig getResultsContainingProperty(String propertyName) { @Override protected boolean propertyIsAvailable(String propertyName) { checkState(); - for (ResultsIpiSwig results : resultsList) { - if (results.containsProperty(propertyName)) { - return true; + synchronized (getResultsLock()) { + for (ResultsIpiSwig results : resultsList) { + if (results.containsProperty(propertyName)) { + return true; + } } + return false; } - return false; } private static < @@ -180,10 +182,27 @@ void populateWeightedValues( } } + /** + * Get the values of a property as a weighted list. This is public, so it + * holds the results lock itself, as every read from the native results + * does. See the + * thread safety specification. + * @param propertyName name of the property to get values for + * @param parametrizedTypes types parametrizing the value, starting with + * {@link List} + * @return values as a list + */ @Override public AspectPropertyValue> getValues(String propertyName, Class[] parametrizedTypes) { checkState(); + synchronized (getResultsLock()) { + return readValues(propertyName, parametrizedTypes); + } + } + + private AspectPropertyValue> readValues(String propertyName, + Class[] parametrizedTypes) { AspectPropertyValue> result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/IPIntelligenceDataBaseOnPremise.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/IPIntelligenceDataBaseOnPremise.java index 32b2729e2..588dd8e59 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/IPIntelligenceDataBaseOnPremise.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/IPIntelligenceDataBaseOnPremise.java @@ -73,6 +73,21 @@ protected IPIntelligenceDataBaseOnPremise( primitiveTypes = getPrimitiveTypeMap(); } + /** + * The lock that every read from the underlying results holds, so that + * only one thread at a time reads from them. A subclass holds it in each + * method that reads the results and can be called without going through + * {@link #tryGetValue(String, Class, Class[])}. The lock is reentrant, so + * a read that calls another read holds it once more. See the + * thread safety specification: + * "Such Element Data MUST let only one thread at a time read from its + * results, for example by holding a lock for each read". + * @return the lock for reads from the results + */ + protected final Object getResultsLock() { + return getLock; + } + /** * Determine whether a property is available to return values from the * underlying results. @@ -281,7 +296,11 @@ protected TryGetResult tryGetValue( // If the complete set of values has not been populated // then we don't want to retrieve values for all // properties so just get the one we want. - if (propertyIsAvailable(key)) { + boolean available; + synchronized (getLock) { + available = propertyIsAvailable(key); + } + if (available) { if (type.equals(Object.class)) { type = (Class)AspectPropertyValue.class; parameterisedTypes = toWeightedListType(getPropertyType(key), false); From b4b0df1a35cfeae7f004920ba1984de7a53e0592 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:27:29 +0000 Subject: [PATCH 32/35] Test: every Device Detection and IP Intelligence builder setter gives its default For the on-premise Pipeline and engine builders of both products, and the country code translation engine builder. reference-implementation-notes.md line 79 at 706de1f0: "In all cases, there SHOULD be some mechanism to specify the default values alongside configuration property setters." (CO-65) --- .../BuilderDefaultValueTests.java | 70 +++++++++++++++++++ .../translation/BuilderDefaultValueTests.java | 63 +++++++++++++++++ .../BuilderDefaultValueTests.java | 70 +++++++++++++++++++ 3 files changed, 203 insertions(+) create mode 100644 device-detection/device-detection/src/test/java/fiftyone/devicedetection/BuilderDefaultValueTests.java create mode 100644 ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/BuilderDefaultValueTests.java create mode 100644 ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/BuilderDefaultValueTests.java diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/BuilderDefaultValueTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/BuilderDefaultValueTests.java new file mode 100644 index 000000000..3d04cdbfb --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/BuilderDefaultValueTests.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the Device Detection on-premise builders + * gives its default value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void builderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + DeviceDetectionOnPremisePipelineBuilder.class, + DeviceDetectionHashEngineBuilder.class)); + } +} diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/BuilderDefaultValueTests.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/BuilderDefaultValueTests.java new file mode 100644 index 000000000..508924ca5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/BuilderDefaultValueTests.java @@ -0,0 +1,63 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.translation; + +import fiftyone.ipintelligence.translation.flowelements.CountryCodeTranslationEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the country code translation engine + * builder gives its default value beside it, with {@link DefaultValue}. + * See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + @Test + public void builderSetters_GiveTheirDefaults() { + List missing = new ArrayList<>(); + for (Method method : + CountryCodeTranslationEngineBuilder.class.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + assertEquals(new ArrayList(), missing); + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/BuilderDefaultValueTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/BuilderDefaultValueTests.java new file mode 100644 index 000000000..96ee86770 --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/BuilderDefaultValueTests.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the IP Intelligence on-premise builders + * gives its default value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void builderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + IPIntelligenceOnPremisePipelineBuilder.class, + IPIntelligenceOnPremiseEngineBuilder.class)); + } +} From 55661b71c22512209bf8953cd56653f3f721e8f9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:29:33 +0000 Subject: [PATCH 33/35] Fix: give the default beside every Device Detection and IP Intelligence builder setter @DefaultValue on the 42 setters that had none, from the constants the engine builders fall back to. Both on-premise Pipeline builders said update on startup defaults to enabled, but an unset value falls back to Constants.DEFAULT_UPDATE_ON_STARTUP, which is false, so their Javadoc now says disabled. reference-implementation-notes.md line 79 at 706de1f0: "In all cases, there SHOULD be some mechanism to specify the default values alongside configuration property setters." (CO-65) --- .../DeviceDetectionHashEngineBuilder.java | 2 ++ ...viceDetectionOnPremisePipelineBuilder.java | 25 ++++++++++++++++++- .../IPIntelligenceOnPremiseEngineBuilder.java | 2 ++ .../CountryCodeTranslationEngineBuilder.java | 2 ++ ...PIntelligenceOnPremisePipelineBuilder.java | 20 ++++++++++++++- 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java index 1de52bba8..64537c183 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java @@ -156,6 +156,7 @@ public DeviceDetectionHashEngineBuilder setPerformanceProfile( } } + @DefaultValue("Balanced") @Override public DeviceDetectionHashEngineBuilder setPerformanceProfile( PerformanceProfiles profile) { @@ -285,6 +286,7 @@ public DeviceDetectionHashEngineBuilder setUsePredictiveGraph(boolean use) { return this; } + @DefaultValue("No results cache, and none can be set for this engine") @Override public DeviceDetectionHashEngineBuilder setCache( CacheConfiguration cacheConfiguration) { diff --git a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java index 983a5eaa9..2aed9d0a6 100644 --- a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java +++ b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java @@ -24,6 +24,7 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.Constants; @@ -138,6 +139,8 @@ DeviceDetectionOnPremisePipelineBuilder setEngineData(byte[] data) { * @param enabled True to enable usage sharing. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = fiftyone.pipeline.engines.fiftyone + .flowelements.Constants.SHARE_USAGE_DEFAULT_ENABLED) public DeviceDetectionOnPremisePipelineBuilder setShareUsage( boolean enabled) { shareUsageEnabled = enabled; @@ -152,6 +155,7 @@ public DeviceDetectionOnPremisePipelineBuilder setShareUsage( * @param enabled True to enable auto update. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_AUTOUPDATE_ENABLED) public DeviceDetectionOnPremisePipelineBuilder setAutoUpdate( boolean enabled) { autoUpdateEnabled = enabled; @@ -166,6 +170,7 @@ public DeviceDetectionOnPremisePipelineBuilder setAutoUpdate( * @param enabled True to enable file system watcher. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_WATCHER_ENABLED) public DeviceDetectionOnPremisePipelineBuilder setDataFileSystemWatcher( boolean enabled) { dataFileSystemWatcher = enabled; @@ -177,6 +182,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataFileSystemWatcher( * @param dataUpdateService an instance of a dataUpdateService * @return This builder instance. */ + @DefaultValue("The DataUpdateService given to the constructor") public DeviceDetectionOnPremisePipelineBuilder setDataUpdateService( DataUpdateService dataUpdateService) { this.dataUpdateService = dataUpdateService; @@ -185,7 +191,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateService( /** * Enable/Disable update on startup. - * Defaults to enabled. + * Defaults to disabled. * If enabled, the auto update system will be used to check for * an update before the device detection engine is created. * If an update is available, it will be downloaded and applied @@ -194,6 +200,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateService( * @param enabled True to enable update on startup. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_UPDATE_ON_STARTUP) public DeviceDetectionOnPremisePipelineBuilder setDataUpdateOnStartup( boolean enabled) { dataUpdateOnStartup = enabled; @@ -207,6 +214,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateOnStartup( * @param pollingIntervalSeconds The number of seconds between checks. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT) public DeviceDetectionOnPremisePipelineBuilder setUpdatePollingInterval( int pollingIntervalSeconds) { @@ -220,6 +228,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUpdatePollingInterval( * @param pollingIntervalMillis The number of milliseconds between checks. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT * 1000) public DeviceDetectionOnPremisePipelineBuilder setUpdatePollingIntervalMillis( long pollingIntervalMillis) { updatePollingInterval = pollingIntervalMillis; @@ -234,6 +243,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUpdatePollingIntervalMillis( * polling interval in seconds. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT) public DeviceDetectionOnPremisePipelineBuilder setUpdateRandomisationMax( int randomisationMaxSeconds) { updateRandomisationMax = (long)randomisationMaxSeconds * 1000; @@ -248,6 +258,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUpdateRandomisationMax( * polling interval in milliseconds. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) public DeviceDetectionOnPremisePipelineBuilder setUpdateRandomisationMaxMillis( long randomisationMaxMillis) { updateRandomisationMax = randomisationMaxMillis; @@ -261,6 +272,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUpdateRandomisationMaxMillis( * @param key The license key. * @return This builder instance. */ + @DefaultValue("No license key") public DeviceDetectionOnPremisePipelineBuilder setDataUpdateLicenseKey( String key) { dataUpdateLicenseKey = key; @@ -273,6 +285,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateLicenseKey( * @param profile The performance profile to use. * @return This builder instance. */ + @DefaultValue("Balanced") public DeviceDetectionOnPremisePipelineBuilder setPerformanceProfile( Constants.PerformanceProfiles profile) { performanceProfile = profile; @@ -286,6 +299,7 @@ public DeviceDetectionOnPremisePipelineBuilder setPerformanceProfile( * @param concurrency expected concurrent accesses * @return this builder */ + @DefaultValue("The result of Runtime#getRuntime().getAvailableProcessors()") public DeviceDetectionOnPremisePipelineBuilder setConcurrency(int concurrency) { this.concurrency = concurrency; return this; @@ -300,6 +314,7 @@ public DeviceDetectionOnPremisePipelineBuilder setConcurrency(int concurrency) { * @param difference to allow * @return this builder */ + @DefaultValue(intValue = 0) public DeviceDetectionOnPremisePipelineBuilder setDifference(int difference) { this.difference = difference; return this; @@ -316,6 +331,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDifference(int difference) { * considered valid * @return this builder */ + @DefaultValue(booleanValue = false) public DeviceDetectionOnPremisePipelineBuilder setAllowUnmatched(boolean allow) { this.allowUnmatched = allow; return this; @@ -328,6 +344,7 @@ public DeviceDetectionOnPremisePipelineBuilder setAllowUnmatched(boolean allow) * @param drift to set * @return this builder */ + @DefaultValue(intValue = 0) public DeviceDetectionOnPremisePipelineBuilder setDrift(int drift) { this.drift = drift; return this; @@ -345,6 +362,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDrift(int drift) { * @param use true if the performance graph should be used * @return this builder */ + @DefaultValue(booleanValue = false) public DeviceDetectionOnPremisePipelineBuilder setUsePerformanceGraph(boolean use) { this.usePerformanceGraph = use; return this; @@ -361,6 +379,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUsePerformanceGraph(boolean us * @param use true if the predictive graph should be used * @return this builder */ + @DefaultValue(booleanValue = true) public DeviceDetectionOnPremisePipelineBuilder setUsePredictiveGraph(boolean use) { this.usePredictiveGraph = use; return this; @@ -372,6 +391,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUsePredictiveGraph(boolean use * @param property the property that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") public DeviceDetectionOnPremisePipelineBuilder setProperty(String property) { this.properties.add(property); return this; @@ -386,6 +406,7 @@ public DeviceDetectionOnPremisePipelineBuilder setProperty(String property) { * @param url the URL to check for a new data file * @return this builder */ + @DefaultValue("51Degrees default, set by the engine's builder") public DeviceDetectionOnPremisePipelineBuilder setDataUpdateUrl(String url) { dataUpdateUrl = url; return this; @@ -402,6 +423,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateUrl(String url) { * False otherwise * @return this builder */ + @DefaultValue(booleanValue = Constants.DEFAULT_VERIFY_MD5) public DeviceDetectionOnPremisePipelineBuilder setDataUpdateVerifyMd5(Boolean verify) { dataUpdateVerifyMd5 = verify; return this; @@ -415,6 +437,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateVerifyMd5(Boolean ve * @param formatter the formatter to use * @return this builder */ + @DefaultValue("51Degrees default, set by the engine's builder") public DeviceDetectionOnPremisePipelineBuilder setDataUpdateUrlFormatter(DataUpdateUrlFormatter formatter) { dataUpdateUrlFormatter = formatter; return this; diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java index cf0b071fe..cd7a9010e 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java @@ -159,6 +159,7 @@ public IPIntelligenceOnPremiseEngineBuilder setPerformanceProfile( } } + @DefaultValue("Balanced") @Override public IPIntelligenceOnPremiseEngineBuilder setPerformanceProfile( PerformanceProfiles profile) { @@ -201,6 +202,7 @@ public IPIntelligenceOnPremiseEngineBuilder setConcurrency(int concurrency) { return this; } + @DefaultValue("No results cache, and none can be set for this engine") @Override public IPIntelligenceOnPremiseEngineBuilder setCache( CacheConfiguration cacheConfiguration) { diff --git a/ip-intelligence/ip-intelligence.translation/src/main/java/fiftyone/ipintelligence/translation/flowelements/CountryCodeTranslationEngineBuilder.java b/ip-intelligence/ip-intelligence.translation/src/main/java/fiftyone/ipintelligence/translation/flowelements/CountryCodeTranslationEngineBuilder.java index 0a23fe9aa..affeec009 100644 --- a/ip-intelligence/ip-intelligence.translation/src/main/java/fiftyone/ipintelligence/translation/flowelements/CountryCodeTranslationEngineBuilder.java +++ b/ip-intelligence/ip-intelligence.translation/src/main/java/fiftyone/ipintelligence/translation/flowelements/CountryCodeTranslationEngineBuilder.java @@ -24,6 +24,7 @@ import fiftyone.ipintelligence.translation.Constants; import fiftyone.ipintelligence.translation.data.CountryCodeTranslationData; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.annotations.ElementBuilder; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; @@ -61,6 +62,7 @@ public CountryCodeTranslationEngineBuilder(ILoggerFactory loggerFactory) { * @param sourceElementDataKey the IP Intelligence element data key * @return this builder */ + @DefaultValue(Constants.IP_INTELLIGENCE_KEY) public CountryCodeTranslationEngineBuilder setSourceElementDataKey( String sourceElementDataKey) { this.sourceElementDataKey = sourceElementDataKey; diff --git a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java index d93bb7784..8585e0c91 100644 --- a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java +++ b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java @@ -24,6 +24,7 @@ import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.Constants; @@ -114,6 +115,8 @@ IPIntelligenceOnPremisePipelineBuilder setFilename( * @param enabled True to enable usage sharing. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = fiftyone.pipeline.engines.fiftyone + .flowelements.Constants.SHARE_USAGE_DEFAULT_ENABLED) public IPIntelligenceOnPremisePipelineBuilder setShareUsage( boolean enabled) { shareUsageEnabled = enabled; @@ -128,6 +131,7 @@ public IPIntelligenceOnPremisePipelineBuilder setShareUsage( * @param enabled True to enable auto update. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_AUTOUPDATE_ENABLED) public IPIntelligenceOnPremisePipelineBuilder setAutoUpdate( boolean enabled) { autoUpdateEnabled = enabled; @@ -142,6 +146,7 @@ public IPIntelligenceOnPremisePipelineBuilder setAutoUpdate( * @param enabled True to enable file system watcher. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_WATCHER_ENABLED) public IPIntelligenceOnPremisePipelineBuilder setDataFileSystemWatcher( boolean enabled) { dataFileSystemWatcher = enabled; @@ -153,6 +158,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataFileSystemWatcher( * @param dataUpdateService an instance of a dataUpdateService * @return This builder instance. */ + @DefaultValue("The DataUpdateService given to the constructor") public IPIntelligenceOnPremisePipelineBuilder setDataUpdateService( DataUpdateService dataUpdateService) { this.dataUpdateService = dataUpdateService; @@ -161,7 +167,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateService( /** * Enable/Disable update on startup. - * Defaults to enabled. + * Defaults to disabled. * If enabled, the auto update system will be used to check for * an update before the IP Intelligence engine is created. * If an update is available, it will be downloaded and applied @@ -170,6 +176,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateService( * @param enabled True to enable update on startup. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_UPDATE_ON_STARTUP) public IPIntelligenceOnPremisePipelineBuilder setDataUpdateOnStartup( boolean enabled) { dataUpdateOnStartup = enabled; @@ -183,6 +190,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateOnStartup( * @param pollingIntervalSeconds The number of seconds between checks. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT) public IPIntelligenceOnPremisePipelineBuilder setUpdatePollingInterval( int pollingIntervalSeconds) { @@ -196,6 +204,7 @@ public IPIntelligenceOnPremisePipelineBuilder setUpdatePollingInterval( * @param pollingIntervalMillis The number of milliseconds between checks. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT * 1000) public IPIntelligenceOnPremisePipelineBuilder setUpdatePollingIntervalMillis( long pollingIntervalMillis) { updatePollingInterval = pollingIntervalMillis; @@ -210,6 +219,7 @@ public IPIntelligenceOnPremisePipelineBuilder setUpdatePollingIntervalMillis( * polling interval in seconds. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT) public IPIntelligenceOnPremisePipelineBuilder setUpdateRandomisationMax( int randomisationMaxSeconds) { updateRandomisationMax = (long)randomisationMaxSeconds * 1000; @@ -224,6 +234,7 @@ public IPIntelligenceOnPremisePipelineBuilder setUpdateRandomisationMax( * polling interval in milliseconds. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) public IPIntelligenceOnPremisePipelineBuilder setUpdateRandomisationMaxMillis( long randomisationMaxMillis) { updateRandomisationMax = randomisationMaxMillis; @@ -237,6 +248,7 @@ public IPIntelligenceOnPremisePipelineBuilder setUpdateRandomisationMaxMillis( * @param key The license key. * @return This builder instance. */ + @DefaultValue("No license key") public IPIntelligenceOnPremisePipelineBuilder setDataUpdateLicenseKey( String key) { dataUpdateLicenseKey = key; @@ -250,6 +262,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateLicenseKey( * @param profile The performance profile to use. * @return This builder instance. */ + @DefaultValue("Balanced") public IPIntelligenceOnPremisePipelineBuilder setPerformanceProfile( Constants.PerformanceProfiles profile) { performanceProfile = profile; @@ -263,6 +276,7 @@ public IPIntelligenceOnPremisePipelineBuilder setPerformanceProfile( * @param concurrency expected concurrent accesses * @return this builder */ + @DefaultValue("The result of Runtime#getRuntime().getAvailableProcessors()") public IPIntelligenceOnPremisePipelineBuilder setConcurrency(int concurrency) { this.concurrency = concurrency; return this; @@ -274,6 +288,7 @@ public IPIntelligenceOnPremisePipelineBuilder setConcurrency(int concurrency) { * @param property the property that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") public IPIntelligenceOnPremisePipelineBuilder setProperty(String property) { this.properties.add(property); return this; @@ -288,6 +303,7 @@ public IPIntelligenceOnPremisePipelineBuilder setProperty(String property) { * @param url the URL to check for a new data file * @return this builder */ + @DefaultValue("51Degrees default, set by the engine's builder") public IPIntelligenceOnPremisePipelineBuilder setDataUpdateUrl(String url) { dataUpdateUrl = url; return this; @@ -304,6 +320,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateUrl(String url) { * False otherwise * @return this builder */ + @DefaultValue(booleanValue = Constants.DEFAULT_VERIFY_MD5) public IPIntelligenceOnPremisePipelineBuilder setDataUpdateVerifyMd5(Boolean verify) { dataUpdateVerifyMd5 = verify; return this; @@ -317,6 +334,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateVerifyMd5(Boolean ver * @param formatter the formatter to use * @return this builder */ + @DefaultValue("51Degrees default, set by the engine's builder") public IPIntelligenceOnPremisePipelineBuilder setDataUpdateUrlFormatter(DataUpdateUrlFormatter formatter) { dataUpdateUrlFormatter = formatter; return this; From 8ad48e847171f53c34ef1fdf208b1e64b606b417 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 22:47:58 +0000 Subject: [PATCH 34/35] Test: a cached lazy cloud engine returns without waiting for the cloud response Lazy loading means process returns at once and the work is done on another thread. The check that keeps a 51Did out of the cache (caching.md line 144 at 706de1f0: "A cache MUST NOT store a result that holds such a value, and MUST NOT serve one that it finds.") must not make process wait for the answer. The test holds the answer back and reads the processing thread's stack, so no result depends on timing. Two more tests check that a lazy response is still cached, and that one with a fodid is not. --- .../CloudRequestEngineLazyCacheTests.java | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLazyCacheTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLazyCacheTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLazyCacheTests.java new file mode 100644 index 000000000..cbdfaffa5 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLazyCacheTests.java @@ -0,0 +1,223 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.configuration.LazyLoadingConfiguration; +import org.json.JSONArray; +import org.junit.jupiter.api.Test; + +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * A lazy loading Cloud Request Engine with a cache returns from process + * without waiting for the cloud service, and still keeps a response with a + * 51Did out of the cache. Lazy loading means "the Engine returns from process + * at once and the work is done on another thread", which the check of + * caching.md + * ("A cache MUST NOT store a result that holds such a value, and MUST NOT + * serve one that it finds.") must not undo. + *

+ * No result depends on timing: the cloud service's answer is held back until + * the test releases it, and the test reads the processing thread's stack to + * see whether it is waiting for that answer. + */ +public class CloudRequestEngineLazyCacheTests + extends CloudRequestEngineTestsBase { + + /** + * Long enough that a process which waits for the held back answer is + * still waiting when the test looks. + */ + private static final int LAZY_TIMEOUT_MILLIS = 600000; + + public CloudRequestEngineLazyCacheTests() throws MalformedURLException { + super(); + } + + /** + * Mock the cloud service so that each data request takes the next + * response in the list, and waits for the gate first. + */ + private void mockService( + final CountDownLatch called, + final CountDownLatch gate, + String... responses) throws Exception { + evidenceKeysResponse = + new JSONArray(Arrays.asList("query.User-Agent")).toString(); + configureMockedClient(); + final List remaining = new ArrayList<>(Arrays.asList(responses)); + doAnswer(invocation -> { + HttpURLConnection connection = invocation.getArgument(0); + String path = connection.getURL().getPath(); + if (path.endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (path.endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + called.countDown(); + gate.await(); + return remaining.remove(0); + }).when(httpClient).postData(any(), anyMap(), (byte[]) any()); + } + + private CloudRequestEngine buildLazyCachedEngine() throws Exception { + return new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .setLazyLoading(new LazyLoadingConfiguration(LAZY_TIMEOUT_MILLIS)) + .build(); + } + + private static boolean waitingForProcessing(Thread thread) { + for (StackTraceElement frame : thread.getStackTrace()) { + if (frame.getMethodName().equals("waitOnAllProcessFutures") || + frame.getClassName().endsWith("FutureTask") && + frame.getMethodName().equals("get")) { + return true; + } + } + return false; + } + + /** + * process returns while the cloud service has not answered. Before the + * fix the cache check read the response, which waited for it. + */ + @Test + public void process_LazyCachedEngineDoesNotWaitForTheService() + throws Exception { + CountDownLatch called = new CountDownLatch(1); + CountDownLatch gate = new CountDownLatch(1); + mockService(called, gate, jsonResponse); + CloudRequestEngine engine = buildLazyCachedEngine(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + final AtomicBoolean returned = new AtomicBoolean(false); + final AtomicReference failure = new AtomicReference<>(); + Thread processing = new Thread(() -> { + try { + data.process(); + returned.set(true); + } catch (Throwable t) { + failure.set(t); + } + }); + processing.start(); + called.await(); + // Look until process has returned, or is seen waiting for the + // answer that is held back. + boolean waited = false; + while (returned.get() == false && failure.get() == null && + processing.isAlive()) { + if (waitingForProcessing(processing)) { + waited = true; + break; + } + Thread.yield(); + } + gate.countDown(); + processing.join(); + assertNull(failure.get()); + assertFalse(waited, + "process waited for the cloud service's answer."); + assertTrue(returned.get()); + assertEquals(jsonResponse, + data.get(CloudRequestData.class).getJsonResponse()); + } + } + + /** + * With lazy loading, a response without a 51Did is still cached, so the + * second identical request is served from the cache. + */ + @Test + public void process_LazyResponseWithoutFodIdIsCached() throws Exception { + CountDownLatch gate = new CountDownLatch(0); + String first = "{'device':{'value':'1'}}"; + mockService(new CountDownLatch(1), gate, first); + CloudRequestEngine engine = buildLazyCachedEngine(); + + List answers = processTwice(engine); + + assertEquals(Arrays.asList(first, first), answers); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false && + c.getURL().getPath().endsWith("evidencekeys") == false), + any(), + any()); + } + + /** + * With lazy loading, a response with a 51Did is not cached, so the second + * identical request goes to the cloud service and gets its own. + */ + @Test + public void process_LazyFodIdResponseIsNotStored() throws Exception { + CountDownLatch gate = new CountDownLatch(0); + String first = "{'device':{'value':'1'},'fodid':'first'}"; + String second = "{'device':{'value':'1'},'fodid':'second'}"; + mockService(new CountDownLatch(2), gate, first, second); + CloudRequestEngine engine = buildLazyCachedEngine(); + + List answers = processTwice(engine); + + assertEquals(Arrays.asList(first, second), answers, + "A response with a 51Did was served from the cache."); + } + + private List processTwice(CloudRequestEngine engine) + throws Exception { + List answers = new ArrayList<>(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (int i = 0; i < 2; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + answers.add(data.get(CloudRequestData.class) + .getJsonResponse()); + } + } + } + return answers; + } +} From 089dfbbc34161a3d8604ec2d25323956b0dc7eaa Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 23:17:44 +0000 Subject: [PATCH 35/35] Fix: decide whether a lazy result may be cached without waiting for it isCacheable read the cloud response through the getter that waits for lazy loading, so a cached lazy Cloud Request Engine waited in process for the cloud service. It now reads the stored response without waiting (AspectDataBase.getWithoutWaiting). With lazy loading the result is offered to the cache by the processing task once processing has finished, as the response is not there to check before then. Without lazy loading nothing changes. The check still answers caching.md line 144 at 706de1f0: "A cache MUST NOT store a result that holds such a value, and MUST NOT serve one that it finds." --- .../CloudRequestDataInternal.java | 11 ++++++++ .../CloudRequestEngineDefault.java | 7 ++++- .../pipeline/engines/data/AspectDataBase.java | 12 +++++++++ .../flowelements/AspectEngineBase.java | 27 ++++++++++++++----- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestDataInternal.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestDataInternal.java index b3145b2b8..92a5c9760 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestDataInternal.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestDataInternal.java @@ -39,6 +39,17 @@ public String getJsonResponse() { return null; } + /** + * The JSON response as it is stored now, without waiting for lazy + * loading processing to finish, so the engine can check it from inside + * that processing. + * @return the JSON response, or null if none has been stored + */ + String getJsonResponseWithoutWaiting() { + Object response = getWithoutWaiting(JSON_RESPONSE_KEY); + return response == null ? null : response.toString(); + } + public CloudRequestDataInternal( Logger logger, diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 4b68daac0..ea836bfa1 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -466,12 +466,17 @@ private void completeDiscovery() throws IOException { * "A cache MUST NOT store a result that holds such a value, and MUST NOT * serve one that it finds." A response that cannot be read as a JSON * object is not cached either, as it cannot be checked. + *

+ * The response is read without waiting for lazy loading, so the check + * never makes process wait for the cloud service. With lazy loading it + * is made by the processing task once the response is stored. * @param data the result to check * @return false if the response has a top-level fodid member */ @Override protected boolean isCacheable(CloudRequestData data) { - String response = data.getJsonResponse(); + String response = + ((CloudRequestDataInternal)data).getJsonResponseWithoutWaiting(); if (response == null) { return false; } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java index 6f1ff15ae..61d69b3d5 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java @@ -199,6 +199,18 @@ public Map asKeyMap() { return super.asKeyMap(); } + /** + * Get the value stored for the key as it is now, without waiting for any + * lazy loading processing to finish, so it can be read from inside that + * processing. The value is null when nothing has been stored for the key, + * including when processing has not stored it yet. + * @param key the key to read + * @return the value stored for the key, or null + */ + protected Object getWithoutWaiting(String key) { + return super.asKeyMap().get(key); + } + /** * Gets the value stored using the specified key with full checks * against the {@link MissingPropertyService}. diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java index e95167c6e..e40600e74 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java @@ -143,7 +143,10 @@ protected final void processInternal(FlowData flowData) throws Exception { * "A cache MUST NOT store a result that holds such a value, and MUST NOT * serve one that it finds." *

- * With lazy loading the result is checked before it has been processed. + * It is only asked about a result that has been processed: with lazy + * loading, the processing task asks once its work is done, so the check + * never makes process wait for a result. A result found in the cache was + * processed before it was stored. * @param data the result to check * @return true, unless an engine overrides this to keep a result out of * the cache @@ -192,11 +195,15 @@ private void processWithCache(final FlowData flowData) throws Exception { // If lazy loading is configured then create a task // to do the processing and assign the task to the // aspect data property. + // The result is offered to the cache by the task, once it + // has been processed, so that process does not wait for it + // to decide whether the result may be cached. ((AspectDataBase) aspectData).addProcessCallable( new ProcessCallable(this) { @Override public Void call() throws Exception { processEngine(flowData, aspectData); + putInCache(flowData, aspectData); return null; } } @@ -205,11 +212,7 @@ public Void call() throws Exception { else { // If not lazy loading, just start processing. processEngine(flowData, aspectData); - } - // If there is a cache then add the result - // of processing to the cache. - if (cache != null && isCacheable(aspectData)) { - cache.put(flowData, flowData.getFromElement(this)); + putInCache(flowData, aspectData); } } else { // We have a result from the cache so add it @@ -218,6 +221,18 @@ public Void call() throws Exception { } } + /** + * Add a processed result to the cache, if there is one and the result + * may be cached. + * @param flowData the {@link FlowData} whose Evidence makes the key + * @param aspectData the processed result + */ + private void putInCache(FlowData flowData, TData aspectData) { + if (cache != null && isCacheable(aspectData)) { + cache.put(flowData, aspectData); + } + } + @Override protected void managedResourcesCleanup() { if (cache != null) {