From b78810fffb6a3f8a8b92c8f9ce624f24c8085553 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 11:32:46 +0000 Subject: [PATCH 01/42] Point AGENTS.md at the one list of environment variable names (#118) * DOC: Point at the one list of environment variable names AGENTS.md gains a section linking common-ci's list of environment variable names (environment-variables.md, made from environment-variables.json) at commit 18f9a570, and saying the list decides. Documentation only. * DOC: Link the list of environment variable names at d8a9c596 The list at that commit holds the names review found missing. --- AGENTS.md | 13 +++++++++++++ README.md | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index b5125dfc6..e27e80b3a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -222,6 +222,19 @@ example. The IP Intelligence cloud examples read their key from example jars as above, and the example tests use the paid data files when CI has them. +## Environment variable names + +[common-ci's list of environment variable names](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.md) +holds every environment variable name that the packages, tests, examples and +CI scripts of every language read, with what each name holds and the +deprecated names each replaced. The list decides. Use the names it gives, and +when a change needs a new name, or reads an old name again, change +[`environment-variables.json`](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.json) +in common-ci in the same piece of work. This repository's own name +check, [`ci/check-environment-variable-names.ps1`](ci/check-environment-variable-names.ps1), still keeps a +list of its own. Where the two differ, common-ci's list is the one to +follow. + ## One reactor, six package trees The root `pom.xml` lists the six package trees as modules but is not their diff --git a/README.md b/README.md index df1cd6427..177fed64e 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,10 @@ The cloud tests need a Resource Key, passed as `-DTestResourceKey=`. The on-premise tests need the free data files, which the script in [Getting the code](#getting-the-code) downloads. For the Enterprise IP Intelligence data file, see `ip-intelligence/README.md`. + +Every environment variable name that the packages, tests, examples and CI +scripts read is in [common-ci's list of environment variable names](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.md), +which decides. Tests find data files by searching from the root of this repository, so run Maven from here rather than from inside a package folder. From 30e4886708974308d4b70e1804424f55560824a7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 22:20:38 +0000 Subject: [PATCH 02/42] Measure the share usage overhead as a median over warmed up rounds (#91) The share usage overhead test now measures the median of warmed-up rounds of 250 calls, timed with System.nanoTime, and fails at about ten times the highest median any runner shows at usual speed: 0.07 ms a call with one Evidence value and 40 ms with 1,000, both set from 82 medians across Ubuntu, Ubuntu ARM, Windows, macOS arm64 and macOS x64. The old single-pass test failed by chance on Intel macOS (0.1083 ms against 0.1 ms in release build 43364257944). Full build 43499659322 passed all 15 jobs with macOS. --- .../performance/ShareUsageOverheadTests.java | 140 +++++++++++------- 1 file changed, 88 insertions(+), 52 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java index 9583850a7..71a362e7e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java @@ -40,7 +40,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.ILoggerFactory; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; @@ -50,6 +49,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -58,8 +58,13 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * Times Process calls through the usage sharing element. Every call in a + * test carries the same Evidence, so after the first call the tracker turns + * each one away as a repeat and nothing more is queued. The measured rounds + * time that path, which finds the values to share and checks the tracker. + */ public class ShareUsageOverheadTests { - static Logger logger = LoggerFactory.getLogger("testLogger"); private Pipeline pipeline; private static final Map overheadMillis = new LinkedHashMap<>(); @@ -115,50 +120,37 @@ public void cleanUp() throws Exception { pipeline.close(); } - // The FlowData are all processed before any is closed, so that closing - // them is not part of the time measured. They are closed together in a - // finally block rather than one by one with try-with-resources. - @Test - @SuppressWarnings("PMD.FlowDataInTryWithResources") - public void ShareUsageOverhead_SingleEvidence() throws Exception { - int iterations = 10_000; - double maxOverheadPerCall = 0.1; - List data = new ArrayList<>(); - long start; - long end; - try { - for (int i = 0; i < iterations; i++) { - FlowData flowData = pipeline.createFlowData(); - data.add(flowData); - flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); - } + private static final String USER_AGENT = + "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) " + + "AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"; - start = System.currentTimeMillis(); - for (FlowData entry : data) { - entry.process(); - } - end = System.currentTimeMillis(); - } finally { - closeAll(data); - } + /** + * Rounds measured, and the number discarded first so that the JIT has + * compiled the code. The median of the rounds is checked, so one round + * slowed by a busy runner or a garbage collection does not fail a test, + * whilst a cost on every call does. + */ + private static final int MeasuredRounds = 7; + private static final int WarmUpRounds = 2; - double msOverheadPerCall = (double)(end - start) / iterations; - overheadMillis.put("ShareUsageOverhead_SingleEvidence_ms", msOverheadPerCall); - System.out.println("ShareUsageOverhead_SingleEvidence: " + msOverheadPerCall + "ms per call"); - logger.info("Overhead was {} millis", msOverheadPerCall); - Assertions.assertTrue(msOverheadPerCall < maxOverheadPerCall, - "Pipeline with share usage overhead per Process call was " + - msOverheadPerCall + "ms. Maximum permitted is " + maxOverheadPerCall); - } + /** + * The most the median Process call may take. Each limit is about ten + * times the highest median the CI runners measure at their usual speed, + * so a runner several times slower than usual still passes. + */ + private static final double SingleEvidenceLimitMillis = 0.07; + private static final double ThousandEvidenceLimitMillis = 40; - // See ShareUsageOverhead_SingleEvidence for why the FlowData are closed - // in a finally block. - @Test + /** + * Returns the milliseconds per Process call, for one round of calls with + * the evidence given. The FlowData are all processed before any is closed, + * so that closing them is not part of the time measured. They are closed + * together in a finally block rather than one by one with + * try-with-resources. + */ @SuppressWarnings("PMD.FlowDataInTryWithResources") - public void ShareUsageOverhead_ThousandEvidence() throws Exception { - int iterations = 1_000; - int evidenceCount = 1_000; - int maxOverheadPerCallMillis = 10; + private double measureRound(int iterations, int evidenceCount) + throws Exception { List data = new ArrayList<>(); long start; long end; @@ -169,24 +161,68 @@ public void ShareUsageOverhead_ThousandEvidence() throws Exception { for (int j = 0; j < evidenceCount; j++) { flowData.addEvidence("header." + j, j); } - flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); + flowData.addEvidence("header.user-agent", USER_AGENT); } - start = System.currentTimeMillis(); + start = System.nanoTime(); for (FlowData entry : data) { entry.process(); } - end = System.currentTimeMillis(); + end = System.nanoTime(); } finally { closeAll(data); } + return (end - start) / 1_000_000.0 / iterations; + } + + /** + * Returns the milliseconds per Process call of each measured round, + * lowest first, after the warm up rounds. + */ + private double[] measureRounds(int iterations, int evidenceCount) + throws Exception { + for (int i = 0; i < WarmUpRounds; i++) { + measureRound(iterations, evidenceCount); + } + double[] rounds = new double[MeasuredRounds]; + for (int i = 0; i < rounds.length; i++) { + rounds[i] = measureRound(iterations, evidenceCount); + } + Arrays.sort(rounds); + return rounds; + } + + /** + * Fails when the median round is not below the limit. The message lists + * every round, because the build log shows only the message, and the + * rounds tell a runner slow for the whole test from a slow round or two. + */ + private static void assertBelowLimit(double[] rounds, double limitMillis) { + double median = rounds[rounds.length / 2]; + Assertions.assertTrue(median < limitMillis, + "Pipeline with share usage overhead per Process call was " + + median + "ms. Maximum permitted is " + limitMillis + + ". Rounds in ms per call " + Arrays.toString(rounds)); + } + + @Test + public void ShareUsageOverhead_RepeatedSingleEvidence() throws Exception { + int iterations = 10_000; + + double[] rounds = measureRounds(iterations, 0); + overheadMillis.put("ShareUsageOverhead_SingleEvidence_ms", + rounds[rounds.length / 2]); + assertBelowLimit(rounds, SingleEvidenceLimitMillis); + } + + @Test + public void ShareUsageOverhead_RepeatedThousandEvidence() throws Exception { + int iterations = 250; + int evidenceCount = 1_000; - double msOverheadPerCall = (double)(end - start) / iterations; - overheadMillis.put("ShareUsageOverhead_HundredEvidence_ms", msOverheadPerCall); - System.out.println("ShareUsageOverhead_HundredEvidence: " + msOverheadPerCall + "ms per call"); - logger.info("Overhead was {} millis", msOverheadPerCall); - Assertions.assertTrue(msOverheadPerCall < maxOverheadPerCallMillis, - "Pipeline with share usage overhead per Process call was " + - msOverheadPerCall + "ms. Maximum permitted is " + maxOverheadPerCallMillis); + double[] rounds = measureRounds(iterations, evidenceCount); + overheadMillis.put("ShareUsageOverhead_HundredEvidence_ms", + rounds[rounds.length / 2]); + assertBelowLimit(rounds, ThousandEvidenceLimitMillis); } private static void closeAll(List data) throws Exception { From 0122ad769ac71b2c81473d7c1d8fc606fa72e1dc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 04:02:24 +0000 Subject: [PATCH 03/42] Test that a header no element reads reaches the usage sharing message (#127) The Usage Sharing Element accepts no Evidence. It finds the values to share in the web request, which the web integration adds to Evidence as `server.http-request` on every request ([`WebRequestEvidenceServiceCore.java` line 73](https://github.com/51Degrees/51degrees-java/blob/30e4886708974308d4b70e1804424f55560824a7/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java#L73), with every value recorded at [line 150](https://github.com/51Degrees/51degrees-java/blob/30e4886708974308d4b70e1804424f55560824a7/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java#L150)). A header that no element reads is therefore not in Evidence at all, and reaches the usage sharing message only through that web request. --- .../web/UsageSharingEvidenceTests.java | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java index b06d5e8b2..be5936f0d 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java @@ -37,21 +37,39 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import com.sun.net.httpserver.HttpServer; import org.mockito.ArgumentCaptor; import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.io.PrintWriter; import java.io.StringWriter; +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; + +import javax.xml.parsers.DocumentBuilderFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -75,6 +93,12 @@ public class UsageSharingEvidenceTests { private static final String SESSION_ID = "1f2e3d4c-5b6a-4987-8a1b-2c3d4e5f6a7b"; + /** + * How long to wait for the usage sharing message to arrive. It is sent + * at once, so this is only an upper bound for a slow runner. + */ + private static final int MESSAGE_WAIT_SECONDS = 60; + private Pipeline pipeline; private final WebRequestEvidenceServiceCore evidenceService = @@ -285,6 +309,93 @@ public void Revalidation_SameETag_Gets304() throws Exception { } } + /** + * A header no element reads, sent through the web integration, reaches + * the usage sharing message. The message is sent over HTTP to a local + * endpoint, as it would be to 51Degrees, and read back from there. + */ + @Test + public void UnreadHeader_ReachesTheUsageSharingMessage() throws Exception { + BlockingQueue received = new LinkedBlockingQueue<>(); + HttpServer server = HttpServer.create( + new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/", exchange -> { + try (InputStream body = exchange.getRequestBody()) { + received.add(readAll(body)); + } + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + try (Pipeline sharing = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(1) + .setMinimumEntriesPerMessage(1) + .setRepeatEvidenceIntervalMinutes(0) + .setShareUsageUrl("http://" + + server.getAddress().getHostString() + ":" + + server.getAddress().getPort() + "/new.ashx") + .build()) + .setAutoCloseElements(true) + .build()) { + assertFalse("A header no element uses is accepted.", + sharing.getEvidenceKeyFilter().include("header.x-test")); + try (FlowData flowData = sharing.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + flowData.process(); + } + + byte[] sent = received.poll( + MESSAGE_WAIT_SECONDS, TimeUnit.SECONDS); + assertNotNull("No usage sharing message was sent.", sent); + Document message = DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new GZIPInputStream(new ByteArrayInputStream(sent))); + assertEquals("The header no element reads was not shared.", + "shared", sharedValue(message, "header", "X-Test")); + assertEquals("The User-Agent was not shared.", + USER_AGENT, sharedValue(message, "header", "User-Agent")); + assertNull("The raw Cookie header was shared.", + sharedValue(message, "header", "Cookie")); + } finally { + server.stop(0); + } + } + + /** + * The value the message gives for the named value of a category, such + * as header, or null when it gives none. Names are compared without + * regard to case. + */ + private static String sharedValue( + Document message, + String category, + String name) { + NodeList values = message.getElementsByTagName(category); + for (int i = 0; i < values.getLength(); i++) { + Element value = (Element) values.item(i); + if (name.equalsIgnoreCase(value.getAttribute("Name"))) { + return value.getTextContent(); + } + } + return null; + } + + private static byte[] readAll(InputStream input) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + output.write(buffer, 0, read); + } + return output.toByteArray(); + } + private static HttpServletResponse response() throws Exception { HttpServletResponse response = mock(HttpServletResponse.class); when(response.getWriter()).thenReturn( From 2aae80172f7ee3774a043484eb5626dd53dc8301 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 04:24:14 +0000 Subject: [PATCH 04/42] Write the JSON Builder's errors as a list of messages (#128) Closes #9. --- .../flowelements/JsonBuilderElement.java | 34 +++++--- .../jsonbuilder/JsonBuilderTests.java | 79 ++++++++++++++++++- 2 files changed, 99 insertions(+), 14 deletions(-) diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java index 72f037049..413113d6e 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java @@ -433,21 +433,29 @@ private List getJavaScriptProperties( return javascriptPropertyNames; } + /** + * Adds the Flow Data's errors under the top-level key errors as a list + * of their messages, in the order the Flow Data holds them. A throwable + * with no message is written as its type, so the list holds only text. + * An error with no throwable has no message and is left out. + */ private void addErrors(FlowData data, Map allProperties) { - // If there are any errors then add them to the Json. - if (data.getErrors() != null && data.getErrors().size() > 0) { - Map> errors = new HashMap<>(); - for (FlowError error : data.getErrors()) - { - if (errors.containsKey(error.getFlowElement().getElementDataKey())) { - errors.get(error.getFlowElement().getElementDataKey()).add(error.getThrowable().getMessage()); - } else { - errors.put(error.getFlowElement().getElementDataKey(), - Collections.singletonList( - error.getThrowable().getMessage())); - } + Collection flowErrors = data.getErrors(); + if (flowErrors == null) { + return; + } + List messages = new ArrayList<>(); + for (FlowError error : flowErrors) { + Throwable throwable = error.getThrowable(); + if (throwable != null) { + String message = throwable.getMessage(); + messages.add(message != null ? + message : + throwable.getClass().getName()); } - allProperties.put("errors", errors); + } + if (messages.isEmpty() == false) { + allProperties.put("errors", messages); } } diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java index fe9923ff7..b81cf6c81 100644 --- a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java @@ -73,7 +73,8 @@ public class JsonBuilderTests { private FlowData flowData; private JsonBuilderData result; private Pipeline pipeline; - + private List flowErrors; + public JsonBuilderTests() { ILoggerFactory internalLogger = mock(ILoggerFactory.class); when(internalLogger.getLogger(anyString())).thenReturn(mock(Logger.class)); @@ -114,6 +115,81 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { Map> propertyMetaData = new HashMap<>(); when(pipeline.getElementAvailableProperties()).thenReturn(propertyMetaData); + + flowErrors = new ArrayList<>(); + } + + /** + * The errors are a list of their messages under the top-level key + * errors, in the order the Flow Data holds them, whichever element + * recorded each. See the Process section of the JSON Builder + * specification. + */ + @Test + public void JsonBuilder_Errors_ListOfMessages() throws Exception { + FlowElement first = mock(FlowElement.class); + when(first.getElementDataKey()).thenReturn("first"); + FlowElement second = mock(FlowElement.class); + when(second.getElementDataKey()).thenReturn("second"); + flowErrors.add(new FlowError.Default( + new Exception("first element, one"), first)); + flowErrors.add(new FlowError.Default( + new Exception("second element"), second)); + flowErrors.add(new FlowError.Default( + new Exception("first element, two"), first)); + + String json = testIteration(1); + + JSONArray errors = new JSONObject(json).getJSONArray("errors"); + assertEquals( + Arrays.asList( + "first element, one", + "second element", + "first element, two"), + errors.toList(), + json); + } + + /** + * An error whose throwable has no message is written as the throwable's + * type, so the list holds only text. + */ + @Test + public void JsonBuilder_Errors_NoMessage_WritesType() throws Exception { + FlowElement element = mock(FlowElement.class); + when(element.getElementDataKey()).thenReturn("element"); + flowErrors.add(new FlowError.Default( + new IllegalStateException(), element)); + + String json = testIteration(1); + + JSONArray errors = new JSONObject(json).getJSONArray("errors"); + assertEquals( + Collections.singletonList(IllegalStateException.class.getName()), + errors.toList(), + json); + } + + /** + * An error with no throwable has no message to write, so it is left out, + * and with no messages at all the key errors is left out too. + */ + @Test + public void JsonBuilder_Errors_NoThrowable_LeftOut() throws Exception { + FlowElement element = mock(FlowElement.class); + flowErrors.add(new FlowError.Default(null, element)); + + String json = testIteration(1); + assertFalse(new JSONObject(json).has("errors"), json); + + flowErrors.add(new FlowError.Default( + new Exception("the message"), element)); + + json = testIteration(1); + assertEquals( + Collections.singletonList("the message"), + new JSONObject(json).getJSONArray("errors").toList(), + json); } /** @@ -598,6 +674,7 @@ private String testIteration( return result[0]; }); when(flowData.getPipeline()).thenReturn(pipeline); + when(flowData.getErrors()).thenReturn(flowErrors); jsonBuilderElement.process(flowData); From 6e03a25fdf6571b40b75de115da2a33b18102dc0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 04:24:23 +0000 Subject: [PATCH 05/42] Name the builder defaults that were literals (#130) Closes #14. --- ...viceDetectionOnPremisePipelineBuilder.java | 4 +- ...nOnPremisePipelineBuilderDefaultsTest.java | 49 ++++++++++++ ...PIntelligenceOnPremisePipelineBuilder.java | 4 +- ...eOnPremisePipelineBuilderDefaultsTest.java | 47 +++++++++++ .../fiftyone/pipeline/core/Constants.java | 14 +++- .../flowelements/PipelineBuilderBase.java | 7 +- .../PipelineBuilderBaseTests.java | 20 +++++ .../fiftyone/flowelements/Constants.java | 7 ++ .../flowelements/FiftyOnePipelineBuilder.java | 2 +- .../FiftyOnePipelineBuilderTests.java | 60 ++++++++++++++ .../fiftyone/pipeline/engines/Constants.java | 27 +++++++ .../PrePackagedPipelineBuilderBase.java | 20 +++-- .../PrePackagedPipelineBuilderBaseTests.java | 78 +++++++++++++++++++ 13 files changed, 325 insertions(+), 14 deletions(-) create mode 100644 device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderDefaultsTest.java create mode 100644 ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderDefaultsTest.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilderTests.java create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBaseTests.java diff --git a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java index 63d98b785..983a5eaa9 100644 --- a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java +++ b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java @@ -48,7 +48,9 @@ public class DeviceDetectionOnPremisePipelineBuilder extends PrePackagedPipelineBuilderBase { - protected boolean shareUsageEnabled = true; + protected boolean shareUsageEnabled = + fiftyone.pipeline.engines.fiftyone.flowelements.Constants + .SHARE_USAGE_DEFAULT_ENABLED; private String filename; private boolean createTempDataCopy; private byte[] engineData; diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderDefaultsTest.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderDefaultsTest.java new file mode 100644 index 000000000..cec79e1ff --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderDefaultsTest.java @@ -0,0 +1,49 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.pipeline.engines.fiftyone.flowelements.Constants; +import fiftyone.pipeline.engines.services.DataUpdateService; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * Usage sharing is on unless the builder is told otherwise, as + * {@link Constants#SHARE_USAGE_DEFAULT_ENABLED} says. + */ +public class DeviceDetectionOnPremisePipelineBuilderDefaultsTest { + + @Test + public void ShareUsage_NotSet_FollowsDefault() { + DeviceDetectionOnPremisePipelineBuilder builder = + new DeviceDetectionOnPremisePipelineBuilder( + mock(DataUpdateService.class), + mock(HttpClient.class)); + assertEquals( + Constants.SHARE_USAGE_DEFAULT_ENABLED, + builder.shareUsageEnabled); + } +} diff --git a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java index 210fa80dd..d93bb7784 100644 --- a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java +++ b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java @@ -47,7 +47,9 @@ public class IPIntelligenceOnPremisePipelineBuilder extends PrePackagedPipelineBuilderBase { - protected boolean shareUsageEnabled = true; + protected boolean shareUsageEnabled = + fiftyone.pipeline.engines.fiftyone.flowelements.Constants + .SHARE_USAGE_DEFAULT_ENABLED; private String filename; private boolean createTempDataCopy; private int concurrency = -1; diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderDefaultsTest.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderDefaultsTest.java new file mode 100644 index 000000000..cd414b08a --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderDefaultsTest.java @@ -0,0 +1,47 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.pipeline.engines.fiftyone.flowelements.Constants; +import fiftyone.pipeline.engines.services.DataUpdateService; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * Usage sharing is on unless the builder is told otherwise, as + * {@link Constants#SHARE_USAGE_DEFAULT_ENABLED} says. + */ +public class IPIntelligenceOnPremisePipelineBuilderDefaultsTest { + + @Test + public void ShareUsage_NotSet_FollowsDefault() { + IPIntelligenceOnPremisePipelineBuilder builder = + new IPIntelligenceOnPremisePipelineBuilder( + mock(DataUpdateService.class)); + assertEquals( + Constants.SHARE_USAGE_DEFAULT_ENABLED, + builder.shareUsageEnabled); + } +} diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/Constants.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/Constants.java index 205fc3f4e..15bc17786 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/Constants.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/Constants.java @@ -48,5 +48,17 @@ public class Constants { public static final String EVIDENCE_PROTOCOL = EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "protocol"; - public static final String EVIDENCE_SESSION_KEY = EVIDENCE_SESSION_PREFIX + EVIDENCE_SEPERATOR + "session"; + public static final String EVIDENCE_SESSION_KEY = EVIDENCE_SESSION_PREFIX + EVIDENCE_SEPERATOR + "session"; + + /** + * Whether a Pipeline closes its elements when it is closed, unless its + * builder is told otherwise. + */ + public static final boolean DEFAULT_AUTO_CLOSE_ELEMENTS = false; + + /** + * Whether a Pipeline records the exceptions thrown while processing as + * errors rather than throwing them, unless its builder is told otherwise. + */ + public static final boolean DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS = false; } diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java index 72cefd9a5..147c7dc1a 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.core.flowelements; +import fiftyone.pipeline.core.Constants; import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.FlowDataFactory; @@ -67,9 +68,11 @@ public abstract class PipelineBuilderBase { */ protected final List services = new ArrayList<>(); - protected boolean autoCloseElements = false; + protected boolean autoCloseElements = + Constants.DEFAULT_AUTO_CLOSE_ELEMENTS; - protected boolean suppressProcessExceptions = false; + protected boolean suppressProcessExceptions = + Constants.DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS; /** * Construct a new instance using the default {@link ILoggerFactory} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBaseTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBaseTests.java index cc5b74fd8..22ad433b5 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBaseTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBaseTests.java @@ -22,12 +22,15 @@ package fiftyone.pipeline.core.flowelements; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import org.junit.Test; +import fiftyone.pipeline.core.Constants; import fiftyone.pipeline.core.testclasses.services.TestAutoCloseableService; import fiftyone.pipeline.core.testclasses.services.TestCloseableService; @@ -53,4 +56,21 @@ public void PipelineBuilderBase_Build() throws Exception { verify(service2, times(1)).close(); } + /** + * The builder's defaults come from {@link Constants} and keep the values + * they have always had. + */ + @Test + public void PipelineBuilderBase_Defaults() { + TestPipelineBuilder builder = new TestPipelineBuilder(); + assertEquals( + Constants.DEFAULT_AUTO_CLOSE_ELEMENTS, + builder.autoCloseElements); + assertEquals( + Constants.DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS, + builder.suppressProcessExceptions); + assertFalse(Constants.DEFAULT_AUTO_CLOSE_ELEMENTS); + assertFalse(Constants.DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS); + } + } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index 48e1e1b88..0641be77e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -58,6 +58,13 @@ public class Constants { fiftyone.pipeline.core.Constants.EVIDENCE_QUERY_PREFIX + fiftyone.pipeline.core.Constants.EVIDENCE_SEPERATOR + EVIDENCE_SEQUENCE_SUFIX; + + /** + * Whether a 51Degrees Pipeline builder adds the Usage Sharing Element, + * unless it is told otherwise. + */ + public static final boolean SHARE_USAGE_DEFAULT_ENABLED = true; + public static final int SHARE_USAGE_DEFAULT_REPEAT_EVIDENCE_MINUTES = 20; public static final double SHARE_USAGE_DEFAULT_SHARE_PERCENTAGE = 1.0; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java index dfe8a3e60..83322e115 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java @@ -33,7 +33,7 @@ */ public class FiftyOnePipelineBuilder extends PipelineBuilder { - private boolean shareUsageEnabled = true; + private boolean shareUsageEnabled = Constants.SHARE_USAGE_DEFAULT_ENABLED; /** * Construct a new builder. diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilderTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilderTests.java new file mode 100644 index 000000000..11c532c17 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilderTests.java @@ -0,0 +1,60 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Usage sharing is on unless the builder is told otherwise, as + * {@link Constants#SHARE_USAGE_DEFAULT_ENABLED} says. + */ +public class FiftyOnePipelineBuilderTests { + + @Test + public void ShareUsage_DefaultValue() { + assertTrue(Constants.SHARE_USAGE_DEFAULT_ENABLED); + } + + @SuppressWarnings("rawtypes") + @Test + public void ShareUsage_NotSet_FollowsDefault() throws Exception { + try (Pipeline pipeline = new FiftyOnePipelineBuilder() + .setAutoCloseElements(true) + .build()) { + int shareUsage = 0; + for (FlowElement element : pipeline.getFlowElements()) { + if (element instanceof ShareUsageElement) { + shareUsage++; + } + } + assertEquals( + Constants.SHARE_USAGE_DEFAULT_ENABLED ? 1 : 0, + shareUsage); + } + } +} diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java index 252924ea7..5cd469993 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java @@ -22,6 +22,8 @@ package fiftyone.pipeline.engines; +import fiftyone.pipeline.engines.configuration.CacheConfiguration; + public class Constants { public static final String FIFTYONE_COOKIE_PREFIX = "51d_"; public static final String DEFAULT_SESSION_COOKIE_NAME = "JSESSIONID"; @@ -34,6 +36,31 @@ public class Constants { public static final boolean DEFAULT_VERIFY_IF_MODIFIED_SINCE = true; public static final boolean DEFAULT_UPDATE_ON_STARTUP = false; + /** + * Whether a pre-packaged Pipeline loads results lazily, unless its builder + * is told otherwise. + */ + public static final boolean DEFAULT_LAZY_LOADING = false; + + /** + * How long a lazily loaded value is waited for when lazy loading is used + * without a timeout. + */ + public static final long DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS = 1000; + + /** + * Whether a pre-packaged Pipeline caches results, unless its builder is + * told otherwise. + */ + public static final boolean DEFAULT_RESULTS_CACHE = false; + + /** + * How many results are cached when the results cache is used without a + * size, which is the cache size the caching specification gives. + */ + public static final int DEFAULT_RESULTS_CACHE_SIZE = + CacheConfiguration.defaultSize; + public enum PerformanceProfiles { LowMemory, MaxPerformance, diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBase.java index 92188b7f2..99a40eeba 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBase.java @@ -23,6 +23,7 @@ package fiftyone.pipeline.engines.flowelements; import fiftyone.pipeline.core.flowelements.PipelineBuilderBase; +import fiftyone.pipeline.engines.Constants; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; @@ -35,13 +36,14 @@ public abstract class PrePackagedPipelineBuilderBase > extends PipelineBuilderBase { - protected boolean lazyLoading = false; + protected boolean lazyLoading = Constants.DEFAULT_LAZY_LOADING; - protected boolean resultsCache = false; + protected boolean resultsCache = Constants.DEFAULT_RESULTS_CACHE; - protected long lazyLoadingTimeoutMillis = 1000; + protected long lazyLoadingTimeoutMillis = + Constants.DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS; - protected int resultsCacheSize = 1000; + protected int resultsCacheSize = Constants.DEFAULT_RESULTS_CACHE_SIZE; /** * Construct a new instance using the default {@link ILoggerFactory} @@ -63,7 +65,8 @@ public PrePackagedPipelineBuilderBase(ILoggerFactory loggerFactory) { } /** - * Enable lazy loading of results. Uses a default timeout of 5 seconds. + * Enable lazy loading of results, with the timeout + * {@link Constants#DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS}. * @return this builder instance */ @SuppressWarnings("unchecked") @@ -75,7 +78,7 @@ public TBuilder useLazyLoading() { /** * Enable lazy loading of results. * @param timeoutMillis the timeout to use when attempting to access - * lazy-loaded values. Default is 5 seconds + * lazy-loaded values * @return this builder */ @SuppressWarnings("unchecked") @@ -86,7 +89,8 @@ public TBuilder useLazyLoading(long timeoutMillis) { } /** - * Enable caching of results. Uses a default cache size of 1000. + * Enable caching of results, with the size + * {@link Constants#DEFAULT_RESULTS_CACHE_SIZE}. * @return this builder */ @SuppressWarnings("unchecked") @@ -98,7 +102,7 @@ public TBuilder useResultsCache() { /** * Enable caching of results. * @param size the maximum number of results to hold in the device detection - * cache. Default is 1000 + * cache * @return this builder */ @SuppressWarnings("unchecked") diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBaseTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBaseTests.java new file mode 100644 index 000000000..5634edef0 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBaseTests.java @@ -0,0 +1,78 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * The defaults of a pre-packaged Pipeline builder come from {@link Constants} + * and keep the values they have always had. + */ +public class PrePackagedPipelineBuilderBaseTests { + + private static class TestBuilder + extends PrePackagedPipelineBuilderBase { + } + + @Test + public void Defaults_FromConstants() { + TestBuilder builder = new TestBuilder(); + assertEquals(Constants.DEFAULT_LAZY_LOADING, builder.lazyLoading); + assertEquals( + Constants.DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS, + builder.lazyLoadingTimeoutMillis); + assertEquals(Constants.DEFAULT_RESULTS_CACHE, builder.resultsCache); + assertEquals( + Constants.DEFAULT_RESULTS_CACHE_SIZE, + builder.resultsCacheSize); + } + + @Test + public void Defaults_Values() { + assertFalse(Constants.DEFAULT_LAZY_LOADING); + assertEquals(1000, Constants.DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS); + assertFalse(Constants.DEFAULT_RESULTS_CACHE); + assertEquals(1000, Constants.DEFAULT_RESULTS_CACHE_SIZE); + assertEquals( + CacheConfiguration.defaultSize, + Constants.DEFAULT_RESULTS_CACHE_SIZE); + } + + @Test + public void UseWithoutSize_KeepsDefaults() { + TestBuilder builder = new TestBuilder() + .useLazyLoading() + .useResultsCache(); + assertEquals( + Constants.DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS, + builder.lazyLoadingTimeoutMillis); + assertEquals( + Constants.DEFAULT_RESULTS_CACHE_SIZE, + builder.resultsCacheSize); + } +} From cd5f5a42e0497fdce9a56fe911386f3577364fb5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 05:22:16 +0000 Subject: [PATCH 06/42] Package descriptions use the words people search for, and source links point at this repository (#27) - **Package descriptions.** The `` in six pom files now says what each package does, in the words developers type when they look for a package like it, and ends by naming the products it is an alternative to. The detail, and a before and after table for each artifact, follow below. - **Source links.** The `` ``, ``, and `` in the four parent poms ([`pipeline/pom.xml`](https://github.com/51Degrees/51degrees-java/blob/7d8b2091582e3f77b8368e27a34fbffe9a7da1b7/pipeline/pom.xml#L374-L379), [`device-detection/pom.xml`](https://github.com/51Degrees/51degrees-java/blob/7d8b2091582e3f77b8368e27a34fbffe9a7da1b7/device-detection/pom.xml#L327-L332), [`ip-intelligence/pom.xml`](https://github.com/51Degrees/51degrees-java/blob/7d8b2091582e3f77b8368e27a34fbffe9a7da1b7/ip-intelligence/pom.xml#L389-L394), and [`location/pom.xml`](https://github.com/51Degrees/51degrees-java/blob/7d8b2091582e3f77b8368e27a34fbffe9a7da1b7/location/pom.xml#L312-L317)) pointed at the earlier repositories [pipeline-java](https://github.com/51Degrees/pipeline-java), [device-detection-java](https://github.com/51Degrees/device-detection-java), [ip-intelligence-java](https://github.com/51Degrees/ip-intelligence-java), and [location-java](https://github.com/51Degrees/location-java). Those repositories are archived, so the source link on each package page was a dead end. Two changes fix that. 1. All three elements now point at this repository in https form, `https://github.com/51Degrees/51degrees-java` for `` and `scm:git:https://github.com/51Degrees/51degrees-java.git` for `` and ``. The `` used git://, which GitHub [permanently turned off on March 15, 2022](https://github.blog/security/application-security/improving-git-protocol-security-github/). The `` named `info@51degrees.com` as its host rather than GitHub. 2. Each parent's `` now sets `child.scm.connection.inherit.append.path`, `child.scm.developerConnection.inherit.append.path`, and `child.scm.url.inherit.append.path` to `"false"`. Child packages inherit these links, and without the attributes Maven adds each child's artifactId to them. The [Maven model reference](https://maven.apache.org/ref/3.10.0/maven-model/maven.html) gives the default for `` `` as "parent value [+ path adjustment] + (artifactId or project.directory property), or just parent value if scm's child.scm.url.inherit.append.path="false"", and says the same for the two connections. That is why the Maven Central page for [`com.51degrees:device-detection` 4.5.130](https://central.sonatype.com/artifact/com.51degrees/device-detection/4.5.130) links its source to https://github.com/51Degrees/device-detection-java/device-detection, which GitHub answers with 404. With the attributes, a child's links are the repository itself. The attributes need Maven 3.6.1 or later, and every runner image this repository's CI uses lists Maven 3.9.12 or later (see "How it was checked"). --- device-detection/device-detection.cloud/pom.xml | 2 +- .../device-detection.hash.engine.on-premise/pom.xml | 2 +- device-detection/device-detection/pom.xml | 2 +- device-detection/pom.xml | 10 ++++++---- ip-intelligence/ip-intelligence.cloud/pom.xml | 2 +- .../ip-intelligence.engine.on-premise/pom.xml | 2 +- ip-intelligence/ip-intelligence/pom.xml | 2 +- ip-intelligence/pom.xml | 10 ++++++---- location/pom.xml | 10 ++++++---- pipeline/pom.xml | 10 ++++++---- 10 files changed, 30 insertions(+), 22 deletions(-) diff --git a/device-detection/device-detection.cloud/pom.xml b/device-detection/device-detection.cloud/pom.xml index ddfbaf069..825e1d517 100644 --- a/device-detection/device-detection.cloud/pom.xml +++ b/device-detection/device-detection.cloud/pom.xml @@ -11,7 +11,7 @@ device-detection.cloud 51Degrees :: Device Detection :: Cloud - Cloud-based device detection for Java that uses the 51Degrees cloud service via the Pipeline API. Parses HTTP headers to identify device hardware, operating systems, browsers, and crawlers. + Device Detection and User-Agent parser for Java, using the 51Degrees cloud service. Detects the device type and model, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. Needs a Resource Key that says which properties to return. Built on the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL. https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection.hash.engine.on-premise/pom.xml b/device-detection/device-detection.hash.engine.on-premise/pom.xml index 7d141d302..b855aab87 100644 --- a/device-detection/device-detection.hash.engine.on-premise/pom.xml +++ b/device-detection/device-detection.hash.engine.on-premise/pom.xml @@ -11,7 +11,7 @@ device-detection.hash.engine.on-premise 51Degrees :: Device Detection :: Hash :: On Premise - High-performance on-premise hash-based device detection engine for Java, built for the 51Degrees Pipeline API. Parses HTTP headers locally to identify device hardware, operating systems, browsers, and crawlers, with no external service calls. + On-premise Device Detection engine and User-Agent parser for Java. Reads a 51Degrees Hash data file on your server to detect the device type and model, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. Built on the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL. https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection/pom.xml b/device-detection/device-detection/pom.xml index 96eefb171..2608c4e91 100644 --- a/device-detection/device-detection/pom.xml +++ b/device-detection/device-detection/pom.xml @@ -12,7 +12,7 @@ device-detection 51Degrees :: Device Detection - Parse HTTP headers to detect the device type, model, OS, browser, and crawler information. This is an alternative to popular UAParser, DeviceAtlas, and WURFL packages. + Device Detection and User-Agent parser for Java. Detects the device type and model, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. Works on-premise with a data file or in the cloud. Built on the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL. https://github.com/51Degrees/51degrees-java diff --git a/device-detection/pom.xml b/device-detection/pom.xml index b255d1e53..a8dc451ad 100644 --- a/device-detection/pom.xml +++ b/device-detection/pom.xml @@ -363,9 +363,11 @@ - - https://github.com/51Degrees/device-detection-java - scm:git:git://github.com/51Degrees/device-detection-java.git - scm:git:info@51degrees.com:51Degrees/device-detection-java.git + + https://github.com/51Degrees/51degrees-java + scm:git:https://github.com/51Degrees/51degrees-java.git + scm:git:https://github.com/51Degrees/51degrees-java.git diff --git a/ip-intelligence/ip-intelligence.cloud/pom.xml b/ip-intelligence/ip-intelligence.cloud/pom.xml index cfc6ffa32..178fbcff0 100644 --- a/ip-intelligence/ip-intelligence.cloud/pom.xml +++ b/ip-intelligence/ip-intelligence.cloud/pom.xml @@ -11,7 +11,7 @@ ip-intelligence.cloud 51Degrees :: IP Intelligence :: Cloud - Provides IP Intelligence for the 51Degrees Pipeline API using the 51Degrees Cloud service. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. + IP Intelligence and IP geolocation for Java, using the 51Degrees cloud service. Finds an IP address's likely location (country, region, town, and coordinates), connection type, and network. Needs a Resource Key that says which properties to return. Built on the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo. https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml b/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml index 0c48477de..da9985ff0 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml +++ b/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml @@ -11,7 +11,7 @@ ip-intelligence.engine.on-premise 51Degrees :: IP Intelligence :: On Premise - Provides the On-premise IP Intelligence engine for the 51Degrees Pipeline API. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. + On-premise IP Intelligence and IP geolocation engine for Java. Reads a 51Degrees IP Intelligence data file on your server to find an IP address's likely location (country, region, town, and coordinates), connection type, and network. Built on the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo. https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence/pom.xml b/ip-intelligence/ip-intelligence/pom.xml index 2baf5e0fd..2f59e3acd 100644 --- a/ip-intelligence/ip-intelligence/pom.xml +++ b/ip-intelligence/ip-intelligence/pom.xml @@ -12,7 +12,7 @@ ip-intelligence 51Degrees :: IP Intelligence - Provides IP Intelligence for the 51Degrees Pipeline API. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. + IP Intelligence and IP geolocation for Java. Finds an IP address's likely location (country, region, town, and coordinates), connection type, and network. Works on-premise with a data file or in the cloud. Built on the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo. https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/pom.xml b/ip-intelligence/pom.xml index 1355b5932..4650e53be 100644 --- a/ip-intelligence/pom.xml +++ b/ip-intelligence/pom.xml @@ -392,9 +392,11 @@ - - https://github.com/51Degrees/ip-intelligence-java - scm:git:git://github.com/51Degrees/ip-intelligence-java.git - scm:git:info@51degrees.com:51Degrees/ip-intelligence-java.git + + https://github.com/51Degrees/51degrees-java + scm:git:https://github.com/51Degrees/51degrees-java.git + scm:git:https://github.com/51Degrees/51degrees-java.git diff --git a/location/pom.xml b/location/pom.xml index 9974783aa..05c2eb72b 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -344,9 +344,11 @@ https://opensource.org/licenses/EUPL-1.2 - - https://github.com/51Degrees/location-java - scm:git:git://github.com/51Degrees/location-java.git - scm:git:info@51degrees.com:51Degrees/location-java.git + + https://github.com/51Degrees/51degrees-java + scm:git:https://github.com/51Degrees/51degrees-java.git + scm:git:https://github.com/51Degrees/51degrees-java.git diff --git a/pipeline/pom.xml b/pipeline/pom.xml index abcd6d8f6..533f13e51 100644 --- a/pipeline/pom.xml +++ b/pipeline/pom.xml @@ -406,9 +406,11 @@ - - https://github.com/51Degrees/pipeline-java - scm:git:git://github.com/51Degrees/pipeline-java.git - scm:git:info@51degrees.com:51Degrees/pipeline-java.git + + https://github.com/51Degrees/51degrees-java + scm:git:https://github.com/51Degrees/51degrees-java.git + scm:git:https://github.com/51Degrees/51degrees-java.git From c5d4559ff37486ac23ad943c14d69364ed539836 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 05:22:25 +0000 Subject: [PATCH 07/42] Start Chrome once before the Selenium contract tests (#124) In [full build 43411790985, attempt 1](https://github.com/51Degrees/51degrees-java/actions/runs/43411790985/attempts/1), the job [MacOS_X64_Java_11](https://github.com/51Degrees/51degrees-java/actions/runs/43411790985/job/131839970848) failed in the Selenium contract tests against the cloud web example, and that cancelled MacOS_X64_Java_17, MacOS_X64_Java_21 and MacOS_Java_11. --- ci/run-integration-tests.ps1 | 12 +++- ci/start-chrome-once.ps1 | 131 +++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 ci/start-chrome-once.ps1 diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index ff9175676..163ffbab4 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -111,7 +111,8 @@ function Invoke-ContractTests { $env:EXAMPLE_URL = "http://localhost:$Port" $env:EXAMPLE_LANG = 'java' - dotnet test selenium-api-tests -c Release --filter TestCategory=Contract + # Normal verbosity prints each test's time, passed or failed. + dotnet test selenium-api-tests -c Release --filter TestCategory=Contract --logger "console;verbosity=normal" # PowerShell does not stop for a native command that returns non-zero, # so without this the browser tests could fail every case and the job # would still be reported as a success. @@ -146,6 +147,15 @@ if (-not $TestResourceKey) { $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" $env:PAID_RESOURCE_KEY = $TestResourceKey + # Chrome's first start on a fresh runner is many times slower than the + # starts after it, so it is made here rather than inside the first test. + # The tests still run when it cannot be made. + try { + & "$PSScriptRoot/start-chrome-once.ps1" + } catch { + Write-Host "::warning title=Chrome not started::$_" + } + $contracts = @( @{ Label = 'cloud' diff --git a/ci/start-chrome-once.ps1 b/ci/start-chrome-once.ps1 new file mode 100644 index 000000000..dc668499d --- /dev/null +++ b/ci/start-chrome-once.ps1 @@ -0,0 +1,131 @@ +# Starts Chrome and its driver once, waits for them to finish and prints how +# long each took. The first start of Chrome on a fresh runner takes many times +# longer than the starts after it, so the Selenium contract tests run this +# first and that cost falls outside the limits of their tests. A start that +# takes longer than the limit is stopped, and one that exits with an error is +# reported with its error output, both as warnings, and the tests then run as +# they would have without it. +param( + # The longest each first start is given. + [int]$LimitSeconds = 300 +) +$ErrorActionPreference = "Stop" + +# The first of the names in a folder on the path, taking each folder in turn, +# as the contract tests search. +function Find-OnPath { + param([string[]]$Names) + foreach ($folder in ($env:PATH -split [System.IO.Path]::PathSeparator)) { + if (-not $folder) { + continue + } + foreach ($name in $Names) { + $candidate = Join-Path $folder $name + if (Test-Path $candidate -PathType Leaf) { + return $candidate + } + } + } + return $null +} + +# The browser the contract tests start, being the file CHROME_BIN names, then +# the first browser on the path. When neither gives one the tests leave the +# driver to find Chrome where it installs itself, so that place is used here. +function Find-Chrome { + if ($env:CHROME_BIN -and (Test-Path $env:CHROME_BIN -PathType Leaf)) { + return $env:CHROME_BIN + } + $onPath = Find-OnPath 'google-chrome', 'google-chrome-stable', 'chromium-browser', 'chromium', 'chrome.exe' + if ($onPath) { + return $onPath + } + $installed = if ($IsMacOS) { + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + } elseif ($IsWindows) { + Join-Path ([Environment]::GetFolderPath('ProgramFiles')) 'Google\Chrome\Application\chrome.exe' + } + if ($installed -and (Test-Path $installed -PathType Leaf)) { + return $installed + } + return $null +} + +# The driver the contract tests start, being the file CHROMEWEBDRIVER names or +# the driver in the folder it names, then the first driver on the path. +function Find-ChromeDriver { + $names = 'chromedriver', 'chromedriver.exe' + $configured = $env:CHROMEWEBDRIVER + if ($configured) { + if (Test-Path $configured -PathType Leaf) { + return $configured + } + foreach ($name in $names) { + $candidate = Join-Path $configured $name + if (Test-Path $candidate -PathType Leaf) { + return $candidate + } + } + } + return Find-OnPath $names +} + +# Runs a program, stopping it after the limit. Gives the seconds it took, its +# exit code, or -1 when it was stopped, and the start of its error output. +function Measure-Start { + param([string]$FilePath, [string[]]$Arguments) + $output = [System.IO.Path]::GetTempFileName() + $errors = [System.IO.Path]::GetTempFileName() + try { + $watch = [System.Diagnostics.Stopwatch]::StartNew() + $process = Start-Process -FilePath $FilePath -ArgumentList $Arguments -PassThru -NoNewWindow ` + -RedirectStandardOutput $output -RedirectStandardError $errors + # Reading the handle now keeps the exit code readable after the exit. + $null = $process.Handle + $exitCode = -1 + if ($process.WaitForExit($LimitSeconds * 1000)) { + $process.WaitForExit() + $exitCode = $process.ExitCode + } else { + Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue + } + return [pscustomobject]@{ + Seconds = [math]::Round($watch.Elapsed.TotalSeconds, 1) + ExitCode = $exitCode + Errors = ((Get-Content $errors -TotalCount 5 -ErrorAction SilentlyContinue) -join ' ') + } + } finally { + Remove-Item $output, $errors -Force -ErrorAction SilentlyContinue + } +} + +# Prints the result of one start, as a warning unless it exited cleanly. +function Write-Start { + param([string]$What, $Result) + if ($Result.ExitCode -eq 0) { + Write-Host "Started $What once before the Selenium contract tests in $($Result.Seconds) s" + } elseif ($Result.ExitCode -eq -1) { + Write-Host "::warning title=Chrome slow to start::$What did not finish its first start within $LimitSeconds s, so it was stopped." + } else { + Write-Host "::warning title=Chrome did not start::$What exited with code $($Result.ExitCode) after $($Result.Seconds) s. $($Result.Errors)" + } +} + +$driver = Find-ChromeDriver +if ($driver) { + Write-Start -What "the Chrome driver $driver" -Result (Measure-Start -FilePath $driver -Arguments '--version') +} + +$chrome = Find-Chrome +if (-not $chrome) { + Write-Host "::warning title=Chrome not started::No Chrome was found to start before the Selenium contract tests." + return +} +$userDataDir = Join-Path ([System.IO.Path]::GetTempPath()) "chrome-first-start-$PID" +try { + Write-Start -What "Chrome $chrome" -Result (Measure-Start -FilePath $chrome -Arguments @( + '--headless', '--disable-gpu', '--no-first-run', '--use-mock-keychain', + "--user-data-dir=$userDataDir", '--dump-dom', 'about:blank')) +} finally { + Remove-Item -Recurse -Force $userDataDir -ErrorAction SilentlyContinue +} From ceab0888d0267c2ebad57c719ea53dcc1f859555 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 06:04:34 +0000 Subject: [PATCH 08/42] Build each pull request on the configurations its changes need (#43) A pull request into `rc` runs all 15 configurations in `ci/options.json`, whatever it changes. Six of them are macOS, and only five macOS jobs run at once across the whole organisation. Every configuration runs the three IP Intelligence tests that load the multi-gigabyte Enterprise data file, every Windows configuration installs WMIC, which is only there for one of those tests, and fourteen configurations run the Selenium contract, the flakiest check in the window. --- .github/workflows/ci-script-tests.yml | 42 +++++++++++++ .github/workflows/cloud-51did.yml | 2 +- .github/workflows/nightly-pipeline.yml | 62 +++++++++---------- .github/workflows/pull-requests.yml | 10 ++++ .github/workflows/release-pull-request.yml | 31 ++++++++++ ci/README.md | 12 ++++ ci/run-integration-tests.ps1 | 9 ++- ci/run-unit-tests.ps1 | 15 ++++- ci/select-options.Tests.ps1 | 67 +++++++++++++++++++++ ci/select-options.ps1 | 70 ++++++++++++++++++++++ ci/setup-environment.ps1 | 8 ++- ci/standard-excludes.txt | 8 +++ 12 files changed, 298 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/ci-script-tests.yml create mode 100644 .github/workflows/release-pull-request.yml create mode 100644 ci/select-options.Tests.ps1 create mode 100644 ci/select-options.ps1 create mode 100644 ci/standard-excludes.txt diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml new file mode 100644 index 000000000..56b7b3aaf --- /dev/null +++ b/.github/workflows/ci-script-tests.yml @@ -0,0 +1,42 @@ +name: CI script tests + +# Runs the Pester tests of the scripts in ci on Linux and Windows, where the +# builds run them. The environment variable names workflow runs its own test. + +on: + pull_request: + paths: + - 'ci/**' + - '.github/workflows/ci-script-tests.yml' + push: + branches: + - main + - rc + paths: + - 'ci/**' + - '.github/workflows/ci-script-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + ci-script-tests: + name: Test the CI scripts on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Run the tests + shell: pwsh + run: | + $tests = Get-ChildItem ./ci -Filter *.Tests.ps1 | + Where-Object Name -ne 'check-environment-variable-names.Tests.ps1' + Invoke-Pester -Path $tests.FullName -CI -Output Detailed diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index e99cdcb13..eee03edfd 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -29,7 +29,7 @@ on: schedule: - cron: '10 5 * * *' pull_request: - branches: [main] + branches: [main, rc] paths: - 'pipeline/pipeline.did/**' - 'pipeline/owid-java/**' diff --git a/.github/workflows/nightly-pipeline.yml b/.github/workflows/nightly-pipeline.yml index ce793b713..54114ab6c 100644 --- a/.github/workflows/nightly-pipeline.yml +++ b/.github/workflows/nightly-pipeline.yml @@ -13,6 +13,35 @@ on: - cron: '0 3 * * *' jobs: + # The publish builds main, which the pull requests into rc do not change, so + # it runs first and has the runners before the pull request builds do. + Publish: + name: Nightly Publish + uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + dryrun: ${{ inputs.dryrun || false }} + common-ci-ref: ${{ inputs.common-ci-ref || '' }} + cache-assets: true + secrets: + token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + MavenSettings: ${{ secrets.MAVEN_SETTINGS }} + JavaGpgKeyPassphrase: ${{ secrets.JAVA_GPG_KEY_PASSPHRASE }} + JavaPGP: ${{ secrets.JAVA_KEY_PGP_FILE }} + CodeSigningKeyVaultName: ${{ secrets.CODE_SIGNING_KEY_VAULT_NAME }} + CodeSigningKeyVaultUrl: ${{ secrets.CODE_SIGNING_KEY_VAULT_URL }} + CodeSigningKeyVaultClientId: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_ID }} + CodeSigningKeyVaultTenantId: ${{ secrets.CODE_SIGNING_KEY_VAULT_TENANT_ID }} + CodeSigningKeyVaultClientSecret: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_SECRET }} + CodeSigningKeyVaultCertificateName: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_NAME }} + CodeSigningKeyVaultCertificateData: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_DATA }} + DataFileChange: name: Nightly Data File Change uses: 51Degrees/common-ci/.github/workflows/nightly-data-file-change.yml@main @@ -50,7 +79,7 @@ jobs: PullRequests: if: ${{ !cancelled() }} - needs: [DataFileChange, PackageUpdate, SubmoduleUpdate] + needs: [Publish, DataFileChange, PackageUpdate, SubmoduleUpdate] name: Nightly Pull Requests uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main with: @@ -59,6 +88,8 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true + # One configuration that fails does not cancel the others. + fail-fast: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} @@ -66,32 +97,3 @@ jobs: IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - - Publish: - if: ${{ !cancelled() }} - needs: PullRequests - name: Nightly Publish - uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - common-ci-ref: ${{ inputs.common-ci-ref || '' }} - cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - MavenSettings: ${{ secrets.MAVEN_SETTINGS }} - JavaGpgKeyPassphrase: ${{ secrets.JAVA_GPG_KEY_PASSPHRASE }} - JavaPGP: ${{ secrets.JAVA_KEY_PGP_FILE }} - CodeSigningKeyVaultName: ${{ secrets.CODE_SIGNING_KEY_VAULT_NAME }} - CodeSigningKeyVaultUrl: ${{ secrets.CODE_SIGNING_KEY_VAULT_URL }} - CodeSigningKeyVaultClientId: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_ID }} - CodeSigningKeyVaultTenantId: ${{ secrets.CODE_SIGNING_KEY_VAULT_TENANT_ID }} - CodeSigningKeyVaultClientSecret: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_SECRET }} - CodeSigningKeyVaultCertificateName: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_NAME }} - CodeSigningKeyVaultCertificateData: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_DATA }} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index c1fda910b..5a8462654 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -9,6 +9,10 @@ on: common-ci-ref: type: string default: '' + full-matrix: + description: 'Build every configuration, not only those the changes need' + type: boolean + default: false jobs: PullRequests: @@ -20,6 +24,12 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true + # Each pull request is built on the configurations its changes need, + # as ci/select-options.ps1 chooses, and one that fails does not cancel + # the others. With nothing eligible, the nightly builds rc instead. + full-matrix: ${{ inputs.full-matrix || false }} + fail-fast: false + build-without-pull-requests: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml new file mode 100644 index 000000000..4dfeb08f9 --- /dev/null +++ b/.github/workflows/release-pull-request.yml @@ -0,0 +1,31 @@ +name: Release Pull Request + +# Builds and tests the release pull request, rc into main, on every entry in +# ci/options.json, and never merges it. A pull request into rc is built only on +# the entries its changes need, so this is the full build of a release before +# it reaches main. See common-ci .github/workflows/release-pull-request.yml. +on: + pull_request: + branches: [main] + +# A push to rc while the release is open builds it again, in place of the +# build of the older rc. +concurrency: + group: release-pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + Release: + name: Release Pull Request + uses: 51Degrees/common-ci/.github/workflows/release-pull-request.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + cache-assets: true + secrets: + token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} diff --git a/ci/README.md b/ci/README.md index b00a5034a..9b205e57f 100644 --- a/ci/README.md +++ b/ci/README.md @@ -20,6 +20,18 @@ repositories used to do for one package tree each. request tests the examples against the libraries it changes. The publish workflow instead tests the examples against the version it built, through [run-integration-tests.ps1](run-integration-tests.ps1). +- [select-options.ps1](select-options.ps1) chooses the entries a pull request + runs. The standard set is JDK 17 and JDK 8 on Linux, without the three IP + Intelligence tests in [standard-excludes.txt](standard-excludes.txt) and + with the Selenium contract on JDK 17 alone. A change to a native module, IP + Intelligence, a `pom.xml` or a file outside the package trees adds to it. + macOS and performance run only when `rc` itself or the release pull request + is built, which + [release-pull-request.yml](../.github/workflows/release-pull-request.yml) + does on every entry. The Pester 5 tests of select-options.ps1 are + [select-options.Tests.ps1](select-options.Tests.ps1), which + [ci-script-tests.yml](../.github/workflows/ci-script-tests.yml) runs on + Linux and Windows for each pull request that changes this folder. - [fetch-assets.ps1](fetch-assets.ps1) fetches every data file as one of common-ci's assets, which each run caches once, and links or copies it into `cxx/data/device-detection` or `cxx/data/ip-intelligence`, where the tests diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 163ffbab4..1192dc342 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -7,7 +7,10 @@ param( [string]$TestResourceKey, [string]$DeviceDetection, [string]$IpIntelligence, - [string]$IpIntelligenceUrl + [string]$IpIntelligenceUrl, + # False, from select-options.ps1, on every entry of a pull request but + # one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) $ErrorActionPreference = "Stop" # Each part's exit code is checked below, so one failing part does not stop @@ -132,7 +135,9 @@ function Invoke-ContractTests { } Write-Host "::group::Selenium contract tests" -if (-not $TestResourceKey) { +if (-not $SeleniumContract) { + Write-Host "Another entry runs the Selenium contract tests for this pull request." +} elseif (-not $TestResourceKey) { Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the web examples." } elseif ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture -eq [System.Runtime.InteropServices.Architecture]::Arm64) { Write-Host "::warning title=Selenium skipped::Selenium contract skipped on linux-arm64, which has no selenium-manager build. It runs on x64 and macOS arm64." diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index a54755337..c905c0912 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -2,7 +2,11 @@ param( [Parameter(Mandatory)][string]$RepoName, [string]$Name, [string]$TestResourceKey, - [string]$DeviceDetection + [string]$DeviceDetection, + # How deeply a pull request is tested, as select-options.ps1 chose. + # standard leaves out the tests listed in standard-excludes.txt. Empty, for + # the branch itself or a release, runs every test. + [string]$Depth ) $ErrorActionPreference = "Stop" @@ -13,12 +17,17 @@ $ErrorActionPreference = "Stop" # does. The dev profile points the examples at the reactor. SuperResourceKey is # read by the Location and Device Detection examples tests, and LicenseKey by # the Device Detection data file update example. The IP Intelligence update -# example's test is ignored, so it does not need a key of its own. -./java/run-unit-tests.ps1 -RepoName $RepoName -Name $Name -ExtraArgs @( +# example's test is ignored, so it does not need a key of its own. A pull +# request at the standard depth leaves out the tests in standard-excludes.txt. +$extraArgs = @( '-Pdev' "-DTestResourceKey=$TestResourceKey" "-DSuperResourceKey=$TestResourceKey" "-DLicenseKey=$DeviceDetection" ) +if ($Depth -eq 'standard') { + $extraArgs += "-Dsurefire.excludesFile=$PSScriptRoot/standard-excludes.txt" +} +./java/run-unit-tests.ps1 -RepoName $RepoName -Name $Name -ExtraArgs $extraArgs exit $LASTEXITCODE diff --git a/ci/select-options.Tests.ps1 b/ci/select-options.Tests.ps1 new file mode 100644 index 000000000..e535a78a4 --- /dev/null +++ b/ci/select-options.Tests.ps1 @@ -0,0 +1,67 @@ +# Pester 5 tests of select-options.ps1, which chooses the options.json +# entries a pull request is built on from the files it changes. + +BeforeAll { + $Script = Join-Path $PSScriptRoot 'select-options.ps1' + $Options = Get-Content -Raw (Join-Path $PSScriptRoot 'options.json') + + $Package = 'pipeline/pipeline.core/src/main/java/fiftyone/A.java' + $NativeFile = 'device-detection/device-detection.hash.engine.on-premise/' + + 'src/main/java/fiftyone/A.java' + + # The entries chosen for the changed files, in options.json order. + function Select-Entries( + [string[]]$Changed, + [string[]]$CommentOnly = @()) { + $entries = @(ConvertFrom-Json $Options -AsHashtable) + @(& $Script -Options $entries -ChangedFiles $Changed ` + -CommentOnlyFiles $CommentOnly 6>$null) + } + + function Select-Names([string[]]$Changed, [string[]]$CommentOnly = @()) { + (Select-Entries $Changed $CommentOnly | ForEach-Object { $_.Name }) -join ',' + } +} + +Describe 'select-options.ps1' { + It 'builds the standard set for a change to a package' { + Select-Names $Package | Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8' + } + + It 'adds Windows and Linux ARM for a change to a native module' { + Select-Names $NativeFile | + Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8,Ubuntu_ARM_Java_17,Windows_Java_17' + } + + It 'leaves a native module change to its comments to the standard set' { + Select-Names $NativeFile $NativeFile | Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8' + } + + It 'adds Windows and Linux ARM at depth deep for the cxx submodule' { + $entries = Select-Entries 'cxx' + ($entries | ForEach-Object { $_.Name }) -join ',' | + Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8,Ubuntu_ARM_Java_17,Windows_Java_17' + $entries | ForEach-Object { $_.Depth | Should -Be 'deep' } + } + + It 'adds JDK 11 and 21 for a pom.xml change' { + Select-Names 'pipeline/pipeline.core/pom.xml' | + Should -Be 'Ubuntu_Java_21,Ubuntu_Java_17,Ubuntu_Java_11,Ubuntu_Java_8' + } + + It 'builds every entry but macOS, at depth deep, for a change outside the package trees' { + $entries = Select-Entries 'ci/run-unit-tests.ps1' + ($entries | ForEach-Object { $_.Name }) -join ',' | Should -Be ( + 'Ubuntu_Java_21,Ubuntu_Java_17,Ubuntu_Java_11,Ubuntu_Java_8,' + + 'Ubuntu_ARM_Java_17,Windows_Java_21,Windows_Java_17,' + + 'Windows_Java_11,Windows_Java_8') + $entries | ForEach-Object { $_.Depth | Should -Be 'deep' } + } + + It 'runs the Selenium contract on Ubuntu JDK 17 alone, and no performance tests' { + $entries = Select-Entries 'ci/run-unit-tests.ps1' + ($entries | Where-Object { $_.SeleniumContract } | + ForEach-Object { $_.Name }) -join ',' | Should -Be 'Ubuntu_Java_17' + $entries | Where-Object { $_.RunPerformance } | Should -BeNullOrEmpty + } +} diff --git a/ci/select-options.ps1 b/ci/select-options.ps1 new file mode 100644 index 000000000..1fb18ac53 --- /dev/null +++ b/ci/select-options.ps1 @@ -0,0 +1,70 @@ +param( + # Every entry of options.json. + [Parameter(Mandatory)][object[]]$Options, + # The files the pull request changes, less documentation. + [string[]]$ChangedFiles, + # Those of the changed files whose change touches only comments. + [string[]]$CommentOnlyFiles +) +# Chooses the options.json entries a pull request is built on, for common-ci's +# steps/select-options.ps1. The branch itself and a release build every entry. +# +# The standard set is JDK 17 and JDK 8 on Linux, the main JDK and the oldest, +# without the three IP Intelligence tests that load the Enterprise data file, +# and with the Selenium contract on JDK 17 alone. A change in more than its +# comments to a native module or the cxx submodule adds Windows and Linux ARM, +# where the native library is built differently. A change to IP Intelligence +# or the cxx submodule runs the three tests, and a change to a pom.xml adds +# JDK 11 and 21. A change to a file outside the package trees, such as ci, +# counts for every entry. Only five macOS jobs run at once across the +# organisation, and the releases need them, so macOS is left to the branch and +# release builds, as is performance. + +$Standard = @('Ubuntu_Java_17', 'Ubuntu_Java_8') +$Contract = 'Ubuntu_Java_17' +$NativeEntries = @('Windows_Java_17', 'Ubuntu_ARM_Java_17') +$Jdk = @('Ubuntu_Java_11', 'Ubuntu_Java_21') + +$PackageTrees = @('pipeline/', 'device-detection/', 'ip-intelligence/', + 'location/', 'examples/') +$NativePaths = @('device-detection/device-detection.hash.engine.on-premise/', + 'ip-intelligence/ip-intelligence.engine.on-premise/') +$IpIntelligencePaths = @('ip-intelligence/', 'examples/ip-intelligence/') + +function Test-Under([string]$Path, [string[]]$Prefixes) { + return @($Prefixes | Where-Object { $Path.StartsWith($_) }).Count -gt 0 +} + +$code = @($ChangedFiles | Where-Object { $CommentOnlyFiles -notcontains $_ }) +$cxx = @($code | Where-Object { $_ -eq 'cxx' }).Count -gt 0 +$native = $cxx -or @($code | Where-Object { Test-Under $_ $NativePaths }).Count -gt 0 +$ipi = $cxx -or + @($code | Where-Object { Test-Under $_ $IpIntelligencePaths }).Count -gt 0 +$poms = @($code | Where-Object { $_ -match '(^|/)pom\.xml$' }).Count -gt 0 +$other = @($code | Where-Object { + $_ -ne 'cxx' -and $_ -notmatch '(^|/)pom\.xml$' -and + !(Test-Under $_ $PackageTrees) +}).Count -gt 0 +Write-Host ("Native: $native. IP Intelligence: $ipi. pom.xml: $poms. " + + "Outside the package trees: $other.") + +foreach ($entry in $Options) { + $name = $entry.Name + if ($entry.Image -like 'macos*') { + Write-Host "$name is left to the branch and release builds" + continue + } + $why = if ($Standard -contains $name) { 'the standard set' } + elseif ($other) { 'a change outside the package trees' } + elseif ($native -and $NativeEntries -contains $name) { 'a native change' } + elseif ($poms -and $Jdk -contains $name) { 'a pom.xml change' } + if (!$why) { + Write-Host "$name is not needed for this change" + continue + } + $entry.Depth = ($ipi -or $other) ? 'deep' : 'standard' + $entry.SeleniumContract = $name -eq $Contract + $entry.RunPerformance = $false + Write-Host "$name runs for $why, at depth $($entry.Depth)" + $entry +} diff --git a/ci/setup-environment.ps1 b/ci/setup-environment.ps1 index ac900bbd4..23e2d11a9 100644 --- a/ci/setup-environment.ps1 +++ b/ci/setup-environment.ps1 @@ -1,6 +1,8 @@ param( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][string]$JavaSDKEnvVar + [Parameter(Mandatory)][string]$JavaSDKEnvVar, + # standard, from select-options.ps1, leaves out the tests WMIC is for. + [string]$Depth ) $ErrorActionPreference = "Stop" $PSNativeCommandUseErrorActionPreference = $true @@ -11,7 +13,9 @@ $PSNativeCommandUseErrorActionPreference = $true # Without WMIC a forked test JVM on Windows can end itself part way through # the long running data file tests. The script explains why. -& "$PSScriptRoot/install-wmic.ps1" +if ($Depth -ne 'standard') { + & "$PSScriptRoot/install-wmic.ps1" +} # The native builds run PreBuild.sh, which has to be executable whatever the # checkout did to its mode. diff --git a/ci/standard-excludes.txt b/ci/standard-excludes.txt new file mode 100644 index 000000000..03d3b4b92 --- /dev/null +++ b/ci/standard-excludes.txt @@ -0,0 +1,8 @@ +# Tests that the standard set of a pull request leaves out, for +# run-unit-tests.ps1. Each loads the multi-gigabyte IP Intelligence Enterprise +# data file. +**/fiftyone/ipintelligence/IPIntelligenceTests.* +**/fiftyone/ipintelligence/examples/console/MetadataOnPremTest.* +**/fiftyone/ipintelligence/examples/console/CompareOnPremTest.* +# Surefire's own default, which a file of excludes replaces. +**/*$* From fb2cb846372612553a63cd6ec7902168d8492bb2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 10:36:03 +0000 Subject: [PATCH 09/42] Give each package its own version and release only what changed (#63) Step 3.5 of the run book in [common-ci#306](https://github.com/51Degrees/common-ci/issues/306). [`ci/build-package.ps1` line 31](https://github.com/51Degrees/51degrees-java/blob/7b5c70fcadf02052b9bb3a5b3bbc298b5571af0a/ci/build-package.ps1#L31) ran `mvn versions:set` in each package tree, so every one of the 45 artifacts this repository publishes to Maven Central got a new version whenever anything changed. The last release, 4.5.130 on 5 November, published all 45 at that version. --- AGENTS.md | 22 +++-- README.md | 9 +- ci/README.md | 53 ++++++++-- ci/build-package.ps1 | 58 +++++++---- ci/build-project.ps1 | 8 ++ ci/check-release-interval.ps1 | 42 ++++++++ ci/check-version-bump.ps1 | 36 +++++++ ci/get-next-package-version.ps1 | 38 +++----- ci/package-update-required.ps1 | 23 +++++ ci/packages.json | 47 +++++++++ ci/packages.ps1 | 141 ++++++++++++++++++++++++++- ci/publish-package.Tests.ps1 | 101 +++++++++++++++++++ ci/publish-package.ps1 | 54 ++++++++++- ci/release-plan-filter.Tests.ps1 | 161 +++++++++++++++++++++++++++++++ ci/release-plan-filter.ps1 | 70 ++++++++++++++ ci/release.ps1 | 125 ++++++++++++++++++++++++ ci/run-integration-tests.ps1 | 21 ++-- ci/run-performance-tests.ps1 | 13 ++- ci/update-packages.ps1 | 2 +- pom.xml | 12 +-- 20 files changed, 945 insertions(+), 91 deletions(-) create mode 100644 ci/check-release-interval.ps1 create mode 100644 ci/check-version-bump.ps1 create mode 100644 ci/package-update-required.ps1 create mode 100644 ci/packages.json create mode 100644 ci/publish-package.Tests.ps1 create mode 100644 ci/release-plan-filter.Tests.ps1 create mode 100644 ci/release-plan-filter.ps1 create mode 100644 ci/release.ps1 diff --git a/AGENTS.md b/AGENTS.md index e27e80b3a..25a332221 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -239,14 +239,20 @@ follow. The root `pom.xml` lists the six package trees as modules but is not their parent, so nothing from it reaches the published poms. Each tree keeps its own -parent pom. They all carry the same version, and the libraries reference each -other through `${project.version}`, so Maven resolves those dependencies from -the reactor. The examples reference published versions unless the `dev` -profile is active. Keep it that way: a reader of the examples should see the -package versions a project of their own would use. - -Because the root is not a parent, `versions:set` has to run in each tree. CI -does that. When changing versions by hand, run it in each of the six folders. +parent pom. In the repository they all carry the same development version, +and the libraries reference each other through `${project.version}`, so Maven +resolves those dependencies from the reactor. The examples reference published +versions unless the `dev` profile is active. Keep it that way, so that a +reader of the examples sees the package versions a project of their own would +use. + +Each published module is released at its own version, tagged +`/`. The publish writes that version into the module's +`pom.xml`, with the versions of its parent and of the packages it depends on, +so leave the versions in the poms as they are. When a module is added, or one +module starts to depend on another, add it to `ci/packages.json` as well, or +the pull request build fails. `ci/README.md` describes how a release is +planned. ## Data files and `project.root` diff --git a/README.md b/README.md index 177fed64e..b5952683a 100644 --- a/README.md +++ b/README.md @@ -136,9 +136,12 @@ does the same. ## Versions -The repository has one version, and every package is released with it. CI -sets the version on each package tree before a release and tags the commit -with the plain version, such as `4.5.127`. +Each package has its own version, and a release publishes only the packages +that changed since their last release and the packages that depend on them. +A change only to documentation, tests, examples or the descriptive parts of a +`pom.xml`, such as its links, releases nothing. +Each is tagged with its name and version, such as `pipeline.core/4.5.131`. +[ci/README.md](ci/README.md) describes how CI works out what to release. ## History diff --git a/ci/README.md b/ci/README.md index 9b205e57f..4ba256c91 100644 --- a/ci/README.md +++ b/ci/README.md @@ -11,10 +11,18 @@ repositories used to do for one package tree each. publish, the two on-premise engines that build a native library, and the generated accessors. The other scripts loop over it rather than naming folders themselves. -- The root `pom.xml` builds every tree in one reactor but is not their parent, - so [build-package.ps1](build-package.ps1) has common-ci set the version in - each tree before it deploys from the root. The examples are built and tested - but left out of the deploy. +- [packages.json](packages.json) lists every published module as a package, + named by its artifactId, with its folder, the other folders its build takes + files from, and the packages it depends on, its parent included. A parent's + folder is its `pom.xml` alone, because the rest of the folder is its + modules. [build-project.ps1](build-project.ps1) fails a pull request whose + poms refer to a package that `DependsOn` does not name, or add a module + that has no entry. +- The root `pom.xml` builds every tree in one reactor but is not their parent. + [build-package.ps1](build-package.ps1) writes each published module's own + version into its `pom.xml`, with the versions of its parent and of the + packages it depends on, and deploys only the modules being released. The + examples are built and tested but left out of the deploy. - [build-project.ps1](build-project.ps1) and [run-unit-tests.ps1](run-unit-tests.ps1) use the `dev` profile, so a pull request tests the examples against the libraries it changes. The publish @@ -44,9 +52,40 @@ repositories used to do for one package tree each. API, Device Detection and IP Intelligence and writes one results file per configuration. The Device Detection and IP Intelligence metrics are prefixed with the product, as both use the same names. -- The repository has one version, tagged as a plain version such as `4.5.127`. - [get-next-package-version.ps1](get-next-package-version.ps1) moves only the - patch number. +- Each package has its own version, tagged `/`, such as + `pipeline.core/4.5.131`. A publish releases a package when a file under its + folders changed since its last tag, and every package that depends on it, + each at its last tag with the patch number moved. Changes only to Markdown, + `LICENSE` files, or files under folders named `docs`, `tests`, `test` or + `examples`, which takes in each module's `src/test`, release nothing, and + [release-plan-filter.ps1](release-plan-filter.ps1) leaves out a `pom.xml` + whose only change is to its `name`, `description`, `url`, `licenses`, + `developers`, `scm` or `issueManagement`, or to its comments and layout. + Its Pester 5 tests are + [release-plan-filter.Tests.ps1](release-plan-filter.Tests.ps1), which + [ci-script-tests.yml](../.github/workflows/ci-script-tests.yml) runs on + Linux and Windows for each pull request that changes this folder. + [release.ps1](release.ps1) takes that plan from common-ci's + `steps/release-plan.ps1`, and the publish scripts build, check and tag from + it. [check-version-bump.ps1](check-version-bump.ps1) and + [check-release-interval.ps1](check-release-interval.ps1) check each package + against its own tags, for a major or minor move and for a release less than + a week after its last. +- [publish-package.ps1](publish-package.ps1) uploads the one publishing + bundle, then pushes every package's tag in one atomic push, so a publish + that stops leaves every package tagged or none, and then makes one GitHub + release per tag. Maven Central refuses a version it already has, so when + the push fails after the upload, push the tags the log names by hand + rather than running the publish again. When `gh release create` fails + part way, every tag is pushed but the packages after the failure have no + GitHub release, and a publish run again plans nothing and so makes none. + Make each missing release by hand. For each tag the log names after + "Pushing" for which `gh release view ` finds no release, run + `gh release create --verify-tag --title " " + --notes " : ."`, with the reason the Configure + job's plan gives for that package, as the script does. Its Pester 5 + tests are [publish-package.Tests.ps1](publish-package.Tests.ps1), which the + same workflow runs. ## Secrets diff --git a/ci/build-package.ps1 b/ci/build-package.ps1 index 75464f09f..99df8cfa3 100644 --- a/ci/build-package.ps1 +++ b/ci/build-package.ps1 @@ -7,7 +7,15 @@ param( $ErrorActionPreference = "Stop" $PSNativeCommandUseErrorActionPreference = $true -. "$PSScriptRoot/packages.ps1" +. "$PSScriptRoot/release.ps1" + +# Builds the packages this publish releases, each at its own version from the +# release plan (see release.ps1). $Version is common-ci's label for the run and +# is not used. +$plan = @(Get-BuildPlan $RepoName) +Write-Host "Building $($plan.Count) packages:" +$plan | Format-Table Name, Previous, Version, Reason -AutoSize | Out-String -Width 200 | + Write-Host # Put the native libraries built on every platform into the engine they came # from, so each engine's jar carries all of its own and none of the other's. @@ -21,30 +29,25 @@ foreach ($package in $NativePackages) { } } -# The examples are never published. They are left out of the deploy so that -# the last project in the reactor is one the publishing plugin is configured -# for. Maven 3 does not exclude a folder's modules with the folder, so each of -# them is named. -$repoPath = (Resolve-Path $RepoName).Path -$examples = foreach ($package in $Packages | Where-Object { -not $_.Publish }) { - Get-ChildItem -Path "$RepoName/$($package.Path)" -Recurse -Filter pom.xml | - Where-Object { $_.FullName -notmatch '[\\/](target|node_modules)[\\/]' } | - ForEach-Object { '!' + [IO.Path]::GetRelativePath($repoPath, $_.DirectoryName).Replace('\', '/') } -} +# Every published module gets its own version, and the versions of its parent +# and of the packages it depends on, including those this publish does not +# release, which stay at their last release. +Set-PomVersions $RepoName (Get-BuildVersions $RepoName $plan) + +# Only the modules in the plan are deployed, so the one publishing bundle holds +# only them. A package they depend on that is not released is taken from Maven +# Central at its last release, the version their poms now require. +$modules = @($plan | ForEach-Object { Get-ModulePath $_ }) -# common-ci sets the version, deploys from the root to a local staging folder -# with signing, and copies the result and the publishing bundle into the -# package folder. The root pom.xml is not the parent of the package trees, so -# setting the version at the root would not reach them. VersionPaths names each -# tree, the examples included, so the whole repository carries the version -# being released. The publishing plugin writes the one bundle for the whole -# reactor under the first tree that publishes, and common-ci finds it there. +# common-ci deploys from the root to a local staging folder with signing, and +# copies the result and the publishing bundle into the package folder. The +# versions are already set, so it is given no folder to set them in. ./java/build-package.ps1 ` -RepoName $RepoName ` -Name $Name ` -Version $Version ` - -VersionPaths $Packages.Path ` - -ExtraArgs "-DskipNativeBuild=true", "-pl", ($examples -join ',') ` + -VersionPaths @() ` + -ExtraArgs "-DskipNativeBuild=true", "-pl", ($modules -join ',') ` -JavaGpgKeyPassphrase $Keys['JavaGpgKeyPassphrase'] ` -JavaPGP $Keys['JavaPGP'] ` -CodeSigningKeyVaultName: $Keys['CodeSigningKeyVaultName'] ` @@ -54,5 +57,18 @@ $examples = foreach ($package in $Packages | Where-Object { -not $_.Publish }) { -CodeSigningKeyVaultClientSecret $Keys['CodeSigningKeyVaultClientSecret'] ` -CodeSigningKeyVaultCertificateName $Keys['CodeSigningKeyVaultCertificateName'] ` -CodeSigningKeyVaultCertificateData $Keys['CodeSigningKeyVaultCertificateData'] +$exitCode = $LASTEXITCODE + +# Maven Central limits each organisation's release size and file count in a +# calendar month, so the log gives both for the bundle this publish uploads. +$bundle = Get-Item package/central-bundle.zip +$zip = [IO.Compression.ZipFile]::OpenRead($bundle.FullName) +try { + $files = $zip.Entries.Count +} finally { + $zip.Dispose() +} +Write-Host ("The publishing bundle for $($plan.Count) packages is " + + "$($bundle.Length) bytes ({0:N1} MiB) in $files files" -f ($bundle.Length / 1MB)) -exit $LASTEXITCODE +exit $exitCode diff --git a/ci/build-project.ps1 b/ci/build-project.ps1 index d8524aa3e..eea288f3f 100644 --- a/ci/build-project.ps1 +++ b/ci/build-project.ps1 @@ -9,6 +9,14 @@ $PSNativeCommandUseErrorActionPreference = $true Write-Host "JAVA_HOME: $ENV:JAVA_HOME" +# A publish releases a package's dependents only when packages.json names +# them, so a module or dependency it leaves out fails here, before the change +# is merged, rather than at the publish. +$gaps = @(Get-PackagesJsonGaps $RepoName) +if ($gaps.Count -gt 0) { + throw "ci/packages.json does not match the poms:`n$($gaps -join "`n")" +} + # One build of the whole reactor, native libraries included. The dev profile # builds the examples against the libraries here rather than the versions # published to Maven Central, so a change to a library is tested by the diff --git a/ci/check-release-interval.ps1 b/ci/check-release-interval.ps1 new file mode 100644 index 000000000..305e15512 --- /dev/null +++ b/ci/check-release-interval.ps1 @@ -0,0 +1,42 @@ +param ( + [Parameter(Mandatory)][string]$RepoName, + [string]$Repository, + [string]$HowToAllow +) +$ErrorActionPreference = "Stop" +. "$PSScriptRoot/release.ps1" + +# The shared publish runs this in place of its own check, because each package +# has its own version and tags (see release.ps1). Every package about to be +# published goes through the shared steps/check-release-interval.ps1, run from +# the common-ci root, against its own '/' tags, so each +# package gets at most one new version a week. That refuses a package +# released less than a week ago unless ALLOW_EARLY_RELEASE is 'true', from the +# date in WEEKLY_RELEASES_FROM. + +$plan = @(Get-ReleasePlan $RepoName) +if ($plan.Count -eq 0) { + Write-Host "No package is to be published, so there is nothing to check." + exit 0 +} + +$arguments = @{ RepoName = $RepoName; Repository = $Repository } +if ($HowToAllow) { + $arguments.HowToAllow = $HowToAllow +} + +$refused = @() +foreach ($release in $plan) { + Write-Host "$($release.Name):" + $global:LASTEXITCODE = 0 + ./steps/check-release-interval.ps1 @arguments -TagPrefix $release.Name + if ($LASTEXITCODE -ne 0) { + $refused += $release.Name + } +} + +if ($refused) { + Write-Host "Nothing is released, because these packages were released less than a week ago: $($refused -join ', ')." + exit 1 +} +exit 0 diff --git a/ci/check-version-bump.ps1 b/ci/check-version-bump.ps1 new file mode 100644 index 000000000..f53fae22d --- /dev/null +++ b/ci/check-version-bump.ps1 @@ -0,0 +1,36 @@ +param ( + [Parameter(Mandatory)][string]$RepoName, + [string]$Version +) +$ErrorActionPreference = "Stop" +. "$PSScriptRoot/release.ps1" + +# The shared publish runs this in place of its own check, because each package +# has its own version (see release.ps1). Every package about to be published +# goes through the shared steps/check-version-bump.ps1, run from the common-ci +# root, against its own '/' tags. That refuses a major or +# minor move unless ALLOW_VERSION_BUMP is 'true'. $Version is common-ci's label +# for the run. + +Write-Host "The publish worked out $Version, its highest. Each package is checked on its own." + +$plan = @(Get-ReleasePlan $RepoName) +if ($plan.Count -eq 0) { + Write-Host "No package is to be published, so there is nothing to check." + exit 0 +} + +$refused = @() +foreach ($release in $plan) { + $global:LASTEXITCODE = 0 + ./steps/check-version-bump.ps1 -Version $release.Version -RepoName $RepoName -TagPrefix $release.Name + if ($LASTEXITCODE -ne 0) { + $refused += $release.Name + } +} + +if ($refused) { + Write-Host "Nothing is published, because these packages were refused: $($refused -join ', ')." + exit 1 +} +exit 0 diff --git a/ci/get-next-package-version.ps1 b/ci/get-next-package-version.ps1 index de0e6c291..ddfc48d64 100644 --- a/ci/get-next-package-version.ps1 +++ b/ci/get-next-package-version.ps1 @@ -3,31 +3,21 @@ param ( [Parameter(Mandatory)][string]$VariableName ) $ErrorActionPreference = "Stop" +. "$PSScriptRoot/release.ps1" -# The whole repository has one version, tagged as a plain version such as -# 4.5.127. Only the patch number moves here. The major and minor numbers -# change only when someone edits FirstVersion before the first release or -# pushes a tag by hand. - -# The version used while the repository has no version tag. It is above every -# version the absorbed repositories published, so the first release from here -# is an ordinary upgrade for each package. -$FirstVersion = '4.5.127' - -$tags = @(git -C $RepoName tag --list | - Where-Object { $_ -match '^\d+\.\d+\.\d+$' } | - Sort-Object { [version]$_ }) -$headTags = @(git -C $RepoName tag --points-at HEAD | - Where-Object { $_ -match '^\d+\.\d+\.\d+$' }) - -if ($headTags) { - # Already released, so the publish sees the tag and does nothing. - $next = $headTags | Sort-Object { [version]$_ } | Select-Object -Last 1 -} elseif ($tags) { - $highest = [version]$tags[-1] - $next = "$($highest.Major).$($highest.Minor).$($highest.Build + 1)" +# common-ci's publish carries one version through its jobs. This repository +# releases each package at its own version (see release.ps1), so that one +# version is only a label for the run, being the highest version this publish +# releases. Whether anything is released is decided by +# package-update-required.ps1, and each package's version and tag by +# build-package.ps1 and publish-package.ps1, all from the same release plan. +$plan = @(Get-ReleasePlan $RepoName) +if ($plan.Count -gt 0) { + $label = ($plan.Version | Sort-Object { [version]$_ })[-1] } else { - $next = $FirstVersion + # Nothing to release. Any valid version will do, because + # package-update-required.ps1 stops the publish before it is used. + $label = '4.5.0' } -Set-Variable -Scope Global -Name $VariableName -Value $next +Set-Variable -Scope Global -Name $VariableName -Value $label diff --git a/ci/package-update-required.ps1 b/ci/package-update-required.ps1 new file mode 100644 index 000000000..da231faad --- /dev/null +++ b/ci/package-update-required.ps1 @@ -0,0 +1,23 @@ +param ( + [Parameter(Mandatory)][string]$RepoName, + [string]$Version +) +$ErrorActionPreference = "Stop" +. "$PSScriptRoot/release.ps1" + +# Called by common-ci's publish in place of its own check, because a plain tag +# lookup cannot tell this repository's packages apart. Exits 0 when the release +# plan has something in it and 1 when it is empty, as common-ci's check does. +# $Version is common-ci's label for the run and is not used here. Each +# package's version is checked by check-version-bump.ps1. + +$plan = @(Get-ReleasePlan $RepoName) +if ($plan.Count -eq 0) { + Write-Output "No package changed since its last release, so there is nothing to publish" + exit 1 +} + +Write-Output "This publish releases $($plan.Count) packages, in this order:" +$plan | Format-Table Name, Previous, Version, Reason -AutoSize | Out-String -Width 200 | + Write-Output +exit 0 diff --git a/ci/packages.json b/ci/packages.json new file mode 100644 index 000000000..153c48995 --- /dev/null +++ b/ci/packages.json @@ -0,0 +1,47 @@ +[ + {"Name": "pipeline", "Path": "pipeline/pom.xml", "DependsOn": [], "Publish": true}, + {"Name": "pipeline.common", "Path": "pipeline/pipeline.common", "DependsOn": ["pipeline"], "Publish": true}, + {"Name": "pipeline.core", "Path": "pipeline/pipeline.core", "DependsOn": ["pipeline", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.did", "Path": "pipeline/pipeline.did", "ExtraPaths": ["pipeline/owid-java"], "DependsOn": ["pipeline"], "Publish": true}, + {"Name": "pipeline.engines", "Path": "pipeline/pipeline.engines", "DependsOn": ["pipeline", "pipeline.core", "pipeline.caching", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.caching", "Path": "pipeline/pipeline.caching", "DependsOn": ["pipeline", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.cloudrequestengine", "Path": "pipeline/pipeline.cloudrequestengine", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.engines.fiftyone", "Path": "pipeline/pipeline.engines.fiftyone", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.translation", "Path": "pipeline/pipeline.translation", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.core"], "Publish": true}, + {"Name": "pipeline.javascriptbuilder", "Path": "pipeline/pipeline.javascriptbuilder", "DependsOn": ["pipeline", "pipeline.jsonbuilder", "pipeline.engines", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.jsonbuilder", "Path": "pipeline/pipeline.jsonbuilder", "DependsOn": ["pipeline", "pipeline.engines.fiftyone", "pipeline.common", "pipeline.engines"], "Publish": true}, + {"Name": "pipeline.web.packages", "Path": "pipeline/pipeline.web.packages/pom.xml", "DependsOn": ["pipeline"], "Publish": true}, + {"Name": "pipeline.web.shared", "Path": "pipeline/pipeline.web.packages/pipeline.web.shared", "DependsOn": ["pipeline.web.packages", "pipeline.javascriptbuilder"], "Publish": true}, + {"Name": "pipeline.web", "Path": "pipeline/pipeline.web.packages/pipeline.web", "DependsOn": ["pipeline.web.packages", "pipeline.web.shared", "pipeline.core"], "Publish": true}, + {"Name": "pipeline.web.mvc", "Path": "pipeline/pipeline.web.packages/pipeline.web.mvc", "DependsOn": ["pipeline.web.packages", "pipeline.web", "pipeline.core"], "Publish": true}, + {"Name": "pipeline-aggregate", "Path": "pipeline/pipeline.aggregate", "DependsOn": ["pipeline", "pipeline.web", "pipeline.javascriptbuilder", "pipeline.cloudrequestengine"], "Publish": true}, + {"Name": "pipeline.developer-examples", "Path": "pipeline/pipeline.developer-examples/pom.xml", "DependsOn": ["pipeline"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.web.shared", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.web.shared", "DependsOn": ["pipeline.developer-examples"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.flowelement", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement", "DependsOn": ["pipeline.developer-examples", "pipeline.core"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.onpremiseengine", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine", "DependsOn": ["pipeline.developer-examples", "pipeline.engines"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.cloudengine", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine", "DependsOn": ["pipeline.developer-examples", "pipeline.cloudrequestengine"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.clientsideelement", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element", "DependsOn": ["pipeline.developer-examples", "pipeline.engines"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.clientsideelement.mvc", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc", "DependsOn": ["pipeline.developer-examples", "pipeline.developerexamples.clientsideelement", "pipeline.web.mvc", "pipeline.developerexamples.web.shared"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.usagesharing", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing", "DependsOn": ["pipeline.developer-examples", "pipeline.engines.fiftyone"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.fodid", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid", "DependsOn": ["pipeline.developer-examples", "pipeline.did"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.device-detection", "Path": "device-detection/pom.xml", "DependsOn": ["pipeline-aggregate"], "Publish": true}, + {"Name": "device-detection.shared", "Path": "device-detection/device-detection.shared", "DependsOn": ["pipeline.device-detection", "pipeline.engines.fiftyone"], "Publish": true}, + {"Name": "device-detection.hash.engine.on-premise", "Path": "device-detection/device-detection.hash.engine.on-premise", "ExtraPaths": ["cxx"], "DependsOn": ["pipeline.device-detection", "device-detection.shared"], "Publish": true}, + {"Name": "device-detection.cloud", "Path": "device-detection/device-detection.cloud", "DependsOn": ["pipeline.device-detection", "device-detection.shared", "pipeline.cloudrequestengine"], "Publish": true}, + {"Name": "device-detection", "Path": "device-detection/device-detection", "DependsOn": ["pipeline.device-detection", "device-detection.hash.engine.on-premise", "device-detection.cloud", "pipeline-aggregate", "device-detection.shared"], "Publish": true}, + {"Name": "pipeline.ip-intelligence", "Path": "ip-intelligence/pom.xml", "DependsOn": ["pipeline-aggregate"], "Publish": true}, + {"Name": "ip-intelligence.shared", "Path": "ip-intelligence/ip-intelligence.shared", "DependsOn": ["pipeline.ip-intelligence", "pipeline.engines.fiftyone"], "Publish": true}, + {"Name": "ip-intelligence.translation", "Path": "ip-intelligence/ip-intelligence.translation", "ExtraPaths": ["cxx"], "DependsOn": ["pipeline.ip-intelligence", "ip-intelligence.shared", "pipeline.translation", "ip-intelligence.engine.on-premise"], "Publish": true}, + {"Name": "ip-intelligence.engine.on-premise", "Path": "ip-intelligence/ip-intelligence.engine.on-premise", "ExtraPaths": ["cxx"], "DependsOn": ["pipeline.ip-intelligence", "ip-intelligence.shared"], "Publish": true}, + {"Name": "ip-intelligence.cloud", "Path": "ip-intelligence/ip-intelligence.cloud", "DependsOn": ["pipeline.ip-intelligence", "ip-intelligence.shared", "pipeline.cloudrequestengine"], "Publish": true}, + {"Name": "ip-intelligence", "Path": "ip-intelligence/ip-intelligence", "DependsOn": ["pipeline.ip-intelligence", "ip-intelligence.engine.on-premise", "ip-intelligence.cloud", "pipeline-aggregate", "ip-intelligence.shared"], "Publish": true}, + {"Name": "pipeline.geo-location", "Path": "location/pom.xml", "DependsOn": [], "Publish": true}, + {"Name": "geo-location.core", "Path": "location/geo-location.core", "DependsOn": ["pipeline.geo-location", "pipeline.cloudrequestengine"], "Publish": true}, + {"Name": "geo-location.cloud", "Path": "location/geo-location.cloud", "DependsOn": ["pipeline.geo-location", "geo-location.core", "pipeline.cloudrequestengine", "pipeline.engines.fiftyone"], "Publish": true}, + {"Name": "geo-location", "Path": "location/geo-location", "DependsOn": ["pipeline.geo-location", "geo-location.cloud", "pipeline.cloudrequestengine", "pipeline.common"], "Publish": true}, + {"Name": "geo-location.examples", "Path": "location/geo-location.shell.examples", "DependsOn": ["pipeline.geo-location", "geo-location", "device-detection", "pipeline.engines"], "Publish": true, "Ignore": ["location/geo-location.shell.examples/**"]}, + {"Name": "geo-location.web.examples", "Path": "location/geo-location.web.examples/pom.xml", "DependsOn": ["pipeline.geo-location"], "Publish": true, "Ignore": ["location/geo-location.web.examples/**"]}, + {"Name": "geo-location.web.examples.mvc", "Path": "location/geo-location.web.examples/geo-location.web.examples.mvc", "DependsOn": ["geo-location.web.examples", "pipeline.web.mvc", "geo-location.cloud", "geo-location.web.examples.shared"], "Publish": true, "Ignore": ["location/geo-location.web.examples/**"]}, + {"Name": "geo-location.web.examples.servlet", "Path": "location/geo-location.web.examples/geo-location.web.examples.servlet", "DependsOn": ["geo-location.web.examples", "pipeline.web", "geo-location.cloud", "geo-location.web.examples.shared"], "Publish": true, "Ignore": ["location/geo-location.web.examples/**"]}, + {"Name": "geo-location.web.examples.shared", "Path": "location/geo-location.web.examples/geo-location.web.examples.shared", "DependsOn": ["geo-location.web.examples"], "Publish": true, "Ignore": ["location/geo-location.web.examples/**"]} +] diff --git a/ci/packages.ps1 b/ci/packages.ps1 index e1319727e..e3e153f2d 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -2,10 +2,12 @@ # relative to the repository root. # The package trees in this repository. The root pom.xml builds them all in -# one reactor but is not their parent, so a version change has to be made in -# each of them. Publish says whether the tree's packages go to Maven Central; -# the examples are built and tested but never published. -$Packages = @( +# one reactor but is not their parent. Publish says whether the tree's +# packages go to Maven Central. The examples are built and tested but never +# published. Each published module is a package of its own in packages.json. +# The example modules inside the published trees are packages too, and give +# an Ignore glob over their whole folder, so a change to one releases nothing. +$Trees = @( @{ Path = 'pipeline'; Publish = $true } @{ Path = 'device-detection'; Publish = $true } @{ Path = 'ip-intelligence'; Publish = $true } @@ -40,3 +42,134 @@ $Accessors = @( # Arguments every Maven call in these scripts passes. $MavenOptions = '--batch-mode', '--no-transfer-progress' + +$PomNamespace = 'http://maven.apache.org/POM/4.0.0' + +# The packages in packages.json that are published, one per Maven module, +# each named by its artifactId. +function Get-PublishedPackages([string]$RepoPath) { + return @(Get-Content -Raw (Join-Path $RepoPath 'ci/packages.json') | + ConvertFrom-Json | Where-Object { $_.Publish -eq $true }) +} + +# A parent's Path is its pom.xml alone, because its folder also holds its +# modules, whose changes are not changes to the parent. +function Get-PomPath($Package) { + if ($Package.Path.EndsWith('/pom.xml') -eq $true) { + return $Package.Path + } + return "$($Package.Path)/pom.xml" +} + +# The folder Maven builds a package in. +function Get-ModulePath($Package) { + return (Get-PomPath $Package) -replace '/pom\.xml$', '' +} + +# A pom.xml read with its whitespace kept, so that writing it back changes +# only what was set. +function Read-Pom([string]$Path) { + # .NET resolves a relative path from the process's folder, which is not + # the folder PowerShell is in after a Set-Location. + $Path = (Resolve-Path -LiteralPath $Path).ProviderPath + $xml = [Xml.XmlDocument]::new() + $xml.PreserveWhitespace = $true + $xml.Load($Path) + $namespaces = [Xml.XmlNamespaceManager]::new($xml.NameTable) + $namespaces.AddNamespace('m', $PomNamespace) + return [pscustomobject]@{ Path = $Path; Xml = $xml; Ns = $namespaces } +} + +function Save-Pom($Pom) { + $settings = [Xml.XmlWriterSettings]::new() + $settings.Encoding = [Text.UTF8Encoding]::new($false) + $settings.NewLineHandling = [Xml.NewLineHandling]::None + $writer = [Xml.XmlWriter]::Create($Pom.Path, $settings) + try { + $Pom.Xml.Save($writer) + } finally { + $writer.Dispose() + } +} + +function Get-PomText($Node, [string]$Name, $Ns) { + $child = $Node.SelectSingleNode("m:$Name", $Ns) + if ($null -eq $child) { + return $null + } + return $child.InnerText.Trim() +} + +# The other packages a pom.xml refers to, being its parent and its +# dependencies on them in any scope, wherever they are declared. Version is the element that +# holds the version, or $null where a parent pom manages it. +function Get-PomReferences($Pom, [string[]]$Names) { + $self = Get-PomText $Pom.Xml.DocumentElement 'artifactId' $Pom.Ns + $parent = $Pom.Xml.SelectSingleNode('/m:project/m:parent', $Pom.Ns) + if ($null -ne $parent) { + $name = Get-PomText $parent 'artifactId' $Pom.Ns + if ($Names -contains $name) { + [pscustomobject]@{ + Name = $name + Version = $parent.SelectSingleNode('m:version', $Pom.Ns) + } + } + } + foreach ($dependency in $Pom.Xml.SelectNodes('//m:dependency', $Pom.Ns)) { + $name = Get-PomText $dependency 'artifactId' $Pom.Ns + $group = Get-PomText $dependency 'groupId' $Pom.Ns + if ($Names -contains $name -and $name -ne $self -and + $group -in 'com.51degrees', '${project.groupId}') { + [pscustomobject]@{ + Name = $name + Version = $dependency.SelectSingleNode('m:version', $Pom.Ns) + } + } + } +} + +# The modules of the published trees, as folders, read from each pom's +# modules. +function Get-PublishedModules([string]$RepoPath) { + $folders = [Collections.Generic.Queue[string]]::new() + foreach ($tree in $Trees | Where-Object { $_.Publish -eq $true }) { + $folders.Enqueue($tree.Path) + } + while ($folders.Count -gt 0) { + $folder = $folders.Dequeue() + $folder + $pom = Read-Pom (Join-Path $RepoPath $folder 'pom.xml') + foreach ($module in $pom.Xml.SelectNodes('/m:project/m:modules/m:module', $pom.Ns)) { + $folders.Enqueue("$folder/$($module.InnerText.Trim())") + } + } +} + +# What packages.json leaves out, as one line each. Every published module must +# be a package, named by its artifactId, and each package's DependsOn must name +# every other package its pom.xml refers to. A dependency it leaves out would +# not be released when the package it depends on is, so its users would not +# get the fix. +function Get-PackagesJsonGaps([string]$RepoPath) { + $packages = Get-PublishedPackages $RepoPath + $names = @($packages.Name) + $folders = @($packages | ForEach-Object { Get-ModulePath $_ }) + foreach ($folder in Get-PublishedModules $RepoPath) { + if ($folders -notcontains $folder) { + "$folder/pom.xml is a published module with no entry in ci/packages.json" + } + } + foreach ($package in $packages) { + $pom = Read-Pom (Join-Path $RepoPath (Get-PomPath $package)) + $artifactId = Get-PomText $pom.Xml.DocumentElement 'artifactId' $pom.Ns + if ($artifactId -ne $package.Name) { + "$(Get-PomPath $package) is $artifactId, but ci/packages.json names it $($package.Name)" + } + $missing = @(Get-PomReferences $pom $names | + Where-Object { $package.DependsOn -notcontains $_.Name } | + ForEach-Object Name | Select-Object -Unique) + foreach ($name in $missing) { + "$(Get-PomPath $package) refers to $name, which the DependsOn of $($package.Name) in ci/packages.json does not name" + } + } +} diff --git a/ci/publish-package.Tests.ps1 b/ci/publish-package.Tests.ps1 new file mode 100644 index 000000000..5a4c177b0 --- /dev/null +++ b/ci/publish-package.Tests.ps1 @@ -0,0 +1,101 @@ +# Pester 5 tests of publish-package.ps1. The upload script is a stub, and git, +# gh and the release plan are mocked, so each test reads only what the publish +# would tag, push and release. + +BeforeAll { + $Script = Join-Path $PSScriptRoot 'publish-package.ps1' + + # release.ps1 needs common-ci's steps/release-plan.ps1, and the publish + # runs ./java/publish-package-maven.ps1 from the folder it is run in. + $Root = Join-Path ([IO.Path]::GetTempPath()) "publish-package-$PID" + $null = New-Item -ItemType Directory -Force ` + (Join-Path $Root 'common-ci' 'steps'), (Join-Path $Root 'java') + Set-Content (Join-Path $Root 'common-ci' 'steps' 'release-plan.ps1') ` + 'function Get-ReleasePlan([string]$RepoPath) { }' + Set-Content (Join-Path $Root 'java' 'publish-package-maven.ps1') ` + 'param($RepoName, $MavenSettings, $Version, $DryRun)' + $script:CommonCi = $env:COMMON_CI + $env:COMMON_CI = Join-Path $Root 'common-ci' + + # Pester mocks only a command it can find, so the plan is defined here + # as well as in the stub. + function Get-ReleasePlan([string]$RepoPath) { } + + function Invoke-Publish([bool]$DryRun = $false) { + Push-Location $Root + try { + & $Script -RepoName 'repo' -OrgName 'org' -Version '0' ` + -Keys @{ MavenSettings = '' } -DryRun $DryRun + } finally { + Pop-Location + } + } +} + +AfterAll { + $env:COMMON_CI = $script:CommonCi + Remove-Item -Recurse -Force $Root +} + +Describe 'publish-package.ps1' { + BeforeEach { + $global:SkipUpdateTag = $false + Mock git { } + Mock gh { } + Mock Get-ReleasePlan { + [pscustomobject]@{ + Name = 'pipeline.core'; Version = '4.5.131' + Previous = $null; Reason = 'changed' + } + [pscustomobject]@{ + Name = 'pipeline.engines'; Version = '4.5.131' + Previous = $null; Reason = 'depends on pipeline.core' + } + } + } + + It 'pushes every tag in one atomic push' { + Invoke-Publish + Should -Invoke git -Times 1 -Exactly -ParameterFilter { + $args -contains 'push' + } + Should -Invoke git -Times 1 -Exactly -ParameterFilter { + $args -contains 'push' -and $args -contains '--atomic' -and + $args -contains 'pipeline.core/4.5.131' -and + $args -contains 'pipeline.engines/4.5.131' + } + } + + It 'makes one release for each tag' { + Invoke-Publish + Should -Invoke gh -Times 2 -Exactly + $global:SkipUpdateTag | Should -BeTrue + } + + It 'makes no release when the push fails' { + Mock git { if ($args -contains 'push') { throw 'push refused' } } + { Invoke-Publish } | Should -Throw '*push refused*' + Should -Invoke gh -Times 0 -Exactly + } + + It 'pushes nothing and makes no release in a dry run' { + Invoke-Publish -DryRun $true + Should -Invoke git -Times 2 -Exactly -ParameterFilter { + $args -contains 'tag' + } + Should -Invoke git -Times 0 -Exactly -ParameterFilter { + $args -contains 'push' + } + Should -Invoke gh -Times 0 -Exactly + $global:SkipUpdateTag | Should -BeTrue + } + + It 'pushes nothing when the plan is empty' { + Mock Get-ReleasePlan { } + Invoke-Publish + Should -Invoke git -Times 0 -Exactly -ParameterFilter { + $args -contains 'push' + } + Should -Invoke gh -Times 0 -Exactly + } +} diff --git a/ci/publish-package.ps1 b/ci/publish-package.ps1 index f2ee093f3..59401d1ed 100644 --- a/ci/publish-package.ps1 +++ b/ci/publish-package.ps1 @@ -1,12 +1,60 @@ param( [Parameter(Mandatory)][string]$RepoName, + [Parameter(Mandatory)][string]$OrgName, [Parameter(Mandatory)][string]$Version, [Parameter(Mandatory)][Hashtable]$Keys, [bool]$DryRun = $false ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -# Uploads the one publishing bundle build-package.ps1 made, which holds the -# packages of every tree that publishes. +. "$PSScriptRoot/release.ps1" + +# The plan is read before anything is tagged, because the tags made below +# change what the plan would say. +$plan = @(Get-ReleasePlan $RepoName) + +# Uploads the one publishing bundle build-package.ps1 made, which holds every +# package this publish releases. ./java/publish-package-maven.ps1 -RepoName $RepoName -MavenSettings $Keys['MavenSettings'] -Version $Version -DryRun $DryRun -exit $LASTEXITCODE +# One tag and one GitHub release per package, '/', in place +# of common-ci's single tag, which would name no package. The tags go up in one +# atomic push, so a publish that stops leaves every package tagged or none. +# A release tagged in part would leave the untagged dependants out of the plan +# of a run made again, because the plan skips a package tagged here and finds +# dependants only from packages that changed. +$tags = @($plan | ForEach-Object { "$($_.Name)/$($_.Version)" }) +foreach ($tag in $tags) { + git -C $RepoName tag $tag +} +if ($tags.Count -eq 0 -or $DryRun -eq $true) { + # A push with no tags named would push the branch instead. + Write-Output "Dry run or empty plan - tagged '$($tags -join ', ')' locally, not pushed, no release made" + $global:SkipUpdateTag = $true + return +} +Write-Output "Pushing $($tags -join ', ')" +git -C $RepoName push --atomic origin @tags + +# The notes list the commits that touched the package's files since its last +# release. +foreach ($release in $plan) { + $tag = "$($release.Name)/$($release.Version)" + $notes = "$($release.Name) $($release.Version): $($release.Reason)." + if ($null -ne $release.Previous) { + $package = Get-PublishedPackages $RepoName | + Where-Object { $_.Name -eq $release.Name } + $range = "$($release.Name)/$($release.Previous)..HEAD" + $commits = git -C $RepoName log --format='- %s (%h)' $range ` + -- (Get-PackagePaths $package) + if ($commits) { + $notes += "`n`n" + ($commits -join "`n") + } + } + gh -R "$OrgName/$RepoName" release create $tag --verify-tag ` + --title "$($release.Name) $($release.Version)" --notes $notes +} + +# Tells common-ci's publish not to add its own single tag and release. +$global:SkipUpdateTag = $true diff --git a/ci/release-plan-filter.Tests.ps1 b/ci/release-plan-filter.Tests.ps1 new file mode 100644 index 000000000..62a854a43 --- /dev/null +++ b/ci/release-plan-filter.Tests.ps1 @@ -0,0 +1,161 @@ +# Pester 5 tests of release-plan-filter.ps1. Each test makes a repository +# whose pom.xml is tagged as released, changes it in a second commit, and asks +# the filter whether the change counts. + +BeforeAll { + $Filter = Join-Path $PSScriptRoot 'release-plan-filter.ps1' + + $Released = @' + + + 4.0.0 + com.51degrees + pipeline.core + 4.5.130 + 51Degrees :: Pipeline :: Core + Pipeline core + https://example.com + + EUPL-1.2 + + + 51Degrees + + + https://github.com/51Degrees/pipeline-java + + + https://github.com/51Degrees/pipeline-java/issues + + + + central + https://repo1.maven.org/maven2 + + + + + org.json + json + 20240303 + + + + + + maven-jar-plugin + 3.3.0 + + + + +'@ + + # A repository holding $Released at the tag 'pkg/4.5.130', then the given + # files in a second commit. Returns the repository's folder. + function New-ChangedRepo([hashtable]$Files, [string[]]$Remove = @()) { + $repo = Join-Path $TestDrive ([Guid]::NewGuid().ToString('N')) + $null = New-Item -ItemType Directory -Path (Join-Path $repo 'pkg') + Set-Content -LiteralPath (Join-Path $repo 'pkg/pom.xml') -Value $Released -NoNewline + Set-Content -LiteralPath (Join-Path $repo 'pkg/Core.java') -Value 'class Core {}' -NoNewline + git -C $repo init -q + git -C $repo config user.email 'test@example.com' + git -C $repo config user.name 'Test' + git -C $repo config core.autocrlf false + git -C $repo add -A + git -C $repo commit -q -m 'released' + git -C $repo tag 'pkg/4.5.130' + foreach ($path in $Files.Keys) { + $full = Join-Path $repo $path + $null = New-Item -ItemType Directory -Force -Path (Split-Path $full) + Set-Content -LiteralPath $full -Value $Files[$path] -NoNewline + } + foreach ($path in $Remove) { + git -C $repo rm -q $path + } + git -C $repo add -A + git -C $repo commit -q --allow-empty -m 'changed' + return $repo + } + + # The released pom with one piece of text replaced, which must be there. + function Edit-Released([string]$Old, [string]$New) { + if ($Released.Contains($Old) -eq $false) { + throw "The released pom has no '$Old'." + } + return $Released.Replace($Old, $New) + } + + function Invoke-Filter([string]$Repo, [string[]]$Files) { + return @(& $Filter -RepoPath $Repo -Package 'pkg' -From 'pkg/4.5.130' -Files $Files) + } +} + +Describe 'release-plan-filter.ps1' { + It 'passes a file that is not a pom.xml through' { + $repo = New-ChangedRepo @{ 'pkg/Core.java' = 'class Core { int a; }' } + Invoke-Filter $repo @('pkg/Core.java') | Should -Be @('pkg/Core.java') + } + + It 'leaves out a pom.xml whose changed' -ForEach @( + @{ Metadata = 'name'; Old = 'Pipeline :: Core'; New = 'Pipeline :: Core API' } + @{ Metadata = 'description'; Old = 'Pipeline core'; New = 'The Pipeline core' } + @{ Metadata = 'url'; Old = 'https://example.com'; New = 'https://example.com/documentation' } + @{ Metadata = 'licenses'; Old = 'EUPL-1.2'; New = 'EUPL 1.2' } + @{ Metadata = 'developers'; Old = '51Degrees'; New = '51Degrees Mobile Experts' } + @{ Metadata = 'scm'; Old = 'pipeline-java'; New = '51degrees-java' } + @{ Metadata = 'issueManagement'; Old = 'pipeline-java/issues'; New = '51degrees-java/issues' } + ) { + $repo = New-ChangedRepo @{ 'pkg/pom.xml' = (Edit-Released $Old $New) } + Invoke-Filter $repo @('pkg/pom.xml') | Should -BeNullOrEmpty + } + + It 'leaves out a pom.xml whose only change is a comment or its layout' { + $pom = Edit-Released '4.0.0' ` + "`n 4.0.0`n" + $repo = New-ChangedRepo @{ 'pkg/pom.xml' = $pom } + Invoke-Filter $repo @('pkg/pom.xml') | Should -BeNullOrEmpty + } + + It 'keeps a pom.xml whose changed' -ForEach @( + @{ Change = 'version'; Old = '4.5.130'; New = '4.5.131' } + @{ Change = 'dependency version'; Old = '20240303'; New = '20250107' } + @{ Change = 'dependencies'; Old = ''; New = 'org.slf4jslf4j-api' } + @{ Change = 'plugin'; Old = '3.3.0'; New = '3.4.1' } + @{ Change = 'build'; Old = ''; New = 'core' } + @{ Change = 'repository url'; Old = 'https://repo1.maven.org/maven2'; New = 'https://repo.maven.apache.org/maven2' } + ) { + $repo = New-ChangedRepo @{ 'pkg/pom.xml' = (Edit-Released $Old $New) } + Invoke-Filter $repo @('pkg/pom.xml') | Should -Be @('pkg/pom.xml') + } + + It 'keeps a pom.xml whose metadata and dependencies both changed' { + $pom = (Edit-Released 'Pipeline core' 'The Pipeline core'). + Replace('20240303', '20250107') + $repo = New-ChangedRepo @{ 'pkg/pom.xml' = $pom } + Invoke-Filter $repo @('pkg/pom.xml') | Should -Be @('pkg/pom.xml') + } + + It 'keeps a pom.xml that is new since the release' { + $repo = New-ChangedRepo @{ 'pkg/sub/pom.xml' = $Released } + Invoke-Filter $repo @('pkg/sub/pom.xml') | Should -Be @('pkg/sub/pom.xml') + } + + It 'keeps a pom.xml that was removed since the release' { + $repo = New-ChangedRepo @{} -Remove @('pkg/pom.xml') + Invoke-Filter $repo @('pkg/pom.xml') | Should -Be @('pkg/pom.xml') + } + + It 'judges each file on its own' { + $repo = New-ChangedRepo @{ + 'pkg/pom.xml' = (Edit-Released 'Pipeline core' 'The Pipeline core') + 'pkg/Core.java' = 'class Core { int a; }' + } + Invoke-Filter $repo @('pkg/pom.xml', 'pkg/Core.java') | Should -Be @('pkg/Core.java') + } + + It 'gives nothing back for no files' { + $repo = New-ChangedRepo @{} + Invoke-Filter $repo @() | Should -BeNullOrEmpty + } +} diff --git a/ci/release-plan-filter.ps1 b/ci/release-plan-filter.ps1 new file mode 100644 index 000000000..7561498f1 --- /dev/null +++ b/ci/release-plan-filter.ps1 @@ -0,0 +1,70 @@ +# Called by common-ci's steps/release-plan.ps1 with the files under a +# package's folders that changed since its last tag, once its Ignore globs are +# applied. Writes out the files whose change releases the package. +# +# A pom.xml counts only when something other than the project's descriptive +# metadata changed, so a new link, description or developer does not release +# the package, while a version, dependency, plugin or build change does. +# Comments and layout do not count either. Every other file counts. +param( + [Parameter(Mandatory)][string]$RepoPath, + # The package the files belong to. Not needed here, because a pom.xml is + # judged the same way whichever package it belongs to. + $Package, + # The revision the package was last released from, normally its tag. + [Parameter(Mandatory)][string]$From, + [string[]]$Files = @() +) +$ErrorActionPreference = 'Stop' +# 'git show' fails for a file that is not in a revision, which is an answer +# here rather than an error. +$PSNativeCommandUseErrorActionPreference = $false + +# The children of that describe the project rather than build it. +$MetadataElements = @( + 'name', 'description', 'url', 'licenses', 'developers', 'scm', + 'issueManagement') + +# The pom.xml at a revision with its comments, its layout and its metadata +# taken out, or $null when the revision has no such file or it is not XML. +function Get-PomBuildText([string]$Revision, [string]$File) { + # git's message for a missing file is not an error of this script. + $ErrorActionPreference = 'Continue' + $text = & git -C $RepoPath show "${Revision}:$File" 2>$null + $found = $LASTEXITCODE -eq 0 + $global:LASTEXITCODE = 0 + if ($found -eq $false) { + return $null + } + $xml = [Xml.XmlDocument]::new() + $xml.PreserveWhitespace = $false + try { + $xml.LoadXml(($text -join "`n")) + } catch { + return $null + } + foreach ($comment in @($xml.SelectNodes('//comment()'))) { + $null = $comment.ParentNode.RemoveChild($comment) + } + $project = $xml.DocumentElement + foreach ($child in @($project.ChildNodes)) { + if ($child.NodeType -eq [Xml.XmlNodeType]::Element -and + $MetadataElements -contains $child.LocalName) { + $null = $project.RemoveChild($child) + } + } + return $project.OuterXml +} + +foreach ($file in $Files) { + if (($file -replace '\\', '/') -notmatch '(^|/)pom\.xml$') { + $file + continue + } + $before = Get-PomBuildText $From $file + $after = Get-PomBuildText 'HEAD' $file + # A pom.xml that is new, removed or unreadable on either side counts. + if ($null -eq $before -or $null -eq $after -or $before -ne $after) { + $file + } +} diff --git a/ci/release.ps1 b/ci/release.ps1 new file mode 100644 index 000000000..13bfa8501 --- /dev/null +++ b/ci/release.ps1 @@ -0,0 +1,125 @@ +# Dot-sourced by the scripts that publish. Each package in packages.json has +# its own version, tagged '/'. A publish releases a package +# when a file under its folders changed since its last tag, and every package +# that depends on it, with the patch number moved. common-ci's +# steps/release-plan.ps1 works out the plan, and the functions here build it. +# +# Kept apart from packages.ps1, which every script loads, so that building and +# testing a pull request does not need the release plan. +. "$PSScriptRoot/packages.ps1" + +# common-ci runs these scripts from its own root, with this repository checked +# out inside it. A checkout of common-ci beside this repository is found too, +# or one elsewhere can be named with COMMON_CI. +$releasePlan = @( + $env:COMMON_CI, + (Join-Path $PSScriptRoot '..' '..'), + (Join-Path $PSScriptRoot '..' '..' 'common-ci')) | + Where-Object { $_ } | + ForEach-Object { Join-Path $_ 'steps' 'release-plan.ps1' } | + Where-Object { Test-Path $_ } | + Select-Object -First 1 +if ($null -eq $releasePlan) { + throw "common-ci's steps/release-plan.ps1 was not found. Set COMMON_CI " + + "to a checkout of common-ci." +} +. $releasePlan +Remove-Variable releasePlan + +# The examples' properties that name the version of a package they use, which +# the publish sets to the version it built. +$ExampleVersionProperties = @{ + 'device-detection.version' = 'device-detection' + 'ip-intelligence.version' = 'ip-intelligence' + 'pipeline.jsonbuilder.version' = 'pipeline.jsonbuilder' +} + +# The packages a publish builds. When nothing is to be released, every package +# is built at its released version, so that the tests have packages to install +# and the nightly publish passes without publishing anything. +function Get-BuildPlan([string]$RepoPath) { + $plan = @(Get-ReleasePlan $RepoPath) + if ($plan.Count -gt 0) { + return $plan + } + $published = Get-PublishedPackages $RepoPath + foreach ($name in (Sort-ByDependency $published @($published.Name))) { + $package = $published | Where-Object { $_.Name -eq $name } + $last = @(Get-ReleasedVersions $RepoPath $name)[-1] + [pscustomobject]@{ + Name = $name + Path = $package.Path + Version = "$last" + Previous = $last + Reason = 'already released, built for the tests' + } + } +} + +# The version of every published package in a build, which is its version in +# the plan, or its last release when the plan leaves it out. +function Get-BuildVersions([string]$RepoPath, $Plan = $null) { + if ($null -eq $Plan) { + $Plan = @(Get-BuildPlan $RepoPath) + } + $versions = @{} + foreach ($package in Get-PublishedPackages $RepoPath) { + $release = $Plan | Where-Object { $_.Name -eq $package.Name } + if ($null -ne $release) { + $versions[$package.Name] = $release.Version + continue + } + $released = @(Get-ReleasedVersions $RepoPath $package.Name) + if ($released.Count -eq 0) { + throw "$($package.Name) has never been released and is not in the plan." + } + $versions[$package.Name] = "$($released[-1])" + } + return $versions +} + +# Writes into every published pom.xml its own version, its parent's and those +# of the packages it depends on, so each module is built at its own version and +# its published pom requires the versions it was built with. The poms in the +# repository keep one development version for every module, which this +# replaces only in the checkout being built. +function Set-PomVersions([string]$RepoPath, [hashtable]$Versions) { + $gaps = @(Get-PackagesJsonGaps $RepoPath) + if ($gaps.Count -gt 0) { + throw "ci/packages.json does not match the poms:`n$($gaps -join "`n")" + } + $packages = Get-PublishedPackages $RepoPath + $names = @($packages.Name) + foreach ($package in $packages) { + $pom = Read-Pom (Join-Path $RepoPath (Get-PomPath $package)) + $project = $pom.Xml.DocumentElement + $version = $project.SelectSingleNode('m:version', $pom.Ns) + if ($null -eq $version) { + # The module took its version from its parent. It gets an element + # of its own after its artifactId, indented as the artifactId is. + $artifactId = $project.SelectSingleNode('m:artifactId', $pom.Ns) + $version = $pom.Xml.CreateElement('version', $PomNamespace) + $null = $project.InsertAfter($version, $artifactId) + $indent = $artifactId.PreviousSibling + if ($null -ne $indent -and + $indent.NodeType -eq [Xml.XmlNodeType]::Whitespace) { + $null = $project.InsertAfter($indent.Clone(), $artifactId) + } + } + $version.InnerText = $Versions[$package.Name] + foreach ($reference in Get-PomReferences $pom $names) { + if ($null -ne $reference.Version) { + $reference.Version.InnerText = $Versions[$reference.Name] + } + } + Save-Pom $pom + } +} + +# The arguments that point the examples at the versions a publish built. +function Get-ExampleVersionArguments([string]$RepoPath) { + $versions = Get-BuildVersions $RepoPath + foreach ($property in $ExampleVersionProperties.Keys | Sort-Object) { + "-D$property=$($versions[$ExampleVersionProperties[$property]])" + } +} diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 1192dc342..b28890c82 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -1,8 +1,9 @@ param( [Parameter(Mandatory)][string]$RepoName, [string]$Name, - # Set by the publish workflow to the version it built and installed. Not - # set when a pull request is tested, which builds the reactor itself. + # Set by the publish workflow, which built and installed the packages, to + # its label for the run. Not set when a pull request is tested, which + # builds the reactor itself. [string]$Version, [string]$TestResourceKey, [string]$DeviceDetection, @@ -23,13 +24,15 @@ $results = "$RepoName/test-results/integration" $failed = @() # The examples take the libraries from Maven Central unless told otherwise. -# Pointing them at the version the publish workflow installed tests the -# packages as a project of your own would use them. For a pull request the -# dev profile points them at the reactor build-project.ps1 installed. -[string[]]$versionArgs = if ($Version) { - "-Ddevice-detection.version=$Version", "-Dip-intelligence.version=$Version" +# Pointing them at the versions the publish workflow built and installed, each +# package's own, tests the packages as a project of your own would use them. +# For a pull request the dev profile points them at the reactor +# build-project.ps1 installed. +if ($Version) { + . "$PSScriptRoot/release.ps1" + [string[]]$versionArgs = Get-ExampleVersionArguments $RepoName } else { - , '-Pdev' + [string[]]$versionArgs = '-Pdev' } # The Pipeline API developer examples, as the pipeline repository ran them. @@ -61,7 +64,7 @@ if ($Version) { Write-Host "::warning title=Examples skipped::Skipping '$($example.Path)', as IpIntelligenceUrl was not provided." continue } - Write-Host "::group::$($example.Path) against $Version" + Write-Host "::group::$($example.Path) against $($versionArgs -join ' ')" Push-Location "$RepoName/$($example.Path)" try { mvn clean test @MavenOptions @versionArgs ` diff --git a/ci/run-performance-tests.ps1 b/ci/run-performance-tests.ps1 index 9b24f7024..a9d57bca5 100644 --- a/ci/run-performance-tests.ps1 +++ b/ci/run-performance-tests.ps1 @@ -1,8 +1,9 @@ param( [Parameter(Mandatory)][string]$RepoName, [Parameter(Mandatory)][string]$Name, - # Set by the publish workflow to the version it built and installed. Not - # set when a pull request is tested, which builds the reactor itself. + # Set by the publish workflow, which built and installed the packages, to + # its label for the run. Not set when a pull request is tested, which + # builds the reactor itself. [string]$Version, [string]$IpIntelligenceUrl ) @@ -21,10 +22,12 @@ $summary = @{ HigherIsBetter = [ordered]@{}; LowerIsBetter = [ordered]@{} } $results = "$RepoName/test-results/performance" $failed = @() -[string[]]$versionArgs = if ($Version) { - "-Ddevice-detection.version=$Version", "-Dip-intelligence.version=$Version" +# A publish measures the versions it built, each package's own. +if ($Version) { + . "$PSScriptRoot/release.ps1" + [string[]]$versionArgs = Get-ExampleVersionArguments $RepoName } else { - , '-Pdev' + [string[]]$versionArgs = '-Pdev' } # Pipeline API. The test writes its figures as JSON itself. diff --git a/ci/update-packages.ps1 b/ci/update-packages.ps1 index adec719d1..5bc798f62 100644 --- a/ci/update-packages.ps1 +++ b/ci/update-packages.ps1 @@ -13,7 +13,7 @@ $PSNativeCommandUseErrorActionPreference = $true # pipeline.version is left alone. The libraries set it to ${project.version} # so that they build against the Pipeline API in this repository, and it must # not be replaced by the latest published version. -foreach ($package in $Packages) { +foreach ($package in $Trees) { $allowSnapshots = (-not $package.Publish).ToString().ToLower() Write-Host "Updating dependencies of '$($package.Path)'. Patch version only. Snapshots allowed: $allowSnapshots." Push-Location "$RepoName/$($package.Path)" diff --git a/pom.xml b/pom.xml index 567875602..f638fd9df 100644 --- a/pom.xml +++ b/pom.xml @@ -9,13 +9,13 @@ nothing here reaches Maven Central and the published poms are the same shape as before the packages shared a repository. - Every module carries the same version. CI sets it on all of them - before a release with versions:set and processAllModules. The + Every module carries the same development version, and the libraries reference each other through ${project.version}, so a - change in pipeline-java is seen by device-detection-java in the same - build. The examples reference the packages published to Maven Central - unless the dev profile is active, which points them at the libraries - here instead. --> + change to the Pipeline API is seen by Device Detection in the same + build. A release gives each published module its own version, which + ci/build-package.ps1 writes into the poms. The examples reference the + packages published to Maven Central unless the dev profile is + active, which points them at the libraries here instead. --> com.51degrees 51degrees-java 4.5.127-SNAPSHOT From b76596bea70e36dc4e9fb6b102e6856f0ee129d7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 10:36:13 +0000 Subject: [PATCH 10/42] Compare usage sharing keys in the root locale (#83) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [`EvidenceKeyFilterShareUsage`](https://github.com/51Degrees/51degrees-java/blob/30e4886708974308d4b70e1804424f55560824a7/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java#L166-L196) decides which Evidence the usage sharing element sends. On `rc` at 30e48867 it lower-cases each key with `key.toLowerCase()` at line 192, and the blocked headers, the query string parameters and the session cookie name at lines 170, 172 and 178, all in the default locale of the JVM. Under a Turkish or Azeri default locale a capital I lower-cases to a dotless i (U+0131), so `ID.EMAIL` becomes `ıd.emaıl`, does not match the never-shared key `id.email`, and is shared. `HEADER.COOKIE` and `ID.SALT` get through the same way, and so does a blocked header written with a capital I. --- .../data/EvidenceKeyFilterShareUsage.java | 9 ++--- .../flowelements/ShareUsageOptionsTests.java | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 3bfd42e8f..e3360af33 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import static fiftyone.pipeline.core.Constants.*; @@ -167,15 +168,15 @@ public EvidenceKeyFilterShareUsage( this.blockedHttpHeaders.add("cookie"); this.includeSession = includeSession; this.sessionCookieName = sessionCookieName == null ? - null : sessionCookieName.toLowerCase(); + null : sessionCookieName.toLowerCase(Locale.ROOT); for (String header : blockedHttpHeaders) { - String lowerHeader = header.toLowerCase(); + String lowerHeader = header.toLowerCase(Locale.ROOT); if (!this.blockedHttpHeaders.contains(lowerHeader)) { this.blockedHttpHeaders.add(lowerHeader); } } for (String parameter : includedQueryStringParams) { - String lowerParameter = parameter.toLowerCase(); + String lowerParameter = parameter.toLowerCase(Locale.ROOT); if (!this.includedQueryStringParams.contains(lowerParameter)) { this.includedQueryStringParams.add(lowerParameter); } @@ -189,7 +190,7 @@ public boolean include(String key) { if (ShareUsageEvidence.HTTP_REQUEST_KEY.equalsIgnoreCase(key)) { return false; } - String lowerKey = key.toLowerCase(); + String lowerKey = key.toLowerCase(Locale.ROOT); for (String neverShared : NEVER_SHARED_EVIDENCE_KEYS) { if (lowerKey.equals(neverShared) || lowerKey.endsWith(EVIDENCE_SEPERATOR + neverShared)) { diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java index 65a920482..3bc53524f 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java @@ -46,6 +46,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -262,6 +263,41 @@ public void keysAreReadWithoutRegardToCase() throws Exception { sent.getElementsByTagName("Sequence").item(0).getTextContent()); } + /** + * The never-shared keys are matched without regard to case under any + * default locale. Under a Turkish default locale "I" lower-cases to a + * dotless "i", so a comparison in the default locale would let + * "ID.EMAIL" through. + */ + @Test + public void neverSharedKeys_TurkishLocale_StillRefused() { + Locale original = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + try { + EvidenceKeyFilterShareUsage filter = new EvidenceKeyFilterShareUsage( + new ArrayList(), + new ArrayList(), + false, + DEFAULT_SESSION_COOKIE_NAME); + EvidenceKeyFilterShareUsage shareAll = + new EvidenceKeyFilterShareUsage( + new ArrayList(), + new ArrayList(), + false, + DEFAULT_SESSION_COOKIE_NAME, + true, + true); + for (String key : new String[] { + "ID.EMAIL", "Query.ID.Email", "HEADER.COOKIE", + "Server.Request-Path", "SERVER.REQUEST-QUERY", "ID.SALT"}) { + assertFalse(filter.include(key), key); + assertFalse(shareAll.include(key), key); + } + } finally { + Locale.setDefault(original); + } + } + @Test public void tracker_BuildsOneCache() { CountingCacheBuilder builder = new CountingCacheBuilder(); From 1c99b72f975209ab30082985f6cceeaf71d9aca1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 10:36:23 +0000 Subject: [PATCH 11/42] Point configuration page links at 51degrees.com/configure (#115) The configuration page has moved from `configure.51degrees.com` to https://51degrees.com/configure. The links and mentions that people read (READMEs, code comments, example output, and the messages that tell someone where to create a Resource Key) now use the new address, and the text that names the page calls it the configuration page. --- AGENTS.md | 8 ++++---- README.md | 2 +- device-detection/README.md | 2 +- .../devicedetection/DeviceDetectionPipelineBuilder.java | 2 +- examples/device-detection/README.md | 4 ++-- .../examples/console/GettingStartedCloud.java | 2 +- .../devicedetection/examples/console/MinimalExample.java | 2 +- .../examples/console/NativeModelCloud.java | 2 +- .../devicedetection/examples/console/TacCloud.java | 4 ++-- .../console/src/main/resources/tacCloud.xml | 2 +- .../devicedetection/examples/shared/KeyHelper.java | 6 +++--- .../fiftyone/devicedetection/examples/shared/settings.xml | 2 +- .../shared/src/test/resources/51Degrees-Cloud-Test.xml | 2 +- .../examples/web/GettingStartedWebCloud.java | 2 +- .../src/main/webapp/WEB-INF/51Degrees-Cloud.xml | 2 +- examples/ip-intelligence/README.md | 4 ++-- .../ipintelligence/examples/shared/KeyHelper.java | 6 +++--- .../fiftyone/ipintelligence/examples/shared/settings.xml | 4 ++-- .../shared/src/test/resources/51Degrees-Cloud-Test.xml | 2 +- ip-intelligence/README.md | 2 +- .../ipintelligence/IPIntelligencePipelineBuilder.java | 2 +- location/README.md | 2 +- .../fiftyone/geolocation/examples/CombiningServices.java | 4 ++-- .../geolocation/examples/cloud/GettingStarted.java | 4 ++-- .../src/main/webapp/WEB-INF/51Degrees-Cloud.xml | 2 +- .../src/main/webapp/WEB-INF/51Degrees-Cloud.xml | 2 +- .../fiftyone/geolocation/GeoLocationPipelineBuilder.java | 4 ++-- .../fiftyone/pipeline/cloudrequestengine/Constants.java | 2 +- .../flowelements/CloudRequestEngineLiveTests.java | 2 +- .../cloudrequestengine/flowelements/LiveCloudKeys.java | 2 +- .../pipeline.core/src/test/resources/options-test.xml | 2 +- .../developerexamples/cloudengine/ExampleTests.java | 2 +- .../test/java/fiftyone/pipeline/did/LiveCloudKeys.java | 4 ++-- 33 files changed, 48 insertions(+), 48 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 25a332221..df51ec15e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -170,10 +170,10 @@ describes. ### Cloud examples -The cloud examples need a resource key, which a person creates in the -configurator, so ask for one if none is set. A key with the properties most of -the Device Detection examples use can be made -[here](https://configure.51degrees.com/jqz435Nc?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=cloud-examples). +The cloud examples need a resource key, which a person creates on the +configuration page, so ask for one if none is set. A key with the properties +most of the Device Detection examples use can be made +[here](https://51degrees.com/configure/jqz435Nc?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=cloud-examples). The Device Detection cloud example takes the key as its first argument, or from a `TestResourceKey` environment variable or system property. diff --git a/README.md b/README.md index b5952683a..75d69badb 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Each product is a separate Maven Central package. For Device Detection, cloud or The product guides below list the other packages. To build from source, see [Getting the code](#getting-the-code). -Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). +Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). Created from `pipeline-java`, `device-detection-java`, `ip-intelligence-java`, `location-java`, `device-detection-java-examples`, and `ip-intelligence-java-examples`, this repository continues their work. diff --git a/device-detection/README.md b/device-detection/README.md index f1e0a9e9e..713872bc5 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -22,7 +22,7 @@ The Java API can either use our cloud service to get its data or it can use a lo You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) on +our [configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) on how to use this. #### On-Premise diff --git a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionPipelineBuilder.java b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionPipelineBuilder.java index 8985dfd27..6e025c28b 100644 --- a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionPipelineBuilder.java +++ b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionPipelineBuilder.java @@ -145,7 +145,7 @@ public DeviceDetectionOnPremisePipelineBuilder useOnPremise( /** * Use the 51Degrees Cloud service to perform device detection. * @param resourceKey The resource key to use when querying the cloud service. - * Obtain one from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=device-detection-src-main-java-fiftyone-devicedetection-devicedetectionpipelinebuilder.java&utm_term=usecloud + * Obtain one from https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=device-detection-src-main-java-fiftyone-devicedetection-devicedetectionpipelinebuilder.java&utm_term=usecloud * @return A builder that can be used to configure and build a pipeline * that will use the cloud device detection engine. */ diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index c83d69956..5e1f575ba 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -40,11 +40,11 @@ argument to use it. You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) on +our [configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) on how to use this. A resource key configured with the properties needed -to run most of the examples can be obtained [here](https://configure.51degrees.com/jqz435Nc?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys). +to run most of the examples can be obtained [here](https://51degrees.com/configure/jqz435Nc?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys). To use the resource key in the example it can be supplied as a command line parameter, pasted into the configuration file (where there is one) or supplied as either an environment variable or a system diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java index 846e1fcb4..48e1efd9c 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java @@ -47,7 +47,7 @@ *

* In order to use the cloud service you will need to obtain a "Resource Key". A free resource key * configured with the properties required by this example may be obtained from - * https://configure.51degrees.com/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java&utm_term=gettingstartedcloud-2 + * https://51degrees.com/configure/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java&utm_term=gettingstartedcloud-2 *

* The concepts of "pipeline", "flow data", "evidence" and "results" are illustrated. */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java index 8c5fb5206..2a9291dc8 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java @@ -28,7 +28,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; /** - * This is the code for the minimal example displayed in the configurator. Please see + * This is the code for the minimal example displayed on the configuration page. Please see * * Getting Started Console Cloud for a fuller example. */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java index 3c6c724ac..2916f169b 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java @@ -131,7 +131,7 @@ static void analyseNativeModel(String nativemodel, Pipeline pipeline, PrintWrite * * Unlike other examples, use of this example requires a license key which can be purchased from [our * pricing page](https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). Once this is done, a resource key with the - * properties required by this example can be created at [here](https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). + * properties required by this example can be created at [here](https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java). */ \ No newline at end of file diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java index ee5ee1e0b..1896616b9 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java @@ -49,7 +49,7 @@ * Unlike other examples, use of this example requires a license key which can be purchased from our * pricing page. Once this is done, a resource key with the * properties required by this example can be created - * here. + * here. *

* This resource key can be supplied as a command line argument to this program, as an environment * variable "SuperResourceKey", as a System Property "SuperResourceKey" or by editing the config @@ -136,7 +136,7 @@ static void analyseTac(String tac, Pipeline pipeline, PrintWriter output) throws * * Unlike other examples, use of this example requires a license key which can be purchased from our * [pricing page](https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). Once this is done, a resource key with the - * properties required by this example can be created [here](https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). + * properties required by this example can be created [here](https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java). * diff --git a/examples/device-detection/console/src/main/resources/tacCloud.xml b/examples/device-detection/console/src/main/resources/tacCloud.xml index e94963325..aed9c9717 100644 --- a/examples/device-detection/console/src/main/resources/tacCloud.xml +++ b/examples/device-detection/console/src/main/resources/tacCloud.xml @@ -30,7 +30,7 @@ from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-resources-taccloud.xml&utm_term=buildparameters. Once this is done, a resource key with the properties required by this example - can be created at https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-resources-taccloud.xml&utm_term=buildparameters. + can be created at https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-resources-taccloud.xml&utm_term=buildparameters. You can paste the resource key here, or set an environment variable or System property called "SuperResourceKey" diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java index 467a443be..06ae82900 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java @@ -43,7 +43,7 @@ public class KeyHelper { "_51DEGREES_RESOURCE_KEY_BESPOKE")); static final String CREATE_KEY_URL = - "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=no-resource-key"; + "https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=no-resource-key"; /** * The resource key for a test that calls the live cloud service, from @@ -163,7 +163,7 @@ public static String getOrSetTestResourceKey(String value, boolean shouldThrow) return getOrSetResourceKey(value, TEST_RESOURCE_KEY, "A free resource key configured with the " + "properties required by this example may be obtained from " + - "https://configure.51degrees.com/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=resource-key-required ", + "https://51degrees.com/configure/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=resource-key-required ", shouldThrow); } public static String getOrSetTestResourceKey(String value) { @@ -210,7 +210,7 @@ public static String getOrSetSuperResourceKey(String value, String variablename, "that you will first need a license key, which can be purchased " + "from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=license-key-required. \nOnce this is " + "done, a resource key with the properties required by this example " + - "can be created at https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=license-key-required. ", + "can be created at https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=license-key-required. ", shouldThrow); } } diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/settings.xml b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/settings.xml index cbbbbc9af..96c2db41c 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/settings.xml +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/settings.xml @@ -45,7 +45,7 @@ your IDE launch configuration or your operating system shell. + https://51degrees.com/configure/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-settings.xml&utm_term=properties --> diff --git a/examples/device-detection/shared/src/test/resources/51Degrees-Cloud-Test.xml b/examples/device-detection/shared/src/test/resources/51Degrees-Cloud-Test.xml index 335efff11..d02f4f902 100644 --- a/examples/device-detection/shared/src/test/resources/51Degrees-Cloud-Test.xml +++ b/examples/device-detection/shared/src/test/resources/51Degrees-Cloud-Test.xml @@ -27,7 +27,7 @@ http://cloud.51degrees.com/api/v4 + created at https://51degrees.com/configure/hYzn3TV3?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-settings.xml&utm_term=properties. --> diff --git a/examples/ip-intelligence/shared/src/test/resources/51Degrees-Cloud-Test.xml b/examples/ip-intelligence/shared/src/test/resources/51Degrees-Cloud-Test.xml index 3af0fba3c..4e16e3e9e 100644 --- a/examples/ip-intelligence/shared/src/test/resources/51Degrees-Cloud-Test.xml +++ b/examples/ip-intelligence/shared/src/test/resources/51Degrees-Cloud-Test.xml @@ -27,7 +27,7 @@ http://cloud.51degrees.com/api/v4 diff --git a/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp/WEB-INF/51Degrees-Cloud.xml b/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp/WEB-INF/51Degrees-Cloud.xml index 23e052eee..5a3b9582f 100644 --- a/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp/WEB-INF/51Degrees-Cloud.xml +++ b/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp/WEB-INF/51Degrees-Cloud.xml @@ -27,7 +27,7 @@ https://cloud.51degrees.com/api/v4 diff --git a/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationPipelineBuilder.java b/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationPipelineBuilder.java index 464d12a4d..ecb3d61a2 100644 --- a/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationPipelineBuilder.java +++ b/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationPipelineBuilder.java @@ -63,7 +63,7 @@ public GeoLocationPipelineBuilder( * Use the 51Degrees cloud service to perform Geo Location. * @param resourceKey the resource key to use when querying the * service. - * Obtain one from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=geo-location-src-main-java-fiftyone-geolocation-geolocationpipelinebuilder.java&utm_term=usecloud + * Obtain one from https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=geo-location-src-main-java-fiftyone-geolocation-geolocationpipelinebuilder.java&utm_term=usecloud * @param provider the Geo Location provider to use * @return builder that can be used to configure and build a pipeline * that will use the cloud Geo Location engine @@ -79,7 +79,7 @@ public GeoLocationCloudPipelineBuilder useCloud( /** * Use the 51Degrees cloud service to perform Geo Location * @param resourceKey the resource to use when querying the cloud service. - * Obtain one from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=geo-location-src-main-java-fiftyone-geolocation-geolocationpipelinebuilder.java&utm_term=usecloud-2 + * Obtain one from https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=geo-location-src-main-java-fiftyone-geolocation-geolocationpipelinebuilder.java&utm_term=usecloud-2 * @param endpoint the 51Degrees cloud URL * @param provider the Geo Location provider to use * @return a builder that can be used to configure and build a pipeline that diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index eebb82eb4..2c72e203d 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -32,7 +32,7 @@ public class Messages { + "to authenticate with, as the cloud service answers 401 to a " + "request that carries neither. Use setResourceKey(String) with " + "a resource key created for free at " - + "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=no-credential"; + + "https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=no-credential"; public static final String LicenseKeyNeedsProperties = "The Cloud Request Engine has a license key and no requested " diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java index 8f8487633..20b2e4d87 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java @@ -98,7 +98,7 @@ public void build_Process_ReadsDeviceDataFromTheLiveService() "The live cloud service listed no products for this resource " + "key. A key carries the products its account holds, so this " + "is an entitlement problem and not a fault in the service. " - + "Create a key at https://configure.51degrees.com."); + + "Create a key at https://51degrees.com/configure."); assertTrue(products.containsKey("device"), "The live cloud service listed the products " + String.join(", ", products.keySet()) + " for this resource " diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java index 6bb87c5fe..20e39841b 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java @@ -88,6 +88,6 @@ static String noResourceKey() { + "as an environment variable or as a system property, tried in " + "that order. The key has to carry the device product, because " + "the test asks the service for device detection properties. " - + "Create a key at https://configure.51degrees.com."; + + "Create a key at https://51degrees.com/configure."; } } diff --git a/pipeline/pipeline.core/src/test/resources/options-test.xml b/pipeline/pipeline.core/src/test/resources/options-test.xml index 335efff11..d02f4f902 100644 --- a/pipeline/pipeline.core/src/test/resources/options-test.xml +++ b/pipeline/pipeline.core/src/test/resources/options-test.xml @@ -27,7 +27,7 @@ http://cloud.51degrees.com/api/v4 - ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-LiteV41.ipi} LowMemory IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/html/index.html b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/html/index.html index 2abdface2..afb91cab8 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/html/index.html +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/html/index.html @@ -92,6 +92,7 @@

Location map

${CONTACT_MESSAGE} + ${FIND_OUT_MORE} diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java index e1831b39b..f995a4ea6 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; @@ -96,6 +97,9 @@ public void testWebOnPrem() throws Exception { // Assert we got a successful response assertEquals("Expected HTTP 200 OK response", 200, code); assertTrue("Response should not be empty", responseBody.length() > 0); + assertTrue("The page should end with the '" + FindOutMore.HEADING + + "' section", responseBody.contains( + "

" + FindOutMore.HEADING + "

")); } @AfterClass diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java new file mode 100644 index 000000000..047f34bdc --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java @@ -0,0 +1,61 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Checks the options file this web example runs with. + */ +public class OptionsFileTest { + + /** + * With no data file set, the example falls back to the free Lite file, + * which a fresh clone downloads, and not to the Enterprise file, which a + * fresh clone never has. + */ + @Test + public void defaultDataFile_FreeLiteFile() throws IOException { + String xml = new String(Files.readAllBytes(Paths.get( + System.getProperty("basedir", "."), "src", "main", "webapp", + "WEB-INF", "51Degrees-OnPrem.xml")), StandardCharsets.UTF_8); + Matcher dataFile = Pattern.compile( + "\\$\\{TestDataFile:-([^}]*)\\}") + .matcher(xml); + assertTrue("51Degrees-OnPrem.xml should give a default IP Intelligence " + + "data file", dataFile.find()); + assertEquals("cxx/data/" + DataFileHelper.LITE_DATA_FILE_REL_PATH, + dataFile.group(1)); + } +} From fcde776b0647d57986c05972335797ef756706db Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 02:32:34 +0000 Subject: [PATCH 19/42] End every Pipeline developer and Location console example with a Find out more section (#134) No Pipeline developer example and no Location example ended with a "Find out more" section, found in the audit of the examples on 13 November. The 51Degrees rule for examples is that every example ends with a clearly separated "Find out more" section that gives the person running it links to follow up, being the relevant repositories as plain addresses and 51degrees.com pages tagged with the campaign convention that common-ci's [`scripts/utm-lint.ps1`](https://github.com/51Degrees/common-ci/blob/1c5c0e0b09a1f77722e20ba2d10cbd1031caca9e/scripts/utm-lint.ps1) checks. --- .../examples/CombiningServices.java | 4 + .../geolocation/examples/FindOutMore.java | 76 +++++++++++ .../examples/cloud/GettingStarted.java | 5 + .../examples/FindOutMoreTests.java | 120 ++++++++++++++++++ .../developerexamples/cloudengine/Main.java | 25 ++++ .../cloudengine/FindOutMoreTests.java | 104 +++++++++++++++ .../developerexamples/flowelement/Main.java | 25 ++++ .../flowelement/FindOutMoreTests.java | 98 ++++++++++++++ .../developerexamples/fodid/Main.java | 25 ++++ .../fodid/FindOutMoreTests.java | 98 ++++++++++++++ .../onpremiseengine/Main.java | 25 ++++ .../onpremiseengine/FindOutMoreTests.java | 98 ++++++++++++++ .../developerexamples/usagesharing/Main.java | 25 ++++ .../usagesharing/FindOutMoreTests.java | 100 +++++++++++++++ 14 files changed, 828 insertions(+) create mode 100644 location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/FindOutMore.java create mode 100644 location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/FindOutMoreTests.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/FindOutMoreTests.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/FindOutMoreTests.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/FindOutMoreTests.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/FindOutMoreTests.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/test/java/pipeline/developerexamples/usagesharing/FindOutMoreTests.java diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java index 09d7de244..05135a033 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java @@ -179,6 +179,10 @@ public static void main(String[] args) throws Exception { System.out.println("IsMobile: " + isMobile.toString()); } } + FindOutMore.write( + System.out, + "https://51degrees.com/documentation/_reverse_geocoding__examples__combining_services.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-combiningservices.java&utm_term=find-out-more", + "https://51degrees.com/location?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-combiningservices.java&utm_term=find-out-more"); } private static void outputUntilCancelled( diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/FindOutMore.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/FindOutMore.java new file mode 100644 index 000000000..c4d878857 --- /dev/null +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/FindOutMore.java @@ -0,0 +1,76 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.examples; + +import java.io.PrintStream; + +/** + * Writes the "Find out more" section that ends each Location example's + * output, set apart by a rule above and below. It gives the example's page + * and the Location page on 51degrees.com, each tagged with the example, then + * the repository. + */ +public final class FindOutMore { + + /** + * The heading the section starts with. + */ + public static final String HEADING = "Find out more"; + + /** + * The rule above and below the section. + */ + public static final String RULE = + "----------------------------------------------------------------"; + + /** + * The repository the examples come from. + */ + public static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private FindOutMore() { + } + + /** + * Writes the section. + * @param output where to write, flushed afterwards + * @param examplePage the address of the example's page, with its + * campaign tags + * @param locationPage the address of the Location page, with its + * campaign tags + */ + public static void write( + PrintStream output, + String examplePage, + String locationPage) { + output.println(); + output.println(RULE); + output.println(HEADING); + output.println("\tThis example: " + examplePage); + output.println("\tLocation: " + locationPage); + output.println("\t" + REPOSITORY); + output.println(RULE); + output.flush(); + } +} diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java index ec935ee32..8585c8960 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java @@ -25,6 +25,7 @@ import fiftyone.geolocation.GeoLocationPipelineBuilder; import fiftyone.geolocation.core.Enums; import fiftyone.geolocation.core.data.GeoData; +import fiftyone.geolocation.examples.FindOutMore; import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -117,6 +118,10 @@ public static void main(String[] args) throws Exception { System.out.println(flowData.get(GeoData.class).getRegion()); } } + FindOutMore.write( + System.out, + "https://51degrees.com/documentation/_reverse_geocoding__examples__getting_started_51_degrees_cloud.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-gettingstarted.java&utm_term=find-out-more", + "https://51degrees.com/location?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-gettingstarted.java&utm_term=find-out-more"); } private static void outputUntilCancelled( diff --git a/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/FindOutMoreTests.java b/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/FindOutMoreTests.java new file mode 100644 index 000000000..68b751cee --- /dev/null +++ b/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/FindOutMoreTests.java @@ -0,0 +1,120 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.examples; + +import fiftyone.geolocation.examples.cloud.GettingStarted; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Each Location example ends its output with a "Find out more" section, + * which lists pages on 51degrees.com, each with its campaign tags, and the + * repository. With no resource key set, each example says where to create + * one and stops, so these tests need no key. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final String NO_KEY = "You need to create a resource key"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + /** + * Runs a Location example as a person would. + */ + private interface Example { + void run() throws Exception; + } + + @Test + public void GettingStarted_EndsWithFindOutMore() throws Exception { + assertFindOutMore(run(() -> GettingStarted.main(new String[0]))); + } + + @Test + public void CombiningServices_EndsWithFindOutMore() throws Exception { + assertFindOutMore(run(() -> CombiningServices.main(new String[0]))); + } + + private static void assertFindOutMore(String output) { + int example = output.lastIndexOf(NO_KEY); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue("The example wrote nothing:\n" + output, example >= 0); + assertTrue( + "No Find out more section after the example's output:\n" + + output, + heading > example); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue("The section has no closing rule:\n" + output, end > 0); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue( + "A 51degrees.com link without its campaign tags: " + + line, + TAGGED.matcher(line).find()); + pages++; + } + } + assertEquals("The pages listed:\n" + section, 2, pages); + assertTrue("The repository is not listed:\n" + section, + section.contains(REPOSITORY)); + } + + /** + * Runs the example and returns what it wrote, with each line ending in + * a line feed. + */ + private static String run(Example example) throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + example.run(); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java index 963827bdb..b94b6b649 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java @@ -34,6 +34,7 @@ import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngine; import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngineBuilder; +import java.io.PrintStream; import java.util.concurrent.atomic.AtomicBoolean; public class Main { @@ -233,7 +234,31 @@ public void run() throws Exception { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThis example: https://51degrees.com/documentation/_pipeline_api__examples__custom_element__cloud.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.cloud-engine-main.java&utm_term=find-out-more"); + output.println("\tCustom flow elements: https://51degrees.com/documentation/_pipeline_api__features__custom_elements.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.cloud-engine-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/FindOutMoreTests.java new file mode 100644 index 000000000..71df66f69 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/FindOutMoreTests.java @@ -0,0 +1,104 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.cloudengine; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + if (Main.configuredResourceKey() == null) { + fail("This test runs the example against the live cloud " + + "service, so it needs a resource key in one of " + + String.join(", ", Main.RESOURCE_KEY_NAMES) + "."); + } + String output = runMain(); + int example = output.lastIndexOf("star sign"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + assertTrue(section.contains(REPOSITORY), + "The repository is not listed:\n" + section); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java index 76d7e4473..2fe22edcf 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java @@ -30,6 +30,7 @@ import pipeline.developerexamples.flowelement.flowelements.SimpleFlowElement; import pipeline.developerexamples.flowelement.flowelements.SimpleFlowElementBuilder; +import java.io.PrintStream; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -69,7 +70,31 @@ public void run() throws Exception { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThis example: https://51degrees.com/documentation/_pipeline_api__examples__custom_element__flow_element.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.flowelement-main.java&utm_term=find-out-more"); + output.println("\tCustom flow elements: https://51degrees.com/documentation/_pipeline_api__features__custom_elements.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.flowelement-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/FindOutMoreTests.java new file mode 100644 index 000000000..b180ac756 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/FindOutMoreTests.java @@ -0,0 +1,98 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.flowelement; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("your star sign is"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + assertTrue(section.contains(REPOSITORY), + "The repository is not listed:\n" + section); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/Main.java index d21426abc..26d0f0b61 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/Main.java @@ -26,6 +26,7 @@ import com.swancommunity.owid.Crypto; import fiftyone.pipeline.did.FodId; +import java.io.PrintStream; import java.util.Arrays; /** @@ -140,7 +141,31 @@ private static String toHex(byte[] bytes) { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\t51Did: https://51degrees.com/documentation/_identifiers__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.fodid-main.java&utm_term=find-out-more"); + output.println("\tAsk about 51Did: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.fodid-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/FindOutMoreTests.java new file mode 100644 index 000000000..e34a4bd11 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/FindOutMoreTests.java @@ -0,0 +1,98 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.fodid; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("Same match key"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + assertTrue(section.contains(REPOSITORY), + "The repository is not listed:\n" + section); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java index 8d803d244..0d585136c 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java @@ -30,6 +30,7 @@ import pipeline.developerexamples.onpremiseengine.flowelements.SimpleOnPremiseEngine; import pipeline.developerexamples.onpremiseengine.flowelements.SimpleOnPremiseEngineBuilder; +import java.io.PrintStream; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -72,7 +73,31 @@ public void run() throws Exception { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThis example: https://51degrees.com/documentation/_pipeline_api__examples__custom_element__on_premise.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.onpremise-engine-main.java&utm_term=find-out-more"); + output.println("\tCustom flow elements: https://51degrees.com/documentation/_pipeline_api__features__custom_elements.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.onpremise-engine-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/FindOutMoreTests.java new file mode 100644 index 000000000..4acc41328 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/FindOutMoreTests.java @@ -0,0 +1,98 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.onpremiseengine; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("your star sign is"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + assertTrue(section.contains(REPOSITORY), + "The repository is not listed:\n" + section); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java index f6da01434..cb4ca6e86 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.PrintStream; import java.util.Objects; /** @@ -115,7 +116,31 @@ public void run() throws Exception { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThis example: https://51degrees.com/documentation/_pipeline_api__examples__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.usage-sharing-main.java&utm_term=find-out-more"); + output.println("\tUsage sharing: https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.usage-sharing-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/test/java/pipeline/developerexamples/usagesharing/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/test/java/pipeline/developerexamples/usagesharing/FindOutMoreTests.java new file mode 100644 index 000000000..9155c5097 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/test/java/pipeline/developerexamples/usagesharing/FindOutMoreTests.java @@ -0,0 +1,100 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.usagesharing; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("Pipeline created with share usage element"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue("The example wrote nothing:\n" + output, example >= 0); + assertTrue( + "No Find out more section after the example's output:\n" + + output, + heading > example); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue("The section has no closing rule:\n" + output, end > 0); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue( + "A 51degrees.com link without its campaign tags: " + + line, + TAGGED.matcher(line).find()); + pages++; + } + } + assertEquals("The pages listed:\n" + section, 2, pages); + assertTrue("The repository is not listed:\n" + section, + section.contains(REPOSITORY)); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} From de74c8c881d635b7e1db6f5120d24393829c8892 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 02:32:43 +0000 Subject: [PATCH 20/42] Supply server.request-query in the web integration (#144) The web integration supplies the method and the path of the request line as Evidence, and not the query string. [web-integration.md lines 95 to 114 at 706de1f0](https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/features/web-integration.md?plain=1#L95-L114) list "Request query string, whole and undecoded. Key is `server.request-query`" beside the method and the path, and say: --- .../WebRequestEvidenceServiceCore.java | 13 ++- .../web/RequestLineEvidenceTests.java | 110 ++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/RequestLineEvidenceTests.java diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java index 03f853e17..c9f69c96e 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java @@ -65,6 +65,9 @@ class Default implements WebRequestEvidenceServiceCore { private static final String EVIDENCE_REQUEST_PATH = EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-path"; + private static final String EVIDENCE_REQUEST_QUERY = + EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-query"; + @Override public void addEvidenceFromRequest( FlowData flowData, @@ -127,12 +130,20 @@ public void addEvidenceFromRequest( checkAndAdd(flowData, EVIDENCE_PROTOCOL, request.getScheme()); - // The method and the path of the request line, as sent. + // The method, the path and the query string of the request line, + // as sent. The query string is empty when the request has none, + // so an absent key means only that the request line is not + // supplied. checkAndAdd(flowData, EVIDENCE_REQUEST_METHOD, request.getMethod()); checkAndAdd( flowData, EVIDENCE_REQUEST_PATH, request.getRequestURI()); + String query = request.getQueryString(); + checkAndAdd( + flowData, + EVIDENCE_REQUEST_QUERY, + query == null ? "" : query); } /** diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/RequestLineEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/RequestLineEvidenceTests.java new file mode 100644 index 000000000..dec55d834 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/RequestLineEvidenceTests.java @@ -0,0 +1,110 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.Evidence; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.web.services.WebRequestEvidenceServiceCore; +import org.junit.Test; + +import javax.servlet.http.HttpServletRequest; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The request line Evidence of the web integration specification's + * Populating Evidence section. An integration that supplies these keys + * supplies all three on every request, with server.request-query empty + * when the request has no query string, and each exactly as it appeared on + * the request line. + */ +public class RequestLineEvidenceTests { + + private final WebRequestEvidenceServiceCore evidenceService = + new WebRequestEvidenceServiceCore.Default(); + + /** + * Flow Data whose Pipeline has an element that accepts the three request + * line keys. + */ + private static FlowData flowData() { + FlowData flowData = mock(FlowData.class); + Evidence evidence = mock(Evidence.class); + when(evidence.asKeyMap()).thenReturn(new HashMap()); + when(flowData.getEvidence()).thenReturn(evidence); + when(flowData.getEvidenceKeyFilter()).thenReturn( + new EvidenceKeyFilterWhitelist(Arrays.asList( + "server.request-method", + "server.request-path", + "server.request-query"))); + return flowData; + } + + private static HttpServletRequest request(String path, String query) { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn(path); + when(request.getQueryString()).thenReturn(query); + when(request.getHeaderNames()) + .thenReturn(Collections.emptyEnumeration()); + when(request.getParameterNames()) + .thenReturn(Collections.emptyEnumeration()); + return request; + } + + /** + * The query string is supplied whole and undecoded, beside the method + * and the path. + */ + @Test + public void RequestQuery_SuppliedWholeAndUndecoded() { + FlowData flowData = flowData(); + + evidenceService.addEvidenceFromRequest( + flowData, request("/a", "x=%2F&y=1")); + + verify(flowData).addEvidence("server.request-method", "GET"); + verify(flowData).addEvidence("server.request-path", "/a"); + verify(flowData).addEvidence("server.request-query", "x=%2F&y=1"); + } + + /** + * A request with no query string still gets server.request-query, empty, + * so an absent key only ever means the request line is not supplied. + */ + @Test + public void RequestQuery_NoQueryString_SuppliedEmpty() { + FlowData flowData = flowData(); + + evidenceService.addEvidenceFromRequest(flowData, request("/a", null)); + + verify(flowData).addEvidence("server.request-path", "/a"); + verify(flowData).addEvidence("server.request-query", ""); + } +} From 2eebbff31cd42d366e2b6f40a70067d0761f9acb Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 04:02:27 +0000 Subject: [PATCH 21/42] Write a no-value Property as null in the JSON, and refuse a bad protocol in the JavaScript Builder's constructor (#145) Two places where the JSON Builder and the JavaScript Builder differ from the specification, found by a conformance check of this repository against [specifications at 706de1f0](https://github.com/51Degrees/specifications/tree/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc). --- .../JavaScriptBuilderElement.java | 18 +++++- .../JavaScriptBuilderElementBuilderTests.java | 46 ++++++++++++++ .../flowelements/JsonBuilderElement.java | 37 ++++++++++- .../jsonbuilder/JsonBuilderTests.java | 61 +++++++++++++++++++ 4 files changed, 156 insertions(+), 6 deletions(-) diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index 509fc1b0b..30e7ffbf2 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -153,8 +153,11 @@ public static boolean isValidObjectName(String name) { * @param host The host that the client JavaScript should query for updates. * If null or blank then the host from the request will be used * @param protocol The protocol (HTTP or HTTPS) that the client JavaScript - * will use when querying for updates. If null or blank + * will use when querying for updates. If null or empty * then the protocol from the request will be used + * @throws PipelineConfigurationException if protocol is not null or empty + * and is neither http nor https in any case of its ASCII letters, or if + * objName is not a valid JavaScript identifier */ public JavaScriptBuilderElement( Logger logger, @@ -182,11 +185,13 @@ public JavaScriptBuilderElement( * @param host The host that the client JavaScript should query for updates. * If null or blank then the host from the request will be used * @param protocol The protocol (HTTP or HTTPS) that the client JavaScript - * will use when querying for updates. If null or blank + * will use when querying for updates. If null or empty * then the protocol from the request will be used * @param contextRoot The <context-root> setting from the web.xml. * This is needed when creating the callback URL. - * @throws PipelineConfigurationException if objName is not null and is + * @throws PipelineConfigurationException if protocol is not null or empty + * and is neither http nor https in any case of its ASCII letters, white + * space included, or if objName is not null and is * not a valid JavaScript identifier, see {@link #isValidObjectName}. * A null name means the default name is used, as no name was * configured, and an empty name is refused @@ -209,6 +214,13 @@ public JavaScriptBuilderElement( this.host = host; this.endpoint = endpoint; + if (protocol != null && + protocol.isEmpty() == false && + CallbackUrl.isScheme(protocol) == false) { + throw new PipelineConfigurationException( + "Invalid protocol in configuration (" + protocol + + "), must be 'http' or 'https'"); + } this.protocol = protocol; if (objName == null) { this.objName = Constants.DEFAULT_OBJECT_NAME; diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderElementBuilderTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderElementBuilderTests.java index 55d0daaa1..8825c30ad 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderElementBuilderTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderElementBuilderTests.java @@ -24,12 +24,15 @@ import fiftyone.common.testhelpers.TestLoggerFactory; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -108,4 +111,47 @@ public void JavaScriptBuilderElement_Builder_SetDefaultProtocol_ValidProtocol( loggerFactory.loggers.forEach(testLogger -> assertTrue(testLogger.warningsLogged.size() == 0)); } + + /** + * A protocol given to the element's constructor that is neither http nor + * https, including one made only of white space, is refused when the + * element is made, as one given to the builder is. + * @param protocol + */ + @ParameterizedTest + @ValueSource(strings = {"htp", "ftp", " ", "\t", "https:"}) + public void JavaScriptBuilderElement_Constructor_InvalidProtocol_Refused( + String protocol) { + assertThrows( + PipelineConfigurationException.class, + () -> new JavaScriptBuilderElement( + loggerFactory.getLogger("test"), + null, + "", + null, + false, + "", + protocol)); + } + + /** + * A null or empty protocol means none is configured, so the protocol + * comes from Evidence, and http or https in any case of its ASCII + * letters is accepted. + * @param protocol + */ + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {"http", "https", "HTTPS", "Http"}) + public void JavaScriptBuilderElement_Constructor_ValidOrNoProtocol_Accepted( + String protocol) { + assertDoesNotThrow(() -> new JavaScriptBuilderElement( + loggerFactory.getLogger("test"), + null, + "", + null, + false, + "", + protocol)); + } } diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java index 413113d6e..b4c1b93a8 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java @@ -47,6 +47,7 @@ import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -468,7 +469,7 @@ private String buildJson(Map allProperties) { for (Map.Entry entry : allProperties.entrySet()) { if (entry.getValue() instanceof Map) { - Map map = new HashMap<>(); + JSONObject element = new JSONObject(); @SuppressWarnings("unchecked") Map properties = (Map)entry.getValue(); @@ -478,9 +479,9 @@ private String buildJson(Map allProperties) { if(value instanceof JavaScript) value = value.toString(); - map.put(resolveName(ent.getKey()), value); + element.put(resolveName(ent.getKey()), toJsonValue(value)); } - json.put(resolveName(entry.getKey()), map); + json.put(resolveName(entry.getKey()), element); } else { json.put(resolveName(entry.getKey()), entry.getValue()); @@ -489,6 +490,36 @@ private String buildJson(Map allProperties) { return json.toString(2); } + /** + * The value as it is written into the JSON. org.json leaves out a Map + * entry whose value is null, so a Map, such as an item of a list of + * Element Data, is written here with each null as JSON null, because a + * Property assigned no value is written as null with its nullreason. + * Every other value is wrapped as org.json wraps it. + */ + private static Object toJsonValue(Object value) { + if (value == null) { + return JSONObject.NULL; + } + if (value instanceof Map) { + JSONObject object = new JSONObject(); + for (Map.Entry entry : ((Map) value).entrySet()) { + object.put( + String.valueOf(entry.getKey()), + toJsonValue(entry.getValue())); + } + return object; + } + if (value instanceof Collection) { + JSONArray array = new JSONArray(); + for (Object item : (Collection) value) { + array.put(toJsonValue(item)); + } + return array; + } + return JSONObject.wrap(value); + } + /** * Executed on first request in order to build some collections diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java index b81cf6c81..f78c871d4 100644 --- a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java @@ -627,6 +627,67 @@ public void JsonBuilder_WeightedValues() throws Exception { } } + /** + * A Property assigned no value is written as null with its nullreason, + * and a Property whose value is null without being assigned no value has + * no entry and no nullreason. See the Process section of the JSON + * Builder specification. + */ + @Test + public void JsonBuilder_NoValue_WrittenAsNullWithReason() + throws Exception { + AspectPropertyValueDefault noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage("There is no value."); + Map map = new HashMap<>(); + map.put("novalue", noValue); + map.put("plainnull", null); + map.put("property", "thisIsAValue"); + when(elementDataMock.asKeyMap()).thenReturn(map); + + String json = testIteration(1); + + JSONObject element = new JSONObject(json).getJSONObject("test"); + assertTrue(element.has("novalue"), json); + assertTrue(element.isNull("novalue"), json); + assertEquals("There is no value.", + element.getString("novaluenullreason"), json); + assertFalse(element.has("plainnull"), json); + assertFalse(element.has("plainnullnullreason"), json); + assertEquals("thisIsAValue", element.getString("property"), json); + } + + /** + * Each item of a list of Element Data follows the same rules, so an + * item's Property assigned no value is written as null with its + * nullreason. + */ + @Test + public void JsonBuilder_ListOfElementData_NoValueWrittenAsNull() + throws Exception { + AspectPropertyValueDefault noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage("No value for this item."); + Map itemValues = new HashMap<>(); + itemValues.put("value1", noValue); + itemValues.put("value2", "xyz"); + ElementData item = mock(ElementData.class); + when(item.asKeyMap()).thenReturn(itemValues); + Map map = new HashMap<>(); + map.put("items", Collections.singletonList(item)); + when(elementDataMock.asKeyMap()).thenReturn(map); + + String json = testIteration(1); + + JSONObject first = new JSONObject(json).getJSONObject("test") + .getJSONArray("items").getJSONObject(0); + assertTrue(first.has("value1"), json); + assertTrue(first.isNull("value1"), json); + assertEquals("No value for this item.", + first.getString("value1nullreason"), json); + assertEquals("xyz", first.getString("value2"), json); + } + private String testIteration(int iteration) throws Exception { return testIteration(iteration, null); } From 67a4e56698790b4bf11961339e195df58d81233c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 04:32:43 +0000 Subject: [PATCH 22/42] Compare 51Did factor words ignoring case, and name the version of a short payload (#141) Two places where the 51Did package differs from the specification, found by a conformance check of this package against [specifications at 706de1f0](https://github.com/51Degrees/specifications/tree/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc). --- .../java/fiftyone/pipeline/did/FodId.java | 32 +++++++++------ .../fiftyone/pipeline/did/RedeemResult.java | 7 +++- .../fiftyone/pipeline/did/DidClientTests.java | 40 ++++++++++++++++++ .../pipeline/did/FodIdParseTests.java | 41 +++++++++++++++++++ 4 files changed, 105 insertions(+), 15 deletions(-) diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java index e94ef24d2..7af5ed21d 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java @@ -257,29 +257,35 @@ private static FodIdParseResult read(OwidParseResult envelope) { * already read or signed. This is the one walk of the payload that every * reader, throwing or not, goes through. *

- * The rules are lower bounds only. The header must be present before the - * type can be read, and the type then sets the least the payload can - * hold. Anything longer is accepted as it stands, because the bytes past - * the match key are the Terms and then a creator context section whose - * shape the cloud judges. The Terms adds nothing to those bounds, since - * a payload that ends at the match key reads as a Terms of zero. + * The version is read from the first byte before anything else, so a + * payload of a version this package does not know is refused naming + * that version however short it is. The other rules are lower bounds + * only. The header must be present before the type can be read, and the + * type then sets the least the payload can hold. Anything longer is + * accepted as it stands, because the bytes past the match key are the + * Terms and then a creator context section whose shape the cloud + * judges. The Terms adds nothing to those bounds, since a payload that + * ends at the match key reads as a Terms of zero. */ private static FodIdParseResult read(Owid owid) { byte[] payload = owid.getPayload(); - if (payload.length < HEADER_LENGTH) { + if (payload.length <= FLAGS_OFFSET) { return FodIdParseResult.failed(FodIdParseStatus.PAYLOAD_TOO_SHORT); } int flags = payload[FLAGS_OFFSET] & 0xFF; - // The version is read before any field, because a later version - // exists precisely because a field moved. Reading a payload of a - // version this package does not know under the layout it does know - // would answer with values that are wrong rather than absent, - // which is worse than refusing, and a version that nothing checks - // protects nothing. + // The version is read before any field, and before the header's + // length is checked, because a later version exists precisely + // because a field moved. Reading a payload of a version this package + // does not know under the layout it does know would answer with + // values that are wrong rather than absent, which is worse than + // refusing, and a version that nothing checks protects nothing. int payloadVersion = payloadVersionOf(flags); if (payloadVersion != SUPPORTED_PAYLOAD_VERSION) { return FodIdParseResult.unsupportedPayloadVersion(payloadVersion); } + if (payload.length < HEADER_LENGTH) { + return FodIdParseResult.failed(FodIdParseStatus.PAYLOAD_TOO_SHORT); + } // A payload with no usage bit set states no usage at all. The cloud // never writes one, so it is damaged or forged, and it is refused // here rather than read, because the only safe answer to it is not diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/RedeemResult.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/RedeemResult.java index a2625f942..fbeeca1b0 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/RedeemResult.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/RedeemResult.java @@ -28,6 +28,7 @@ import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; /** @@ -196,15 +197,17 @@ public String getValue() { * Verified, misconfigured and notrecorded are each read on their own, * because none of them is a mismatch. Anything else, including a word * this client does not know, is a mismatch, so an unexpected value - * never reads as a pass. + * never reads as a pass. The comparison ignores case, so the word is + * lower-cased in the root locale first. * * @param value the factor's string from the cloud, or null * @return the matching outcome, or {@link #MISMATCH} */ public static Factor fromValue(String value) { if (value != null) { + String word = value.toLowerCase(Locale.ROOT); for (Factor factor : values()) { - if (factor.value.equals(value)) { + if (factor.value.equals(word)) { return factor; } } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java index 501a2cabf..08f8af717 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Locale; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; @@ -1038,6 +1039,45 @@ public void redeem_UnknownFactorValue_IsStillAMismatch() result.getFactors().get("transport")); } + /** + * The comparison of a factor word ignores case, so each word in other + * letters reads as its own outcome rather than as a mismatch. The + * default locale is Turkish, where lower-casing an I in the default + * locale gives a dotless i. + */ + @Test + public void redeem_FactorWordsInOtherLetters_ReadAsTheirOutcome() + throws Exception { + Locale saved = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + try { + transport.queue(200, "{\"signature\":\"verified\"," + + "\"context\":\"mismatch\"," + + "\"factors\":{\"transport\":\"Verified\"," + + "\"device\":\"NotRecorded\"," + + "\"browserip\":\"MISCONFIGURED\"," + + "\"connectionip\":\"VERIFIED\"," + + "\"asn\":\"Mismatch\"}}"); + FodId fodId = key2.fodIdAt(canonicalPayload(), WEEK2); + + RedeemResult result = + client.redeem(fodId, "sealed", "abc").join(); + + assertEquals(RedeemResult.Factor.VERIFIED, + result.getFactors().get("transport")); + assertEquals(RedeemResult.Factor.NOT_RECORDED, + result.getFactors().get("device")); + assertEquals(RedeemResult.Factor.MISCONFIGURED, + result.getFactors().get("browserip")); + assertEquals(RedeemResult.Factor.VERIFIED, + result.getFactors().get("connectionip")); + assertEquals(RedeemResult.Factor.MISMATCH, + result.getFactors().get("asn")); + } finally { + Locale.setDefault(saved); + } + } + /** Every outcome reports the cloud's own word for itself. */ @Test public void factor_GetValue_IsTheCloudsWord() { diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdParseTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdParseTests.java index fec3bafaa..360f9ca70 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdParseTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdParseTests.java @@ -491,6 +491,47 @@ public void version_NotZero_IsRefused() throws Exception { } } + /** + * The version is read from the first byte, before the layout that sets + * the header's length, so a payload too short for that header is still + * refused naming the version it gives. A short payload of version 0 is + * too short. + */ + @Test + public void version_ShortPayload_IsRefusedNamingTheVersion() + throws Exception { + for (int length = 1; length < FodId.HEADER_LENGTH; length++) { + byte[] shortPayload = Arrays.copyOf(canonicalPayload(), length); + + assertEquals("length " + length + " version 0", + FodIdParseStatus.PAYLOAD_TOO_SHORT, + FodId.tryFromBase64(factory + .signedOwidAt(withVersion(shortPayload, 0), DATE) + .asBase64()).getStatus()); + + for (int version : new int[] { 1, 2, 3 }) { + String base64 = factory + .signedOwidAt(withVersion(shortPayload, version), DATE) + .asBase64(); + String label = "length " + length + " version " + version; + + FodIdParseResult result = FodId.tryFromBase64(base64); + assertEquals(label, + FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, + result.getStatus()); + assertEquals(label, version, result.getPayloadVersion()); + assertNull(label, result.getValue()); + try { + FodId.fromBase64(base64); + fail(label + " should have been refused"); + } catch (IllegalArgumentException thrown) { + assertTrue(label + ": " + thrown.getMessage(), + thrown.getMessage().contains("version " + version)); + } + } + } + } + /** * The throwing readers name the version they found, so whoever reads * the message knows which layout the identifier claims rather than only From 30e8df60a6e9aab33b3b5ffa7cf7b7439f18a9df Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 04:32:52 +0000 Subject: [PATCH 23/42] Build and run the examples from a fresh clone on the runner's own JDK (#137) Two gaps against the examples rule, that every example builds and runs both from a fresh clone and against the published packages, found in the audit of the examples on 13 November. --- .github/workflows/examples-fresh-clone.yml | 124 +++++++++++++++++++++ AGENTS.md | 24 ++-- ci/README.md | 8 ++ ci/run-fresh-clone-example.ps1 | 44 ++++++++ examples/device-detection/pom.xml | 2 +- examples/ip-intelligence/pom.xml | 2 +- 6 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/examples-fresh-clone.yml create mode 100644 ci/run-fresh-clone-example.ps1 diff --git a/.github/workflows/examples-fresh-clone.yml b/.github/workflows/examples-fresh-clone.yml new file mode 100644 index 000000000..e4eb3185b --- /dev/null +++ b/.github/workflows/examples-fresh-clone.yml @@ -0,0 +1,124 @@ +name: Examples from a fresh clone + +# Builds each product's console examples as AGENTS.md says, from a clone +# with its submodules, against the packages published on Maven Central, with +# the JDK and Maven the runner has. Then runs the getting started example +# from the repository root with no settings. A fresh clone holds only a link +# for each free data file. On Ubuntu the example downloads its file, which +# tests the download, and the job keeps the file in the cache. The other +# systems take it from there, so each file is downloaded once a run. + +on: + pull_request: + paths: + - 'examples/**' + - 'cxx' + - '.github/workflows/examples-fresh-clone.yml' + - 'ci/run-fresh-clone-example.ps1' + push: + branches: + - main + - rc + schedule: + - cron: '30 4 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + download: + name: ${{ matrix.product }} on ubuntu-latest, downloading its data file + strategy: + fail-fast: false + matrix: + include: + - product: device-detection + example: fiftyone.devicedetection.examples.console.GettingStartedOnPrem + file: cxx/data/device-detection/51Degrees-LiteV4.1.hash + - product: ip-intelligence + example: fiftyone.ipintelligence.examples.console.GettingStartedOnPrem + file: cxx/data/ip-intelligence/51Degrees-LiteV41.ipi + runs-on: ubuntu-latest + steps: + - name: Clone the repository with its submodules + uses: actions/checkout@v4 + with: + submodules: recursive + persist-credentials: false + + - name: The runner's Java and Maven + shell: pwsh + run: | + java -version + mvn -v + + - name: Build the console examples against the published packages + working-directory: examples/${{ matrix.product }} + shell: pwsh + run: mvn -B --no-transfer-progress -pl console -am package -DskipTests + + - name: Run the getting started example from the repository root + shell: pwsh + run: >- + ./ci/run-fresh-clone-example.ps1 -Product '${{ matrix.product }}' + -Example '${{ matrix.example }}' -ExpectDownload $true + + # The key follows the link, so a new free file gets a new entry, and + # the same file is kept once rather than once a run. + - name: Keep the data file for the other systems + uses: actions/cache/save@v4 + with: + path: ${{ matrix.file }} + key: fresh-clone-${{ matrix.product }}-${{ hashFiles(format('{0}.link', matrix.file)) }} + enableCrossOsArchive: true + + other-systems: + name: ${{ matrix.product }} on ${{ matrix.os }} + needs: download + strategy: + fail-fast: false + matrix: + # macOS runners are shared by the whole organisation, so pull + # requests leave them out and the pushes and the daily run use them. + os: ${{ fromJSON(github.event_name == 'pull_request' && '["windows-latest"]' || '["windows-latest", "macos-latest"]') }} + product: [device-detection, ip-intelligence] + include: + - product: device-detection + example: fiftyone.devicedetection.examples.console.GettingStartedOnPrem + file: cxx/data/device-detection/51Degrees-LiteV4.1.hash + - product: ip-intelligence + example: fiftyone.ipintelligence.examples.console.GettingStartedOnPrem + file: cxx/data/ip-intelligence/51Degrees-LiteV41.ipi + runs-on: ${{ matrix.os }} + steps: + - name: Clone the repository with its submodules + uses: actions/checkout@v4 + with: + submodules: recursive + persist-credentials: false + + - name: Take the data file the Ubuntu job downloaded + uses: actions/cache/restore@v4 + with: + path: ${{ matrix.file }} + key: fresh-clone-${{ matrix.product }}-${{ hashFiles(format('{0}.link', matrix.file)) }} + enableCrossOsArchive: true + fail-on-cache-miss: true + + - name: The runner's Java and Maven + shell: pwsh + run: | + java -version + mvn -v + + - name: Build the console examples against the published packages + working-directory: examples/${{ matrix.product }} + shell: pwsh + run: mvn -B --no-transfer-progress -pl console -am package -DskipTests + + - name: Run the getting started example from the repository root + shell: pwsh + run: >- + ./ci/run-fresh-clone-example.ps1 -Product '${{ matrix.product }}' + -Example '${{ matrix.example }}' -ExpectDownload $false diff --git a/AGENTS.md b/AGENTS.md index df51ec15e..fa77ffdfc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,9 @@ the native engines from `cxx` and does need them, as the README says. ### Prerequisites - Git. -- A JDK, 8 or later. CI tests 8, 11, 17 and 21 (`ci/options.json`). +- A JDK, 8 or later. CI tests 8, 11, 17 and 21 (`ci/options.json`), and + builds and runs the examples on the JDK each runner has (see + [What CI tests](#pitfalls)). - Maven 3.8 or later, or the `mvnw` wrapper at the root. - Network access to GitHub, to Maven Central for the packages the examples use, and to `distributor.51degrees.com` for the data files. @@ -215,12 +217,20 @@ example. The IP Intelligence cloud examples read their key from for Windows x64 and x86, Linux x64 and aarch64, and macOS x64 and aarch64. The `ip-intelligence.engine.on-premise` 4.5.128 jar has the same, without Windows x86. On a platform a jar lacks, the example cannot load the engine. -- **What CI does not test.** On a pull request CI builds and tests the - examples with the `dev` profile (`ci/build-project.ps1` and - `ci/run-unit-tests.ps1`), and against the published packages only when it - publishes (`ci/run-integration-tests.ps1`). It never runs the console - example jars as above, and the example tests use the paid data files when - CI has them. +- **What CI tests.** On a pull request CI builds and tests the examples + with the `dev` profile (`ci/build-project.ps1` and + `ci/run-unit-tests.ps1`), and the example tests use the paid data files + when CI has them. Publish tests them against the packages it has just + built (`ci/run-integration-tests.ps1`). + [examples-fresh-clone.yml](.github/workflows/examples-fresh-clone.yml) + does what [Run the examples](#run-the-examples) says, with the JDK and + Maven each runner has. It + builds the console examples against the published packages and runs each + product's `GettingStartedOnPrem` jar from the repository root with no + settings. On Ubuntu the free data file is downloaded, as from a fresh + clone, and the Windows and macOS jobs take that file from the run's cache, + so each file is downloaded once a run. It runs on each push to `main` and + `rc`, daily, and on a pull request that changes the examples. ## Environment variable names diff --git a/ci/README.md b/ci/README.md index 4ba256c91..1c3d50fc1 100644 --- a/ci/README.md +++ b/ci/README.md @@ -48,6 +48,14 @@ repositories used to do for one package tree each. of cxx. No test job downloads a data file, so a file added to the data repository needs an entry in common-ci's `steps/fetch-assets.ps1` before CI can use it. +- [run-fresh-clone-example.ps1](run-fresh-clone-example.ps1) runs a + product's console example jar from the repository root for + [examples-fresh-clone.yml](../.github/workflows/examples-fresh-clone.yml). + It fails unless the example exits with 0 and prints results, and unless it + downloads its free data file exactly when the job expects it to. That + workflow's Ubuntu job downloads each product's free data file once a run, + as a test of the download, and its other jobs take the file from the cache + the Ubuntu job saves. - [run-performance-tests.ps1](run-performance-tests.ps1) measures the Pipeline API, Device Detection and IP Intelligence and writes one results file per configuration. The Device Detection and IP Intelligence metrics are prefixed diff --git a/ci/run-fresh-clone-example.ps1 b/ci/run-fresh-clone-example.ps1 new file mode 100644 index 000000000..5c1e2102b --- /dev/null +++ b/ci/run-fresh-clone-example.ps1 @@ -0,0 +1,44 @@ +param ( + [Parameter(Mandatory)][string]$Product, + [Parameter(Mandatory)][string]$Example, + # True where the clone holds only the data file's link, so the example + # has to download the file. False where the file is already in place. + [Parameter(Mandatory)][bool]$ExpectDownload +) + +# Runs a product's console example jar from the repository root with no +# settings, as AGENTS.md's "Run the examples" does, and fails unless it ends +# with exit code 0, prints results, and downloads its data file exactly when +# expected. + +$jar = Get-ChildItem "examples/$Product/console/target" ` + -Filter '*-jar-with-dependencies.jar' | Select-Object -First 1 +if ($null -eq $jar) { + Write-Output '::error::The build made no jar with dependencies.' + exit 1 +} +$output = & java -cp $jar.FullName $Example 2>&1 | ForEach-Object { "$_" } +$code = $LASTEXITCODE +$output +$text = $output -join "`n" +$failed = $false +if ($code -ne 0) { + Write-Output "::error::The example exited with $code." + $failed = $true +} +$downloaded = $text -match 'is missing, so it is being downloaded' +if ($ExpectDownload -and -not $downloaded) { + Write-Output '::error::The example did not download its free data file.' + $failed = $true +} +if (-not $ExpectDownload -and $downloaded) { + Write-Output '::error::The example downloaded its data file, which should have come from the cache.' + $failed = $true +} +if ($text -notmatch 'Results:') { + Write-Output '::error::The example printed no results.' + $failed = $true +} +if ($failed) { + exit 1 +} diff --git a/examples/device-detection/pom.xml b/examples/device-detection/pom.xml index 8e9dd78bc..05c6fe34a 100644 --- a/examples/device-detection/pom.xml +++ b/examples/device-detection/pom.xml @@ -62,7 +62,7 @@ 2.0.20 1.3.16 - 3.26.0 + 3.28.0 diff --git a/examples/ip-intelligence/pom.xml b/examples/ip-intelligence/pom.xml index 8d7ff14cf..585aaa426 100644 --- a/examples/ip-intelligence/pom.xml +++ b/examples/ip-intelligence/pom.xml @@ -63,7 +63,7 @@ 2.0.20 1.3.16 - 3.26.0 + 3.28.0 From 206e9e9a465cdf1cc6188017c2eb1a84b9f4456d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 09:31:31 +0000 Subject: [PATCH 24/42] Test the positions of the elements the web integration adds (#155) Tests only. The web integration adds the Sequence Element, the JSON Builder Element, the JavaScript Builder Element and the Set Headers Element where the application's Pipeline lacks them, in the positions the specification gives, but the only test checked that each one is there, not where. This adds a test of the positions. --- .../web/AddedElementPositionTests.java | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/AddedElementPositionTests.java diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/AddedElementPositionTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/AddedElementPositionTests.java new file mode 100644 index 000000000..efd2b6607 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/AddedElementPositionTests.java @@ -0,0 +1,135 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The positions the web integration gives the elements it adds, as the web + * integration specification's Pipeline construction section sets them out. + * The Sequence Element is added first, the JavaScript Builder Element after + * every other element, the JSON Builder Element immediately before the + * JavaScript Builder Element, and the Set Headers Element after every other + * element, so the last two are the JavaScript Builder and the Set Headers + * Element in either order. + */ +public class AddedElementPositionTests { + + private static final String MULTIPLY_BY = "MultiplyByElement"; + + private static final String JAVASCRIPT_BUILDER = + JavaScriptBuilderElement.class.getSimpleName(); + + private static ElementOptions element(String builderName) { + ElementOptions element = new ElementOptions(); + element.builderName = builderName; + return element; + } + + private static ElementOptions multiplyBy() { + ElementOptions element = element(MULTIPLY_BY); + element.buildParameters.put("Multiple", "8"); + return element; + } + + /** + * Builds the Pipeline as the servlet filter does, with client-side + * properties, the Sequence Element and the Set Headers Element turned on, + * and returns the simple class names of its elements in order. + */ + private static List namesBuiltFrom(ElementOptions... configured) + throws Exception { + PipelineOptions options = new PipelineOptions(); + for (ElementOptions element : configured) { + options.elements.add(element); + } + List names = new ArrayList<>(); + try (Pipeline pipeline = StartupHelpers.buildFromConfiguration( + new PipelineBuilder(), options, true, null, + Constants.DEFAULT_JSON_PATH, true, true)) { + for (FlowElement element : pipeline.getFlowElements()) { + names.add(element.getClass().getSimpleName()); + } + } + return names; + } + + private static void assertPositions(List names) { + int last = names.size() - 1; + int sequence = names.indexOf(SequenceElement.class.getSimpleName()); + int json = names.indexOf(JsonBuilderElement.class.getSimpleName()); + int javaScript = names.indexOf(JAVASCRIPT_BUILDER); + int setHeaders = names.indexOf(SetHeadersElement.class.getSimpleName()); + assertEquals("The Sequence Element should be first. " + names, + 0, sequence); + assertEquals("The JSON Builder Element should be immediately before " + + "the JavaScript Builder Element. " + names, + javaScript - 1, json); + assertTrue("The JavaScript Builder and Set Headers Elements should " + + "be the last two. " + names, + Math.min(javaScript, setHeaders) == last - 1 && + Math.max(javaScript, setHeaders) == last); + } + + /** + * With only an element of the application's configured, each added + * element takes its position around it. + */ + @Test + public void AddedElements_TakeTheirPositions() throws Exception { + List names = namesBuiltFrom(multiplyBy()); + + assertEquals(names.toString(), 5, names.size()); + assertEquals(names.toString(), 1, names.indexOf(MULTIPLY_BY)); + assertPositions(names); + } + + /** + * With a JavaScript Builder Element configured, the JSON Builder Element + * is added immediately before it. + */ + @Test + public void JsonBuilder_AddedImmediatelyBeforeAConfiguredJavaScriptBuilder() + throws Exception { + List names = namesBuiltFrom( + multiplyBy(), element(JAVASCRIPT_BUILDER)); + + assertEquals(names.toString(), 5, names.size()); + assertPositions(names); + } +} From 50e63e49ffffd07325a5dc390cbacb02972bb619 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 09:31:41 +0000 Subject: [PATCH 25/42] Test that FodId keeps the byte layout, the payload version and its constructor out of reach (#157) Tests only. Three rules of the 51Did specification limit what `FodId` offers a caller, and the Java package meets all three, but no test would fail if a later change broke one of them. This adds those tests. It is the follow-up that [#141](https://github.com/51Degrees/51degrees-java/pull/141) names under What remains. --- .../pipeline/did/FodIdSurfaceTests.java | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdSurfaceTests.java diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdSurfaceTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdSurfaceTests.java new file mode 100644 index 000000000..d8865e67a --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdSurfaceTests.java @@ -0,0 +1,131 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import com.swancommunity.owid.Owid; +import com.swancommunity.owid.Version; +import org.junit.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Member; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Checks that {@link FodId} keeps the byte layout out of its public + * surface, has no member for the payload version, and is only ever made by + * reading a complete Envelope, as the 51Did specification asks. + */ +public class FodIdSurfaceTests { + + /** + * True when a caller outside the package can reach the member. + */ + private static boolean isVisible(Member member) { + int modifiers = member.getModifiers(); + return Modifier.isPublic(modifiers) || Modifier.isProtected(modifiers); + } + + /** + * did-specification/README.md lines 33 to 35, "Application code MUST + * NOT read the layout by hand, and the packages therefore keep the + * offsets and lengths out of their public surface." + */ + @Test + public void layout_OffsetsAndLengths_AreNotPublic() { + List visible = new ArrayList<>(); + for (Field field : FodId.class.getDeclaredFields()) { + if (isVisible(field)) { + visible.add(field.getName()); + } + } + assertEquals( + "FodId fields a caller can reach", + new ArrayList(), + visible); + } + + /** + * did-specification/package-surface.md lines 55 and 56, "The version + * belongs to the refusal and MUST NOT appear as a member of the + * identifier". The only version a caller reads from a FodId is the + * Envelope's own, through getVersion. + */ + @Test + public void payloadVersion_IsNotAMemberOfTheIdentifier() { + List visible = new ArrayList<>(); + for (Method method : FodId.class.getDeclaredMethods()) { + boolean namesVersion = method.getName() + .toLowerCase(Locale.ROOT) + .contains("version"); + if (isVisible(method) && + namesVersion && + method.getReturnType() != Version.class) { + visible.add(method.getName()); + } + } + assertEquals( + "FodId methods that give a version other than the Envelope's", + new ArrayList(), + visible); + } + + /** + * did-specification/identifier-layout.md lines 42 and 43, "a 51Did + * package MUST create instances only by reading bytes that are already + * a complete Envelope." + */ + @Test + public void instances_AreMadeOnlyByReadingAnEnvelope() { + assertTrue( + "FodId is final, so no subclass can make an instance", + Modifier.isFinal(FodId.class.getModifiers())); + for (Constructor constructor : + FodId.class.getDeclaredConstructors()) { + assertTrue( + "constructor " + constructor + " is private", + Modifier.isPrivate(constructor.getModifiers())); + } + List> readable = Arrays.>asList( + String.class, byte[].class, Owid.class); + for (Method method : FodId.class.getDeclaredMethods()) { + if (isVisible(method) && + Modifier.isStatic(method.getModifiers()) && + method.getReturnType() == FodId.class) { + Class[] parameters = method.getParameterTypes(); + assertTrue( + method.getName() + " takes one encoded Envelope", + parameters.length == 1 && + readable.contains(parameters[0])); + } + } + } +} From 8e53055e2b0395eea73844728cd6536b10a9ee5f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 09:31:52 +0000 Subject: [PATCH 26/42] Say at start-up what the Enterprise data file adds in the IP Intelligence API example (#159) The IP Intelligence API example answers in JSON, so on the free Lite data file a caller saw the three registration properties and was never told what the Enterprise data file adds, where to ask for it, or where to find out more. --- .../examples/web/GettingStartedApi.java | 35 +++++++- .../web/GettingStartedApiNotesTest.java | 80 +++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 examples/ip-intelligence/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiNotesTest.java diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 34b00b036..3ecc63b86 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -50,6 +50,7 @@ import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.FlowElement; @@ -74,6 +75,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Paths; import java.security.MessageDigest; @@ -94,7 +97,10 @@ * and a JSON builder element which serialises the results. *

* The data file can be supplied as a command line argument. When no argument is given - * the enterprise data file is looked for in the cxx/data/ip-intelligence directory. + * the free Lite data file is used, and downloaded when it is missing. + *

+ * The service answers in JSON, so when it starts it writes what the Enterprise data file + * adds, when the file in use is a free one, and the "Find out more" section. *

* Note that the .NET version of this example also includes a device detection engine * in the same pipeline. This Java version is IP Intelligence only. @@ -105,6 +111,14 @@ public class GettingStartedApi extends HttpServlet { public static final String DATA_FILE_INIT_PARAM = "dataFile"; public static Logger logger = LoggerFactory.getLogger(GettingStartedApi.class); + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "web-getting-started.api-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedapi.java"; + + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.api-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedapi.java&utm_term=enterprise-data-file"; + private Pipeline pipeline; private IPIntelligenceOnPremiseEngine engine; @@ -153,6 +167,25 @@ public void init() throws ServletException { throw new ServletException("Failed to create pipeline from data file '" + dataFileLocation + "'", e); } + writeStartupNotes(System.out, engine); + } + + /** + * Writes what the Enterprise data file adds, with the link to the + * contact page, when the engine uses a free data file, then the "Find + * out more" section. + * @param output where to write + * @param engine the engine the service uses + */ + static void writeStartupNotes( + PrintStream output, + IPIntelligenceOnPremiseEngine engine) { + if (DataFileHelper.isFreeDataFile(engine)) { + DataFileHelper.writeEnterpriseOffer( + new PrintWriter(output), CONTACT_US); + } + FindOutMore.write(output, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } @Override diff --git a/examples/ip-intelligence/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiNotesTest.java b/examples/ip-intelligence/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiNotesTest.java new file mode 100644 index 000000000..897d74f93 --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiNotesTest.java @@ -0,0 +1,80 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import fiftyone.ipintelligence.examples.shared.FindOutMore; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.assertTrue; + +/** + * The service answers in JSON, so what it writes when it starts is the only + * place the person running it reads what the Enterprise data file adds and + * where to find out more. + */ +public class GettingStartedApiNotesTest { + + private static final String CONTACT_US = "https://51degrees.com/contact-us?"; + + /** + * On the free Lite data file, the service says what the Enterprise data + * file adds with the link to the contact page, then writes the "Find out + * more" section, closed by its rule. + */ + @Test + public void startupNotes_FreeDataFile() throws Exception { + String lite = DataFileHelper.getDataFileLocation( + DataFileHelper.LITE_DATA_FILE_REL_PATH); + try (IPIntelligenceOnPremiseEngine engine = + new IPIntelligenceOnPremiseEngineBuilder( + LoggerFactory.getILoggerFactory()) + .setAutoUpdate(false) + .build(lite, false)) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + GettingStartedApi.writeStartupNotes( + new PrintStream(buffer, true, "UTF-8"), engine); + String output = buffer.toString("UTF-8"); + + assertTrue("The output should say what the Enterprise data " + + "file adds:\n" + output, + output.contains("This example used a free data file.")); + assertTrue("The output should link the contact page:\n" + output, + output.contains(CONTACT_US)); + int heading = output.lastIndexOf(FindOutMore.HEADING); + assertTrue("The output should have the '" + FindOutMore.HEADING + + "' section after the link to the contact " + + "page:\n" + output, + heading > output.lastIndexOf(CONTACT_US)); + assertTrue("The '" + FindOutMore.HEADING + + "' section should end with its rule:\n" + output, + output.indexOf(FindOutMore.RULE, heading) > heading); + } + } +} From c2d982ac5d81f1b2ac2a70cf70bfa9ad85c811e4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 09:32:01 +0000 Subject: [PATCH 27/42] Keep the configured interval test's update check from running before its verify (#161) Tests only. `DataUpdateServiceTests.DataUpdateService_Register_AutoUpdateConfiguredInterval` fails about once in 600 runs, and it failed `Ubuntu_Java_17` in full build [43667464820](https://github.com/51Degrees/51degrees-java/actions/runs/43667464820) of [#156](https://github.com/51Degrees/51degrees-java/pull/156), which changes nothing in `pipeline.engines`. This fixes the cause rather than leaving it to a re-run, which is the rule for a test that fails on timing. [#122](https://github.com/51Degrees/51degrees-java/pull/122), which is open and not yet merged, adds that rule to AGENTS.md. --- .../pipeline/engines/services/DataUpdateServiceTests.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 0d1796004..6f139fca2 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -289,6 +289,13 @@ public void DataUpdateService_Register_AutoUpdateConfiguredInterval() { file.setEngine(engine); file.setConfiguration(config); + // With no polling interval the delay is the random number of whole + // seconds alone, which can be 0. The check is not run, so a check + // that starts at once cannot schedule again before the verify. + doReturn(mock(ScheduledFuture.class)).when(futureFactory).schedule( + any(Runnable.class), + anyLong()); + // Act dataUpdate.registerDataFile(file); From 4e537d33e0df4daefe7878d6e04faa88e71d8eec Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 10:35:35 +0000 Subject: [PATCH 28/42] Throw for a cloud property the response leaves out, as Location does (#143) The Device Detection and IP Intelligence cloud engines add an entry to the Element Data for every property in their metadata, whether or not the cloud response holds it ([`DeviceDetectionCloudEngine` line 148](https://github.com/51Degrees/51degrees-java/blob/b33daa0f32964dbf5b5f77ba5e947ef1ac75c55d/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java#L148) and [`IPIntelligenceCloudEngine` line 160](https://github.com/51Degrees/51degrees-java/blob/b33daa0f32964dbf5b5f77ba5e947ef1ac75c55d/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java#L160) at `rc` b33daa0f). A property the response leaves out therefore reads as a value that "has no value" with a null message, where it should throw `PropertyMissingException` with a message saying why the property is not there. --- .../DeviceDetectionCloudEngine.java | 17 +++++- .../flowelements/ProductMissingTests.java | 55 +++++++++++++++++-- .../IPIntelligenceCloudEngine.java | 17 +++++- .../flowelements/ProductMissingTests.java | 55 +++++++++++++++++-- 4 files changed, 132 insertions(+), 12 deletions(-) diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java index 400ca1dad..128e4ebc2 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.TreeMap; @@ -146,6 +147,12 @@ protected void processEngine(FlowData data, DeviceDataCloud aspectData) { new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (AspectPropertyMetaData property : getProperties()) { + // A property the response leaves out has no entry, so + // reading it throws a PropertyMissingException. + if (deviceObj.has( + property.getName().toLowerCase(Locale.ROOT)) == false) { + continue; + } String type = property.getType().getSimpleName(); switch(type) { case ("List"): @@ -389,6 +396,12 @@ private AspectPropertyValue getStringAspectPropertyValue( return stringValue; } + /** + * The message for a property with no value when the response gives no + * reason. + */ + private static final String UNKNOWN_NULL_REASON = "Unknown"; + /** * Get the reason for the value of a property not being present in the cloud * engine's JSON response. @@ -398,7 +411,9 @@ private AspectPropertyValue getStringAspectPropertyValue( */ private String getNoValueReason(JSONObject deviceObj, String key){ Object reason = tryToGet(deviceObj, key + "nullreason"); - return reason == null ? null : reason.toString(); + return reason == null || reason == JSONObject.NULL ? + UNKNOWN_NULL_REASON : + reason.toString(); } /** diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java index 2cd708eb6..163f9b00a 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java @@ -99,19 +99,64 @@ public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { /** * With the Device Detection product included, a response that holds no - * part for it leaves the element data without values and throws - * nothing, as the Processing section of the cloud aspect engine - * specification says. + * part for it raises nothing while processing, and reading a property + * throws PropertyMissingException, as for any property the response + * leaves out. */ @Test - public void ProductPresent_ResponseWithoutPart_NoValues() + public void ProductPresent_ResponseWithoutPart_PropertyIsMissing() throws Exception { - try (DeviceDetectionCloudEngine engine = buildEngine("device", "hardwarevendor")) { + try (DeviceDetectionCloudEngine engine = + buildEngine("device", "hardwarevendor")) { DeviceDataCloud data = process( engine, "{ 'ip': { 'registeredname': 'Example' }, " + "'javascriptProperties': [] }"); + try { + data.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response held nothing for the product. + } + } + } + + /** + * A property the response's part leaves out has no entry, so reading it + * throws PropertyMissingException rather than giving no value. + */ + @Test + public void ProductPresent_PropertyLeftOut_PropertyIsMissing() + throws Exception { + try (DeviceDetectionCloudEngine engine = + buildEngine("device", "hardwarevendor")) { + DeviceDataCloud data = process( + engine, + "{ 'device': { 'another': 'Example' }, " + + "'javascriptProperties': [] }"); + try { + data.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response's part did not hold the property. + } + } + } + + /** + * A null with no reason beside it is no value with the message Unknown. + */ + @Test + public void NullWithoutReason_MessageIsUnknown() throws Exception { + try (DeviceDetectionCloudEngine engine = + buildEngine("device", "hardwarevendor")) { + DeviceDataCloud data = process( + engine, + "{ 'device': { 'hardwarevendor': null }, " + + "'javascriptProperties': [] }"); assertFalse(data.getHardwareVendor().hasValue()); + assertEquals("Unknown", + data.getHardwareVendor().getNoValueMessage()); } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java index 99b84cbb3..0b0051a0b 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java @@ -49,6 +49,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -158,6 +159,12 @@ protected void processEngine(FlowData data, IPIntelligenceDataCloud aspectData) new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (AspectPropertyMetaData property : getProperties()) { + // A property the response leaves out has no entry, so + // reading it throws a PropertyMissingException. + if (ipiObj.has( + property.getName().toLowerCase(Locale.ROOT)) == false) { + continue; + } if (weightedPropertyNames.contains( property.getName().toLowerCase())) { propertyMap.put( @@ -590,6 +597,12 @@ private AspectPropertyValue getStringAspectPropertyValue( return stringValue; } + /** + * The message for a property with no value when the response gives no + * reason. + */ + private static final String UNKNOWN_NULL_REASON = "Unknown"; + /** * Get the reason for the value of a property not being present in the cloud * engine's JSON response. @@ -599,7 +612,9 @@ private AspectPropertyValue getStringAspectPropertyValue( */ private String getNoValueReason(JSONObject ipiObj, String key){ Object reason = tryToGet(ipiObj, key + "nullreason"); - return reason == null ? null : reason.toString(); + return reason == null || reason == JSONObject.NULL ? + UNKNOWN_NULL_REASON : + reason.toString(); } /** diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java index a597f4662..1c0d68e37 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -99,19 +99,64 @@ public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { /** * With the IP Intelligence product included, a response that holds no - * part for it leaves the element data without values and throws - * nothing, as the Processing section of the cloud aspect engine - * specification says. + * part for it raises nothing while processing, and reading a property + * throws PropertyMissingException, as for any property the response + * leaves out. */ @Test - public void ProductPresent_ResponseWithoutPart_NoValues() + public void ProductPresent_ResponseWithoutPart_PropertyIsMissing() throws Exception { - try (IPIntelligenceCloudEngine engine = buildEngine("ip", "registeredname")) { + try (IPIntelligenceCloudEngine engine = + buildEngine("ip", "registeredname")) { IPIntelligenceDataCloud data = process( engine, "{ 'device': { 'hardwarevendor': 'Example' }, " + "'javascriptProperties': [] }"); + try { + data.getRegisteredName(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response held nothing for the product. + } + } + } + + /** + * A property the response's part leaves out has no entry, so reading it + * throws PropertyMissingException rather than giving no value. + */ + @Test + public void ProductPresent_PropertyLeftOut_PropertyIsMissing() + throws Exception { + try (IPIntelligenceCloudEngine engine = + buildEngine("ip", "registeredname")) { + IPIntelligenceDataCloud data = process( + engine, + "{ 'ip': { 'another': 'Example' }, " + + "'javascriptProperties': [] }"); + try { + data.getRegisteredName(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response's part did not hold the property. + } + } + } + + /** + * A null with no reason beside it is no value with the message Unknown. + */ + @Test + public void NullWithoutReason_MessageIsUnknown() throws Exception { + try (IPIntelligenceCloudEngine engine = + buildEngine("ip", "registeredname")) { + IPIntelligenceDataCloud data = process( + engine, + "{ 'ip': { 'registeredname': null }, " + + "'javascriptProperties': [] }"); assertFalse(data.getRegisteredName().hasValue()); + assertEquals("Unknown", + data.getRegisteredName().getNoValueMessage()); } } From 8e4244e0e85dad3b40ddd5353617e9b5fcd87ac7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 10:35:45 +0000 Subject: [PATCH 29/42] Write the usage sharing record as the specification's Processing section sets out (#139) Part of [#28](https://github.com/51Degrees/51degrees-java/issues/28), which stays open for one point only, the order of the Evidence entries (point 9 below). --- .../fiftyone/flowelements/ShareUsageBase.java | 58 ++- .../flowelements/ShareUsageElement.java | 37 +- .../flowelements/ShareUsageElementTests.java | 13 +- .../ShareUsageRecordFormTests.java | 386 ++++++++++++++++++ 4 files changed, 468 insertions(+), 26 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageRecordFormTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 1fd5bf088..a0da6bced 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -43,6 +43,9 @@ import java.net.SocketException; import java.net.UnknownHostException; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.*; @@ -53,7 +56,7 @@ /** * Abstract base class for ShareUsage elements. Contains common functionality * such as filtering the evidence and building the XML records. - * @see Specification + * @see Specification */ public abstract class ShareUsageBase extends FlowElementBase { @@ -62,8 +65,26 @@ public abstract class ShareUsageBase * IP Addresses of local host device. */ private static final InetAddress[] localHosts; + + /** The form of DateSent, which is the time in UTC. */ + private static final String DATE_SENT_PATTERN = "yyyy-MM-dd'T'HH:mm:ss"; + + private static final DateTimeFormatter DATE_SENT_FORMAT = + DateTimeFormatter.ofPattern(DATE_SENT_PATTERN, Locale.ROOT) + .withZone(ZoneOffset.UTC); + + /** + * Formats a date as DateSent, in UTC. + * @deprecated the record takes DateSent from {@link #dateSentNow()}, + * which is safe to call from any thread. + */ + @Deprecated protected static final SimpleDateFormat DATE_FMT = - new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + new SimpleDateFormat(DATE_SENT_PATTERN); + + static { + DATE_FMT.setTimeZone(TimeZone.getTimeZone("UTC")); + } static { InetAddress[] localHosts1; @@ -184,8 +205,9 @@ public abstract class ShareUsageBase private List flowElements = null; /** - * Version of the OS the pipeline is being run on, as reported by - * System.getProperty("os.name"). + * The name of the operating system the Pipeline runs on, followed by + * its version where the JVM gives one, as System.getProperty("os.name") + * and System.getProperty("os.version") report them. Written as Platform. */ protected final String osVersion; @@ -536,7 +558,8 @@ protected ShareUsageBase( // Some data is going to stay the same on all requests so we can // gather that now. languageVersion = System.getProperty("java.version"); - osVersion = System.getProperty("os.name"); + osVersion = platform( + System.getProperty("os.name"), System.getProperty("os.version")); this.enginesVersion = getClass().getPackage().getImplementationVersion(); coreVersion = Pipeline.class.getPackage().getImplementationVersion(); @@ -576,7 +599,28 @@ protected ShareUsageBase( } /** - * Return a list of {@link FlowElement} in the pipeline. + * The time now in UTC, in the form DateSent is written in. + * @return the text for DateSent + */ + protected static String dateSentNow() { + return DATE_SENT_FORMAT.format(Instant.now()); + } + + /** + * The text for Platform, which is the name of the operating system + * followed by its version where there is one. + */ + private static String platform(String name, String version) { + String platform = name == null ? "" : name.trim(); + if (version != null && version.trim().isEmpty() == false) { + platform = (platform + " " + version.trim()).trim(); + } + return platform; + } + + /** + * Return the fully qualified class name of each {@link FlowElement} in + * the pipeline. * If the list is null then populate from the pipeline. * If there are multiple or no pipelines then log an error. * @return list of flow elements @@ -589,7 +633,7 @@ protected List getFlowElements() { pipeline = getPipelines().get(0); List list = new ArrayList<>(); for ( FlowElement element : pipeline.getFlowElements()) { - list.add(element.getClass().getSimpleName()); + list.add(element.getClass().getName()); } flowElements = list; } else { diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index 58109cb2a..26809fd20 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -52,7 +52,7 @@ /** * Flow element that sends usage data to 51Degrees for analysis. - * @see Specification + * @see Specification */ public class ShareUsageElement extends ShareUsageBase { // max time to wait to send data @@ -426,7 +426,7 @@ public class ShareUsageElement extends ShareUsageBase { shareAllEvidence, shareAllQueryStringParameters); - headers.put("Content-Type", "text/xml; charset=utf-8"); + headers.put("Content-Type", "text/xml"); // assume default GZIP connection headers.put("Content-Encoding", "gzip"); @@ -521,7 +521,6 @@ protected void sendAsXML(List allData) throws Exception { private void streamXml(List allData, OutputStream os) throws XMLStreamException { XMLStreamWriter writer = xmlOutputFactory.createXMLStreamWriter(os, "UTF-8"); writer.writeStartElement("Devices"); - writer.writeAttribute("version", "1.1"); for (ShareUsageData data : allData) { writeXmlData(writer, data); } @@ -537,15 +536,17 @@ private void streamXml(List allData, OutputStream os) throws XML */ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws XMLStreamException { writer.writeStartElement("Device"); - // The elements follow the order of the specification's example. + // The elements follow the order of the specification's example, and + // each one the example does not mark optional is always written, + // empty when there is no value. // The SessionID used to track a series of requests - writeXmlElement(writer, "SessionId", data.sessionId); + writeRequiredXmlElement(writer, "SessionId", data.sessionId); // The sequence number of the request in a series of requests. writeXmlElement(writer, "Sequence", String.valueOf(data.sequence)); // The UTC date/time this entry was written - writeXmlElement(writer, "DateSent", DATE_FMT.format(new Date())); + writeXmlElement(writer, "DateSent", dateSentNow()); // The version number of the Pipeline API - writeXmlElement(writer, "Version", coreVersion); + writeRequiredXmlElement(writer, "Version", coreVersion); // Write Pipeline information // The product name writeXmlElement(writer, "Product", "Pipeline"); @@ -555,13 +556,13 @@ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws } writeXmlElement(writer, "Language", "java"); // The software language version - writeXmlElement(writer, "LanguageVersion", languageVersion); + writeRequiredXmlElement(writer, "LanguageVersion", languageVersion); // The client IP of the request - writeXmlElement(writer, "ClientIP", data.clientIP); + writeRequiredXmlElement(writer, "ClientIP", data.clientIP); // The IP of this server - writeXmlElement(writer, "ServerIP", getHostAddress()); + writeRequiredXmlElement(writer, "ServerIP", getHostAddress()); // The OS name and version - writeXmlElement(writer, "Platform", osVersion); + writeRequiredXmlElement(writer, "Platform", osVersion); // Write all other evidence data that has been included. for (Map.Entry> category : data.evidenceData.entrySet()) { @@ -597,6 +598,20 @@ protected void writeXmlElement(XMLStreamWriter writer, } } + /** + * Write an element the record always holds, empty when there is no + * value. + */ + private void writeRequiredXmlElement( + XMLStreamWriter writer, + String elementName, + String elementContent) throws XMLStreamException { + writeXmlElement( + writer, + elementName, + elementContent == null ? "" : elementContent); + } + /** * replace characters that cause problems in XML with the "Replacement character" */ diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index eb719098d..8107c8114 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -864,17 +864,14 @@ public void ShareUsageElement_RecordFollowsTheSpecificationExample() String[] expected = {"SessionId", "Sequence", "DateSent", "Version", "Product", "FlowElement", "Language", "LanguageVersion", "ClientIP", "ServerIP", "Platform"}; - List required = Arrays.asList("SessionId", "Sequence", - "DateSent", "Product", "Language", "LanguageVersion"); + // Every element is always written, empty when there is no value. + // FlowElement is written once for each element of the Pipeline, so + // not at all here, where the test's Pipeline holds no elements. int last = -1; for (String name : expected) { int index = names.indexOf(name); - if (!required.contains(name)) { - // Only present when the Pipeline, the request and the - // platform give one. - if (index < 0) { - continue; - } + if (index < 0 && name.equals("FlowElement")) { + continue; } assertTrue(index >= 0, name + " missing from " + names); assertTrue(index > last, name + " out of order in " + names); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageRecordFormTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageRecordFormTests.java new file mode 100644 index 000000000..1637f72b1 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageRecordFormTests.java @@ -0,0 +1,386 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.services.HttpClient; +import fiftyone.pipeline.engines.trackers.Tracker; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; +import static fiftyone.pipeline.core.Constants.EVIDENCE_HEADER_USERAGENT_KEY; +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The usage sharing record is written as the specification's Processing + * section sets it out. Each test sends one record through a Pipeline that + * holds a Sequence Element and the Usage Sharing Element, and reads what + * the element wrote. + */ +public class ShareUsageRecordFormTests { + + /** Keeps what the element writes. */ + private static class Uploader implements DataUploader { + final ByteArrayOutputStream written = new ByteArrayOutputStream(); + + @Override + public OutputStream getOutputStream() { + return written; + } + + @Override + public int getResponseCode() { + return 200; + } + } + + /** Keeps the headers the element sends through an HTTP client. */ + private static class HeaderClient implements HttpClient { + Map sent; + + @Override + public HttpURLConnection connect(URL url) { + return new HttpURLConnection(url) { + @Override + public int getResponseCode() { + return 200; + } + + @Override + public String getResponseMessage() { + return ""; + } + + @Override + public void disconnect() { + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public void connect() { + } + }; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + sent = new HashMap<>(headers); + return ""; + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return ""; + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return ""; + } + } + + private static final String DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss"; + + private Pipeline pipeline; + private ShareUsageElement element; + private Uploader uploader; + + @AfterEach + public void cleanUp() throws Exception { + if (pipeline != null) { + pipeline.close(); + } + } + + /** + * Build a Pipeline of a Sequence Element and a Usage Sharing Element + * that sends every record as soon as it has it. + */ + private void build() throws Exception { + SequenceElement sequence = new SequenceElement( + LoggerFactory.getLogger(SequenceElement.class)); + Tracker tracker = new Tracker() { + @Override + public boolean track(FlowData flowData) { + return true; + } + }; + element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 2, + 100, + 100, + 1, + true, + "http://51Degrees.com/test", + new ArrayList(), + new ArrayList(), + new ArrayList>(), + DEFAULT_SESSION_COOKIE_NAME, + tracker); + uploader = new Uploader(); + element.dataUploader = uploader; + pipeline = new PipelineBuilder() + .addFlowElement(sequence) + .addFlowElement(element) + .build(); + } + + /** Send one record and return the text the element wrote. */ + private String send() throws Exception { + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_HEADER_USERAGENT_KEY, " & co"); + evidence.put(EVIDENCE_CLIENTIP_KEY, "192.0.2.1"); + return send(evidence); + } + + /** + * Send one record of the Evidence given and return the text the + * element wrote. + */ + private String send(Map evidence) throws Exception { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + data.process(); + assertNotNull(element.getSendDataFuture()); + element.getSendDataFuture().get(); + } + return new String( + uploader.written.toByteArray(), StandardCharsets.UTF_8); + } + + private static Document parse(String xml) throws Exception { + return DocumentBuilderFactory.newInstance().newDocumentBuilder() + .parse(new ByteArrayInputStream( + xml.getBytes(StandardCharsets.UTF_8))); + } + + private static String text(Document doc, String name) { + NodeList nodes = doc.getElementsByTagName(name); + assertEquals(1, nodes.getLength(), "One " + name + " expected"); + return nodes.item(0).getTextContent(); + } + + /** The root is Devices alone, with no attribute. */ + @Test + public void Root_IsDevicesWithNoAttribute() throws Exception { + build(); + String xml = send(); + + assertTrue(xml.startsWith(""), xml); + assertTrue(xml.endsWith(""), xml); + assertEquals(0, + parse(xml).getDocumentElement().getAttributes().getLength()); + } + + /** + * DateSent is the time in UTC, whatever time zone the JVM runs in. + * {@code DATE_FMT} keeps the zone that was the default when the class + * loaded, so the test gives it the zone a JVM started in that zone + * has. + */ + @Test + @SuppressWarnings("deprecation") + public void DateSent_IsUtc_WhenTheDefaultTimeZoneIsNot() throws Exception { + TimeZone defaultBefore = TimeZone.getDefault(); + TimeZone formatBefore = ShareUsageBase.DATE_FMT.getTimeZone(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("Pacific/Kiritimati")); + ShareUsageBase.DATE_FMT.setTimeZone(TimeZone.getDefault()); + build(); + + long before = System.currentTimeMillis(); + String sent = text(parse(send()), "DateSent"); + long after = System.currentTimeMillis(); + + SimpleDateFormat utc = new SimpleDateFormat(DATE_PATTERN); + utc.setTimeZone(TimeZone.getTimeZone("UTC")); + long written = utc.parse(sent).getTime(); + // DateSent has no part of a second, so allow one either side. + assertTrue( + written >= before - 1000 && written <= after + 1000, + "DateSent was " + sent + ", which is not the time in UTC, " + + utc.format(before)); + } finally { + TimeZone.setDefault(defaultBefore); + ShareUsageBase.DATE_FMT.setTimeZone(formatBefore); + } + } + + /** Each FlowElement holds the fully qualified name of the class. */ + @Test + public void FlowElement_IsTheFullyQualifiedClassName() throws Exception { + build(); + NodeList names = parse(send()).getElementsByTagName("FlowElement"); + + assertEquals(2, names.getLength()); + assertEquals( + "fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement", + names.item(0).getTextContent()); + assertEquals( + "fiftyone.pipeline.engines.fiftyone.flowelements." + + "ShareUsageElement", + names.item(1).getTextContent()); + } + + /** Platform holds the name of the operating system and its version. */ + @Test + public void Platform_IsTheNameAndTheVersion() throws Exception { + String name = System.getProperty("os.name"); + String version = System.getProperty("os.version"); + assertNotNull(version, "This JVM gives no os.version to test with"); + build(); + + assertEquals( + name + " " + version, text(parse(send()), "Platform")); + } + + /** + * ClientIP is written, empty, when the Evidence holds no client IP, as + * the specification's example does not mark it optional. It stays in + * its place between LanguageVersion and ServerIP. + */ + @Test + public void ClientIP_IsWrittenEmpty_WhenTheEvidenceHasNone() + throws Exception { + build(); + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_HEADER_USERAGENT_KEY, " & co"); + String xml = send(evidence); + + assertEquals("", text(parse(xml), "ClientIP")); + assertTrue( + Pattern.compile( + "| ?/>)") + .matcher(xml).find(), + xml); + } + + /** The message is sent as text/xml, compressed with gzip. */ + @Test + public void ContentType_IsTextXml() throws Exception { + build(); + + assertEquals("text/xml", element.headers.get("Content-Type")); + assertEquals("gzip", element.headers.get("Content-Encoding")); + + // The same headers go to an HTTP client the element is given. + HeaderClient client = new HeaderClient(); + element.httpClient = client; + send(); + assertEquals("text/xml", client.sent.get("Content-Type")); + assertEquals("gzip", client.sent.get("Content-Encoding")); + } + + /** + * The whole record, compared as text. DateSent and ServerIP are read + * from the record, as they differ from run to run, and the values that + * name the runtime are the ones this JVM gives. Version is written + * even when the classes carry no version, as when the tests run from + * the build's class folders, and is then empty. + */ + @Test + public void WholeRecord_MatchesTheSpecificationExample() throws Exception { + build(); + String xml = send(); + + Matcher date = Pattern.compile( + "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2})") + .matcher(xml); + assertTrue(date.find(), xml); + Matcher session = Pattern.compile( + "([^<]+)").matcher(xml); + assertTrue(session.find(), xml); + Matcher server = Pattern.compile( + "([^<]*)").matcher(xml); + assertTrue(server.find(), xml); + String version = Pipeline.class.getPackage() + .getImplementationVersion(); + + StringBuilder expected = new StringBuilder(); + expected.append("") + .append("").append(session.group(1)) + .append("") + .append("1") + .append("").append(date.group(1)).append(""); + expected.append("").append(version == null ? "" : version) + .append(""); + expected.append("Pipeline") + .append("fiftyone.pipeline.engines.fiftyone." + + "flowelements.SequenceElement") + .append("fiftyone.pipeline.engines.fiftyone." + + "flowelements.ShareUsageElement") + .append("java") + .append("") + .append(System.getProperty("java.version")) + .append("") + .append("192.0.2.1") + .append("").append(server.group(1)) + .append("") + .append("").append(System.getProperty("os.name")) + .append(" ").append(System.getProperty("os.version")) + .append("") + .append("

") + .append("<iPhone> & co
") + .append(""); + + assertEquals(expected.toString(), xml); + } +} From e2472ee3abb7837f572f74961bcf75218b3bbbed Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 10:35:55 +0000 Subject: [PATCH 30/42] Stop sending usage data and discard the rest when closing has waited 5 seconds (#147) Closing the Usage Sharing Element waits up to 5 seconds for the data in its queue to be sent. When that time runs out, `managedResourcesCleanup` logs "Could not send final data on close down" and calls `shutdownNow` on the element's executor ([`ShareUsageBase` lines 753 to 755 at rc b33daa0f](https://github.com/51Degrees/51degrees-java/blob/b33daa0f32964dbf5b5f77ba5e947ef1ac75c55d/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java#L753-L755)). `shutdownNow` only interrupts the sending thread. A send that is waiting on the network carries on, and the thread then goes back to its loop, which keeps sending while the element is stopping and the queue is not empty ([`ShareUsageElement` line 489](https://github.com/51Degrees/51degrees-java/blob/b33daa0f32964dbf5b5f77ba5e947ef1ac75c55d/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java#L489)). So after `close()` has returned, the element can go on sending everything that was queued, for as long as the remote service takes to answer each message, and nothing says how much was left. --- .../fiftyone/flowelements/ShareUsageBase.java | 27 ++- .../flowelements/ShareUsageElement.java | 7 +- .../ShareUsageCloseTimeoutTests.java | 197 ++++++++++++++++++ 3 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index a0da6bced..48d28f657 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -240,6 +240,12 @@ public abstract class ShareUsageBase */ private volatile boolean stopping = false; + /** + * Set when closing has waited as long as it may for the queue to be + * sent. The data left is discarded and a consumer sends nothing more. + */ + private volatile boolean discarding = false; + /** * Keep a count of data lost */ @@ -751,6 +757,15 @@ protected boolean isStopping() { return stopping; } + /** + * Returns true once closing has stopped waiting for the queue to be + * sent. A consumer that sees this sends nothing more. + * @return true if the data left is being discarded + */ + protected boolean isDiscarding() { + return discarding; + } + /** * Returns true if there is a thread attempting to send usage data. * @return true if data is being sent @@ -795,7 +810,17 @@ protected void managedResourcesCleanup() { try { executor.shutdown(); if (!executor.awaitTermination(Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT, TimeUnit.MILLISECONDS)){ - logger.warn("Could not send final data on close down"); + // Sending stops rather than hold up the process that closes + // the element, and the data left is discarded. + discarding = true; + int discarded = + evidenceCollection.drainTo(new ArrayList()); + logger.warn( + "{} usage sharing entries were not sent, as sending had " + + "not finished {} milliseconds after usage sharing " + + "started closing.", + discarded, + Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT); executor.shutdownNow(); } } catch (InterruptedException interruptedException) { diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index 26809fd20..bc40513d0 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -467,7 +467,7 @@ protected void sendUsageData() { // final consumer but the consumer that was already running has // drained everything - and there is no point posting a document // with no entries in it. - if (allData.isEmpty() == false) { + if (allData.isEmpty() == false && isDiscarding() == false) { boolean accepted = false; try { sendAsXML(allData); @@ -485,8 +485,9 @@ protected void sendUsageData() { logger.error("Exception sending usage data", exception); } // send in minEntries batches unless closing in which case drain the queue - } while (evidenceCollection.size() >= minEntriesPerMessage || - isStopping() && evidenceCollection.size() > 0); + } while (isDiscarding() == false && + (evidenceCollection.size() >= minEntriesPerMessage || + isStopping() && evidenceCollection.size() > 0)); logger.debug(threadMarker, "Stopping sending. Queue size is {}", evidenceCollection.size()); } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java new file mode 100644 index 000000000..cf6507034 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java @@ -0,0 +1,197 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Closing waits up to 5 seconds for the data waiting to be sent. If it has + * not all been sent by then, sending stops, the rest is discarded and a + * warning gives the number discarded. See Cleanup in the usage sharing + * element specification. + */ +public class ShareUsageCloseTimeoutTests { + + /** + * The longest the test waits for something it expects to happen. + */ + private static final long WAIT_LIMIT_MILLIS = 30000; + + /** + * Entries processed, enough for several messages of at most two. + */ + private static final int ENTRIES = 10; + + /** + * An upload whose first answer does not come until the test releases + * it, as with a remote service that has stopped answering. Interrupting + * the waiting thread does not end the wait, as it would not end a + * stalled connection. + */ + private static class StalledUploader implements DataUploader { + private final CountDownLatch sending = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + private final AtomicInteger sends = new AtomicInteger(); + + @Override + public OutputStream getOutputStream() { + return new ByteArrayOutputStream(); + } + + @Override + public int getResponseCode() { + sends.incrementAndGet(); + sending.countDown(); + while (release.getCount() > 0) { + try { + release.await(); + } catch (InterruptedException ignored) { + // Keeps waiting, as a stalled connection would. + } + } + return 200; + } + + @Override + public String getResponseContent() { + return ""; + } + } + + @Test + public void ShareUsageElement_CloseTimesOut_StopsSendingAndDiscards() + throws Exception { + List warnings = new CopyOnWriteArrayList<>(); + StalledUploader uploader = new StalledUploader(); + Pipeline pipeline = new PipelineBuilder().build(); + ShareUsageElement element = new ShareUsageElement( + recordingLogger(warnings), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + flowData -> true); + element.addPipeline(pipeline); + element.dataUploader = uploader; + try { + for (int i = 1; i <= ENTRIES; i++) { + process(pipeline, element, "10.0.0." + i); + } + assertTrue( + uploader.sending.await(WAIT_LIMIT_MILLIS, TimeUnit.MILLISECONDS), + "No send was started."); + + element.close(); + uploader.release.countDown(); + + assertTrue( + element.executor.awaitTermination( + WAIT_LIMIT_MILLIS, + TimeUnit.MILLISECONDS), + "Sending did not stop after closing."); + assertEquals(1, uploader.sends.get(), + "Data was sent after closing gave up waiting."); + assertTrue( + warnings.stream().anyMatch(warning -> + warning.contains("were not sent") && + warning.matches(".*\\b[1-9][0-9]*\\b.*")), + "Discarding the data left was not logged with its count. " + + "Warnings: " + warnings); + } + finally { + uploader.release.countDown(); + if (element.executor.isShutdown() == false) { + element.close(); + } + pipeline.close(); + } + } + + private static void process( + Pipeline pipeline, + ShareUsageElement element, + String ip) throws Exception { + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_CLIENTIP_KEY, ip); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + element.process(data); + } + } + + /** + * A logger that records the warnings it is given. + */ + private static Logger recordingLogger(List warnings) { + return (Logger) Proxy.newProxyInstance( + Logger.class.getClassLoader(), + new Class[] {Logger.class}, + (proxy, method, args) -> { + if (method.getName().equals("warn")) { + warnings.add(Arrays.deepToString(args)); + return null; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == String.class) { + return "test"; + } + return null; + }); + } +} From ebaf75bb099e1247ec20c716f3e5e3fae8ccaed5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 00:05:39 +0000 Subject: [PATCH 31/42] Say in AGENTS.md that a test failing on timing is fixed (#122) Tests that fail on a time limit, a performance threshold or a timeout, and pass when run again, keep coming back at random, and each time someone has to look into the failure before running it again. [`AGENTS.md`](https://github.com/51Degrees/51degrees-java/blob/30e4886708974308d4b70e1804424f55560824a7/AGENTS.md) says nothing about them, so a contributor who meets one runs it again. --- AGENTS.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index fa77ffdfc..c5ab041b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -316,3 +316,11 @@ into `main` from any other branch. Plain words, no em dashes, sentence case headings. Nothing in this public repository names a private repository or an issue in one. + +## Tests that fail on timing + +A test that fails on a time limit, a performance threshold or a timeout is +fixed, not run again until it passes. Either the limit is set to fit the +slowest runner the test runs on, with each runner's measurement behind it, or +the reason it takes too long is found and fixed. A failure that comes and goes +costs everyone who meets it the time to look into it. From 20f9089efce35f540cc3015d21cb4384b165499f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 04:43:31 +0000 Subject: [PATCH 32/42] Specification conformance, 51degrees-java, section 15 (#175) The Java packages differed from seven SHOULD sentences of the specification. --- .../DeviceDetectionCloudEngine.java | 15 +- .../devicedetection/cloud/TestsBase.java | 2 +- .../devicedetection/cloud/WrapperCloud.java | 4 +- .../flowelements/CloudRequestFailedTests.java | 185 +++++++++++ .../flowelements/DeviceDataHashDefault.java | 5 + .../hash/engine/onpremise/TestsBase.java | 6 +- .../onpremise/flowelements/EngineTests.java | 6 +- .../flowelements/PropertyIndexTests.java | 4 +- .../flowelements/ReadByPositionTests.java | 124 ++++++++ .../shared/testhelpers/FileUtils.java | 8 + .../shared/testhelpers/KeyUtils.java | 5 + .../shared/PublishedPackageTests.java | 101 +++++++ .../shared/TestHelpersStayPublishedTests.java | 62 ++++ .../shared/testhelpers/FileUtilsTest.java | 2 +- .../shared/testhelpers/KeyUtilsTest.java | 4 +- .../shared/testhelpers/TestFileUtils.java | 169 +++++++++++ .../shared/testhelpers/TestKeyUtils.java | 63 ++++ ...DetectionOnPremisePipelineBuilderTest.java | 4 +- .../devicedetection/DeviceDetectionTests.java | 6 +- examples/device-detection/README.md | 1 + .../examples/console/ManualDataUpdate.java | 204 +++++++++++++ .../examples/console/MatchMetrics.java | 2 +- .../console/PerformanceBenchmark.java | 2 +- .../examples/console/TacCloud.java | 3 +- .../examples/console/UpdateDataFile.java | 10 +- .../console/ExamplesWithoutSettingsTest.java | 8 + .../console/GettingStartedOnPremTest.java | 8 +- .../console/ManualDataUpdateTest.java | 70 +++++ .../examples/console/MetadataOnPremTest.java | 8 +- .../console/OfflineProcessingTest.java | 10 +- .../examples/console/UpdateDataFileTest.java | 6 +- .../examples/shared/DataFileHelper.java | 44 ++- .../examples/shared/KeyHelper.java | 52 +++- .../web/getting-started.onprem/pom.xml | 7 + .../web/GettingStartedWebOnPremTest.java | 8 +- .../examples/console/CompareOnPrem.java | 4 +- .../examples/console/GettingStartedMixed.java | 4 + .../console/GettingStartedOnPrem.java | 4 + .../examples/console/MetricsOnPrem.java | 4 +- .../examples/console/OfflineProcessing.java | 5 +- .../console/PerformanceBenchmark.java | 4 +- .../examples/console/SuspiciousOnPrem.java | 4 +- .../examples/console/UpdateDataFile.java | 4 + .../console/UsageSharingCoverageTest.java | 166 ++++++++++ .../IPIntelligenceCloudEngine.java | 15 +- .../flowelements/CloudRequestFailedTests.java | 161 ++++++++++ .../IPIntelligenceDataHashDefault.java | 5 + .../flowelements/ReadByPositionTests.java | 86 ++++++ .../flowelements/GeoLocationCloudEngine.java | 12 + .../GeoLocationCloudRequestFailedTests.java | 163 ++++++++++ .../flowelements/CloudAspectEngineBase.java | 54 +++- .../CloudRequestFailedReasonTests.java | 271 +++++++++++++++++ .../fiftyone/pipeline/core/data/FlowData.java | 61 +++- .../core/flowelements/FlowDataDefault.java | 2 +- .../flowelements/FlowDataThrowsTests.java | 187 ++++++++++++ .../fiftyone/pipeline/engines/Constants.java | 40 ++- .../pipeline/engines/data/AspectDataBase.java | 23 +- .../flowelements/CloudAspectEngine.java | 9 + .../services/MissingPropertyReason.java | 5 + .../services/MissingPropertyService.java | 27 ++ .../MissingPropertyServiceDefault.java | 141 ++++++--- .../services/MissingPropertyMessageTests.java | 211 +++++++++++++ .../services/MissingPropertyReasonTests.java | 286 ++++++++++++++++++ .../services/MissingPropertyServiceTests.java | 8 +- 64 files changed, 3049 insertions(+), 135 deletions(-) create mode 100644 device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailedTests.java create mode 100644 device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadByPositionTests.java create mode 100644 device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/PublishedPackageTests.java create mode 100644 device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/TestHelpersStayPublishedTests.java create mode 100644 device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestFileUtils.java create mode 100644 device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestKeyUtils.java create mode 100644 examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java create mode 100644 examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java create mode 100644 examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UsageSharingCoverageTest.java create mode 100644 ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ReadByPositionTests.java create mode 100644 location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudRequestFailedTests.java create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestFailedReasonTests.java create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataThrowsTests.java create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyMessageTests.java create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyReasonTests.java diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java index 128e4ebc2..f2e429217 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java @@ -136,8 +136,19 @@ protected void processEngine(FlowData data, DeviceDataCloud aspectData) { else { CloudRequestData requestData = data.getFromElement(cloudRequestEngine); - String json = ""; - json = requestData.getJsonResponse(); + setCloudRequestFailed(data, aspectData, cloudRequestEngine); + String json = requestData.getJsonResponse(); + + // A failed request can leave no response, which the Cloud + // Request Engine has reported. + if (json == null || json.isEmpty()) { + logger.warn("The " + getClass().getSimpleName() + " did " + + "not process as the JSON response from the " + + "CloudRequestEngine was null or empty. Please refer to " + + "errors generated by the CloudRequestEngine in the logs " + + "as this indicates an error occurred there."); + return; + } // Extract data from json to the aspectData instance. JSONObject jsonObj = new JSONObject(json); diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java index 44d233d6a..f307376c2 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java @@ -25,7 +25,7 @@ import fiftyone.devicedetection.shared.testhelpers.UserAgentGenerator; import fiftyone.pipeline.util.FileFinder; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.UA_FILE_PATH; public class TestsBase { diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/WrapperCloud.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/WrapperCloud.java index 420d074c9..109d393ad 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/WrapperCloud.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/WrapperCloud.java @@ -24,7 +24,7 @@ import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngine; import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngineBuilder; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; +import fiftyone.devicedetection.shared.testhelpers.TestKeyUtils; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngineBuilder; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -47,7 +47,7 @@ public class WrapperCloud implements Closeable { public WrapperCloud() throws Exception { cloudRequestEngine = new CloudRequestEngineBuilder(loggerFactory) - .setResourceKey(KeyUtils.getNamedKey("TestResourceKey")) + .setResourceKey(TestKeyUtils.getNamedKey("TestResourceKey")) .build(); deviceDetectionCloudEngine = new DeviceDetectionCloudEngineBuilder(loggerFactory) .build(); diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailedTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailedTests.java new file mode 100644 index 000000000..5616e6187 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailedTests.java @@ -0,0 +1,185 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * When the Cloud Request Engine's request for the Flow Data failed, a + * property of the Device Detection cloud engine is missing with the + * CloudRequestFailed reason, rather than a reason about the resource key. + */ +public class CloudRequestFailedTests { + + private static final String CLOUD_REQUEST_FAILED = + "Property 'hardwarevendor' not found in data for element 'device'. " + + "This is because an upstream cloud request failed, so no data was " + + "available to populate the property."; + + private ILoggerFactory loggerFactory; + + private CloudRequestEngine requestEngine; + + @Before + public void init() { + Logger logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void RequestFailedWithNoResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed(null); + } + + @Test + public void RequestFailedWithAnErrorResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed("{ \"errors\": [ \"an error\" ] }"); + } + + @Test + public void RequestSucceeded_ReasonIsAboutTheResourceKey() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process( + engine, + "{ \"device\": { \"ismobile\": true } }", + Collections.emptyList()); + assertMissing( + "PropertyNotAccessibleWithResourceKey", + null, + data); + } + } + + private void assertCloudRequestFailed(String response) throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process( + engine, + response, + Collections.singletonList(new FlowError.Default( + new Exception("The cloud request failed."), + requestEngine))); + assertMissing("CloudRequestFailed", CLOUD_REQUEST_FAILED, data); + } + } + + private static void assertMissing( + String reason, + String message, + DeviceDataCloud data) { + try { + data.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals(e.getMessage(), reason, e.getReason().name()); + if (message != null) { + assertEquals(message, e.getMessage()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the device product, + * holding IsMobile alone. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private DeviceDetectionCloudEngine buildEngine() throws Exception { + DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("ismobile", "Boolean", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put("device", product); + + requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + private DeviceDataCloud process( + DeviceDetectionCloudEngine engine, + String response, + Collection errors) throws Exception { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + when(cloudData.getProcessStarted()).thenReturn(true); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getErrors()).thenReturn(errors); + + DeviceDataCloud device = new DeviceDataCloudInternal( + loggerFactory.getLogger(DeviceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + try { + engine.processEngine(flowData, device); + } catch (RuntimeException e) { + // A failed request's response may not be readable. The + // property is read afterwards either way, as it is through a + // Pipeline that suppresses process exceptions. + } + return device; + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java index 691761125..dbd7c4fb5 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java @@ -259,6 +259,11 @@ private AspectPropertyValue> getUserAgentsInternal() { @Override protected boolean propertyIsAvailable(String propertyName) { checkState(); + // A property the engine has a position for is held by its results, + // so that needs no lookup of the name in native code. + if (indexOf(propertyName) != null) { + return resultsList.isEmpty() == false; + } for (ResultsHashSwig results : resultsList) { if (results.containsProperty(Swig.asBytes(propertyName))) { return true; diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java index f3826ee0e..25a36a709 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java @@ -23,10 +23,10 @@ package fiftyone.devicedetection.hash.engine.onpremise; import fiftyone.devicedetection.shared.testhelpers.UserAgentGenerator; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import fiftyone.pipeline.engines.Constants; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.UA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; public class TestsBase { @@ -44,7 +44,7 @@ protected UserAgentGenerator getUserAgents() { protected void testInitialize(Constants.PerformanceProfiles profile) throws Exception { wrapper = new WrapperHash( - FileUtils.getHashFile(), + TestFileUtils.getHashFile(), profile); userAgents = new UserAgentGenerator( getFilePath(UA_FILE_PATH)); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java index 198e43bb2..c3c0ae5f8 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java @@ -34,11 +34,11 @@ import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.TAC_HASH_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; public class EngineTests { diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java index 0302a3a95..a58c909cc 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java @@ -53,8 +53,8 @@ import java.util.Map; import java.util.Set; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.TAC_HASH_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadByPositionTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadByPositionTests.java new file mode 100644 index 000000000..f26a0a81e --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadByPositionTests.java @@ -0,0 +1,124 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.TestsBase; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.BoolValueSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ResultsHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.StringValueSwig; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.*; + +/** + * A read of a property the engine has a position for goes by that + * position, and asks the native results nothing by name, not even whether + * they hold the property. + */ +public class ReadByPositionTests extends TestsBase { + + private static final Logger logger = mock(Logger.class); + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.HighPerformance); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void BooleanRead_ByPosition_NoLookupByName() throws Exception { + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + Integer index = engine.getPropertyIndex("IsMobile"); + assertNotNull(index); + try (FlowData flowData = + getWrapper().getPipeline().createFlowData()) { + ResultsHashSwig results = results(); + BoolValueSwig value = mock(BoolValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn(true); + when(results.getValueAsBool(index.intValue())).thenReturn(value); + DeviceDataHashDefault data = data(flowData, engine, results); + + assertEquals(Boolean.TRUE, data.getIsMobile().getValue()); + + verify(results).getValueAsBool(index.intValue()); + verify(results, never()).containsProperty(any(byte[].class)); + } + } + + @Test + public void StringRead_ByPosition_NoLookupByName() throws Exception { + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + Integer index = engine.getPropertyIndex("BrowserName"); + assertNotNull(index); + try (FlowData flowData = + getWrapper().getPipeline().createFlowData()) { + ResultsHashSwig results = results(); + StringValueSwig value = mock(StringValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn("Safari"); + when(results.getValueAsString(index.intValue())).thenReturn(value); + DeviceDataHashDefault data = data(flowData, engine, results); + + assertEquals("Safari", data.getBrowserName().getValue()); + + verify(results).getValueAsString(index.intValue()); + verify(results, never()).containsProperty(any(byte[].class)); + } + } + + /** + * Native results that say they hold every property, so that a read + * which asks by name still reaches the value. + */ + private static ResultsHashSwig results() { + ResultsHashSwig results = mock(ResultsHashSwig.class); + when(results.containsProperty(any(byte[].class))).thenReturn(true); + return results; + } + + private static DeviceDataHashDefault data( + FlowData flowData, + DeviceDetectionHashEngine engine, + ResultsHashSwig results) { + DeviceDataHashDefault data = new DeviceDataHashDefault( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + data.setResults(results); + return data; + } +} diff --git a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java index fe9e13a9e..aa5b01ddf 100644 --- a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java +++ b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java @@ -36,6 +36,14 @@ import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +/** + * Finds the data and evidence files the tests use. + * @deprecated These are test helpers, which the published package should + * not hold. Example code takes the data file names from + * {@code fiftyone.devicedetection.examples.shared.DataFileHelper}, in the + * device-detection-java-examples.shared module. + */ +@Deprecated @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class FileUtils { public static final String ENTERPRISE_HASH_DATA_FILE_NAME = "Enterprise-HashV41.hash"; diff --git a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/KeyUtils.java b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/KeyUtils.java index fdc8d2d6d..86f8f922e 100644 --- a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/KeyUtils.java +++ b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/KeyUtils.java @@ -27,7 +27,12 @@ /** * Helpers to obtain keys from the environment + * @deprecated These are test helpers, which the published package should + * not hold. Example code reads and checks keys with + * {@code fiftyone.devicedetection.examples.shared.KeyHelper}, in the + * device-detection-java-examples.shared module. */ +@Deprecated public class KeyUtils { /** * Obtain a key either from environment variable or from a property. diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/PublishedPackageTests.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/PublishedPackageTests.java new file mode 100644 index 000000000..6412f2651 --- /dev/null +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/PublishedPackageTests.java @@ -0,0 +1,101 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.shared; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertTrue; + +/** + * The published Device Detection packages hold no tests or test helpers, + * other than deprecated test helpers that earlier releases published, which + * stay because a patch release removes no public name. Each module's main + * sources are what its published jar holds, so they are read for a test + * helper package or a test class. + */ +public class PublishedPackageTests { + + /** + * The published Device Detection modules, which sit beside this one. + */ + private static final String[] MODULES = { + "device-detection.shared", + "device-detection.cloud", + "device-detection.hash.engine.on-premise", + "device-detection" + }; + + @Test + public void publishedModules_HoldNoTestCode() throws IOException { + Path root = Paths.get(System.getProperty("basedir", ".")) + .toAbsolutePath() + .getParent(); + List found = new ArrayList<>(); + for (String module : MODULES) { + Path sources = root.resolve(module) + .resolve("src").resolve("main").resolve("java"); + assertTrue(sources + " should hold the module's sources", + Files.isDirectory(sources)); + List files; + try (Stream walk = Files.walk(sources)) { + files = walk + .filter(p -> p.toString().endsWith(".java")) + .map(p -> sources.relativize(p).toString() + .replace('\\', '/')) + .collect(Collectors.toList()); + } + for (String file : files) { + if ((file.contains("/testhelpers/") && + isDeprecated(file) == false) || + file.matches(".*Tests?\\.java")) { + found.add(module + ": " + file); + } + } + } + assertTrue("These published sources are tests or test helpers: " + + found, found.isEmpty()); + } + + /** + * Whether the class a source file holds is deprecated, which is false + * for a class this module cannot load. + */ + private static boolean isDeprecated(String file) { + String name = file.substring(0, file.length() - ".java".length()) + .replace('/', '.'); + try { + return Class.forName(name).isAnnotationPresent(Deprecated.class); + } catch (ClassNotFoundException e) { + return false; + } + } +} diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/TestHelpersStayPublishedTests.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/TestHelpersStayPublishedTests.java new file mode 100644 index 000000000..2eb8094e1 --- /dev/null +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/TestHelpersStayPublishedTests.java @@ -0,0 +1,62 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.shared; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * FileUtils and KeyUtils, which earlier releases published, stay in the + * published jar, deprecated, as a patch release removes no public name. + * They are found by name, because the build treats the use of a + * deprecated class as an error. + */ +public class TestHelpersStayPublishedTests { + + private static final String PACKAGE = + "fiftyone.devicedetection.shared.testhelpers."; + + @Test + public void FileUtils_InTheMainJar_Deprecated() throws Exception { + assertPublishedAndDeprecated(PACKAGE + "FileUtils"); + } + + @Test + public void KeyUtils_InTheMainJar_Deprecated() throws Exception { + assertPublishedAndDeprecated(PACKAGE + "KeyUtils"); + } + + private static void assertPublishedAndDeprecated(String name) + throws ClassNotFoundException { + Class type = Class.forName(name); + String location = type.getProtectionDomain() + .getCodeSource().getLocation().toString(); + assertFalse(name + " comes from " + location + ", not the main jar", + location.contains("test-classes") || + location.endsWith("-tests.jar")); + assertTrue(name + " should be deprecated", + type.isAnnotationPresent(Deprecated.class)); + } +} diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/FileUtilsTest.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/FileUtilsTest.java index d6cef9ea9..16a3d9845 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/FileUtilsTest.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/FileUtilsTest.java @@ -34,7 +34,7 @@ public class FileUtilsTest { - Logger logger = LoggerFactory.getLogger(FileUtils.class); + Logger logger = LoggerFactory.getLogger(TestFileUtils.class); /** * location of "." is the same as user.dir diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/KeyUtilsTest.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/KeyUtilsTest.java index ce847992b..1b9f01bfe 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/KeyUtilsTest.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/KeyUtilsTest.java @@ -29,8 +29,8 @@ public class KeyUtilsTest { @Test public void testGetEnv(){ - assertNotNull(KeyUtils.getNamedKey("path")); - System.out.println(KeyUtils.getNamedKey("path")); + assertNotNull(TestKeyUtils.getNamedKey("path")); + System.out.println(TestKeyUtils.getNamedKey("path")); } } \ No newline at end of file diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestFileUtils.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestFileUtils.java new file mode 100644 index 000000000..63a51de8a --- /dev/null +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestFileUtils.java @@ -0,0 +1,169 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.shared.testhelpers; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +/** + * Finds the data and evidence files the tests use. + */ +@SuppressWarnings("OptionalUsedAsFieldOrParameterType") +public class TestFileUtils { + public static final String ENTERPRISE_HASH_DATA_FILE_NAME = "Enterprise-HashV41.hash"; + + public static final String TAC_HASH_DATA_FILE_NAME = "TAC-HashV41.hash"; + public static final String LITE_HASH_DATA_FILE_NAME = "51Degrees-LiteV4.1.hash"; + private static Optional HASH_DATA_FILE; + public static final String UA_FILE_NAME = "20000 User Agents.csv"; + private static File UA_FILE; + public static final String EVIDENCE_FILE_NAME = "20000 Evidence Records.yml"; + private static Optional EVIDENCE_FILE; + // Where each free file is in the data repository. A search for the path + // rather than the name does not stop at another file of the same name, + // such as a Git LFS pointer in an old device-detection-data folder. + public static final String LITE_HASH_DATA_FILE_PATH = + "device-detection/" + LITE_HASH_DATA_FILE_NAME; + public static final String UA_FILE_PATH = "device-detection/" + UA_FILE_NAME; + public static final String EVIDENCE_FILE_PATH = + "device-detection/" + EVIDENCE_FILE_NAME; + + /** + * Helper to find the location of an Enterprise or Lite Hash file in the default search scope + * + * @return a file or null if not found + */ + public static String getHashFileName() { + return Objects.isNull(getHashFile()) ? null : getHashFile().getPath(); + } + + /** + * Helper to find the location of an Enterprise or Lite Hash file in the default search scope + * + * @return a file or empty if not found + */ + public static File getHashFile() { + try { + if (Objects.nonNull(HASH_DATA_FILE)){ + return HASH_DATA_FILE.orElse(null); + } + // following throws exception if not found + HASH_DATA_FILE = Optional.of(getFilePath(TAC_HASH_DATA_FILE_NAME)); + return HASH_DATA_FILE.get(); + } catch (Exception e) { + try { + // following throws exception if not found + HASH_DATA_FILE = Optional.of(getFilePath(ENTERPRISE_HASH_DATA_FILE_NAME)); + return HASH_DATA_FILE.get(); + } catch (Exception ex) { + try { + // following throws exception if not found + HASH_DATA_FILE = Optional.of(getFilePath(LITE_HASH_DATA_FILE_PATH)); + return HASH_DATA_FILE.get(); + } catch (Exception exc) { + HASH_DATA_FILE = Optional.empty(); + return null; + } + } + } + } + + public static File getEvidenceFile() { + if (Objects.nonNull(EVIDENCE_FILE)) { + return EVIDENCE_FILE.orElse(null); + } + try { + EVIDENCE_FILE = Optional.of(getFilePath(EVIDENCE_FILE_PATH)); + return EVIDENCE_FILE.get(); + } catch (Exception e) { + EVIDENCE_FILE = Optional.empty(); + return null; + } + } + + /** + * Prefix for temp files that are created by {@link #jarFileHelper(String)} + **/ + public static String TEMP_FILE_PREFIX = "DDTempFile"; + + /** + * Search the classpath for a resource. If it doesn't exist throw an exception. + * If the file is in a jar then copy it to a temp file, so it can be used as an actual file. + * + * Callers might wish to delete the temp file created when one is created and + * can assess whether this is the case as the name of the filename created will + * start with {@link #TEMP_FILE_PREFIX} which they can alter to suit their needs. + * + * Files created here have the property deleteOnExit; + * + * @param file a filename + * @return a File representing the resource + */ + public static File jarFileHelper(String file) { + // find the resource in the classpath + URL resource = Objects.requireNonNull(TestFileUtils.class.getClassLoader().getResource(file), + file + " could not be found on the classpath"); + try { + return new File(resource.toURI()); + } catch (Exception e) { + if (!resource.getPath().contains(".jar!")) + throw new IllegalStateException(resource.getPath() + " can't be loaded", e); + } + // make a copy of the file if it is in a jar file (assumption is that if the + // name matches the pattern then it is in a jar, if not, no harm is done + // but an irrelevant tmp file may be created) + String filePath; + try { + filePath = URLDecoder.decode(resource.getPath(), "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + // we will try to match the extension + int index = filePath.lastIndexOf("."); + String extension = index < 0 ? "tmp" : filePath.substring(index); + try { + Path temp = Files.createTempFile(TEMP_FILE_PREFIX, extension); + try (InputStream is = TestFileUtils.class.getClassLoader().getResourceAsStream(file)) { + //noinspection ConstantConditions + Files.copy(is, temp, REPLACE_EXISTING); + } + filePath = temp.toAbsolutePath().toString(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + File f = new File(filePath); + f.deleteOnExit(); + return f; + } +} \ No newline at end of file diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestKeyUtils.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestKeyUtils.java new file mode 100644 index 000000000..678cdcfc5 --- /dev/null +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestKeyUtils.java @@ -0,0 +1,63 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.shared.testhelpers; + +import java.util.Base64; +import java.util.Objects; + +/** + * Helpers to obtain keys from the environment + */ +public class TestKeyUtils { + /** + * Obtain a key either from environment variable or from a property. + *

+ * Try resource key as env var, then as upper case env var, the system property + */ + public static String getNamedKey(String keyName) { + String resourceKey = System.getenv(keyName); + if (Objects.isNull(resourceKey)) { + resourceKey = System.getenv(keyName.toUpperCase()); + if (Objects.isNull(resourceKey)) { + resourceKey = System.getProperty(keyName); + } + } + return resourceKey; + } + + /** + * Evaluate whether a key might be valid + * @param keyValue value to test + * @return boolean + */ + public static boolean isInvalidKey(String keyValue){ + Base64.Decoder decoder = Base64.getUrlDecoder(); + try { + return Objects.isNull(keyValue) || + keyValue.trim().length() < 19 || + decoder.decode(keyValue).length < 14; + } catch (IllegalArgumentException e) { + return true; + } + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java index aed639665..24d8fbd2d 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java @@ -24,7 +24,7 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ConfigHashSwig; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.configuration.DataFileConfiguration; @@ -51,7 +51,7 @@ @SuppressWarnings({"rawtypes", "deprecation"}) public class DeviceDetectionOnPremisePipelineBuilderTest { - private static final String HASH_DATA_FILE_NAME = FileUtils.getHashFileName(); + private static final String HASH_DATA_FILE_NAME = TestFileUtils.getHashFileName(); private DeviceDetectionOnPremisePipelineBuilder builder; private File datafile = FileFinder.getFilePath(HASH_DATA_FILE_NAME); private ILoggerFactory loggerFactory; diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java index a59a0a6d6..8a9fe3159 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java @@ -25,7 +25,7 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.devicedetection.shared.testhelpers.UserAgentGenerator; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.FlowError; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -52,7 +52,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.UA_FILE_PATH; import static fiftyone.pipeline.core.Constants.EVIDENCE_HTTPHEADER_PREFIX; import static fiftyone.pipeline.core.Constants.EVIDENCE_SEPERATOR; import static fiftyone.pipeline.engines.Constants.PerformanceProfiles.*; @@ -68,7 +68,7 @@ public class DeviceDetectionTests { private static final Logger logger = LoggerFactory.getLogger(DeviceDetectionTests.class); - private static final String HASH_DATA_FILE_NAME = FileUtils.getHashFileName(); + private static final String HASH_DATA_FILE_NAME = TestFileUtils.getHashFileName(); private static UserAgentGenerator userAgents; TestConfig[] hashConfigs = { diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 5e1f575ba..bdbf5889d 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -94,6 +94,7 @@ The tables below describe the examples available in this repository. | OfflineProcessing | Example showing how to ingest a file containing data from web requests and perform detection against the entries. | | PerformanceBenchmark | How to configure the various performance options and run some simple performance tests. | | UpdateOnStartup | How to configure the Pipeline to automatically update the device detection data file on startup. Also illustrates 'file watcher'. This will refresh the device detection engine if the specified data file is updated on disk. | +| ManualDataUpdate | How to make the device detection engine load a new data file when the application decides to, rather than through an automatic update, with no restart. | ## Running built examples from command line diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java new file mode 100644 index 000000000..68a9447e7 --- /dev/null +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java @@ -0,0 +1,204 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; +import fiftyone.devicedetection.examples.shared.DataFileHelper; +import fiftyone.devicedetection.examples.shared.FindOutMore; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.OutputStream; +import java.io.PrintWriter; +import java.text.SimpleDateFormat; + +import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; +import static fiftyone.devicedetection.examples.shared.PropertyHelper.asString; +import static fiftyone.pipeline.util.FileFinder.getFilePath; + +/** + * Shows how to make the on-premise device detection engine load a new data + * file when the application decides to, rather than through an automatic + * update. The engine goes on detecting while the file loads, so the + * application does not need to restart. + *

+ * In practice a newer data file is copied over the one the engine was built + * with, for example by a scheduled task, and then refreshData is called. + * This example has no newer file, so it reloads the file it started with + * and the results stay the same. refreshData can also be given the + * contents of a data file held in memory. + */ +public class ManualDataUpdate { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-manualdataupdate.java"; + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-manualdataupdate.java&utm_term=enterprise-data-file"; + + private static final String DESKTOP_USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"; + + private static final String IPHONE_USER_AGENT = + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) " + + "AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114"; + + private static final Logger logger = + LoggerFactory.getLogger(ManualDataUpdate.class); + + public static void main(String[] args) throws Exception { + configureLogback(getFilePath("logback.xml")); + // The free Lite data file is used unless another is given. + String dataFile = args.length > 0 ? + args[0] : GettingStartedOnPrem.LITE_V_4_1_HASH; + run(dataFile, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); + } + + /** + * Run the example. + * @param dataFile a device detection data file + * @param outputStream somewhere for the results + */ + public static void run(String dataFile, OutputStream outputStream) + throws Exception { + String dataFileLocation; + try { + dataFileLocation = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); + } catch (Exception e) { + DataFileHelper.cantFindDataFile(dataFile); + throw e; + } + PrintWriter writer = new PrintWriter(outputStream, true); + logger.info("Constructing pipeline from file " + dataFileLocation); + + // The data file changes only when this code says so, because + // automatic updates and the file system watcher are off. + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder() + // The engine works from a copy of the data file, so a newer + // file can be copied over the original while it runs. + .useOnPremise(dataFileLocation, true) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false) + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. + .setShareUsage(false) + .build()) { + DeviceDetectionHashEngine engine = + pipeline.getElement(DeviceDetectionHashEngine.class); + + boolean needsEnterprise = false; + needsEnterprise |= checkIfMobile(pipeline, engine, + DESKTOP_USER_AGENT, writer); + needsEnterprise |= checkIfMobile(pipeline, engine, + IPHONE_USER_AGENT, writer); + + writer.println("Reloading the data file published on " + + published(engine) + "."); + // Load the data file again, from the place the engine was built + // with. Call this once a newer data file has been copied there. + engine.refreshData(engine.getDataFileMetaData().getIdentifier()); + writer.println("Reloaded the data file published on " + + published(engine) + "."); + writer.println(); + + needsEnterprise |= checkIfMobile(pipeline, engine, + DESKTOP_USER_AGENT, writer); + needsEnterprise |= checkIfMobile(pipeline, engine, + IPHONE_USER_AGENT, writer); + + if (needsEnterprise) { + writer.println("This example reloaded the free Lite data " + + "file. The Enterprise data file adds daily automatic " + + "updates."); + DataFileHelper.writeEnterpriseNote(writer, CONTACT_US); + } + } + writer.flush(); + } + + /** + * Detect the device a User-Agent comes from and write whether it is a + * mobile device, and its hardware vendor, which only the Enterprise data + * file has. + * @return true if the hardware vendor needs the Enterprise data file + */ + private static boolean checkIfMobile(Pipeline pipeline, + DeviceDetectionHashEngine engine, + String userAgent, + PrintWriter writer) + throws Exception { + // The flow data is closed straight away, which releases the results + // of the detection. Results that are not released keep the data + // they were read from in memory, even after the data is refreshed. + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", userAgent); + data.process(); + DeviceData device = data.get(DeviceData.class); + writer.println("Is User-Agent '" + userAgent + + "' a mobile device? " + asString(device.getIsMobile())); + if (DataFileHelper.hasProperty(engine, "HardwareVendor")) { + writer.println("\tHardware Vendor: " + + asString(device.getHardwareVendor())); + writer.println(); + return false; + } + writer.println("\tHardware Vendor: " + + DataFileHelper.NEEDS_ENTERPRISE); + writer.println(); + return true; + } + } + + /** + * The date the engine's data file was published. + */ + private static String published(DeviceDetectionHashEngine engine) { + return new SimpleDateFormat("yyyy-MM-dd").format( + engine.getDataFileMetaData().getDataPublishedDateTime()); + } +} +/*! + * @example console/ManualDataUpdate.java + * This example shows how to make the on-premise device detection engine + * load a new data file when the application decides to, rather than through + * an automatic update. + *

+ * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java). + * @include{doc} example-require-datafile.txt + */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java index 3bcd93807..ae753978d 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java @@ -46,7 +46,7 @@ import java.util.Objects; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.examples.shared.DataFileHelper.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.util.stream.Collectors.groupingBy; diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java index cda12774b..3a527d479 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java @@ -47,7 +47,7 @@ import static fiftyone.devicedetection.examples.shared.DataFileHelper.getDataFileLocation; import static fiftyone.devicedetection.examples.shared.DataFileHelper.getEvidenceFile; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.examples.shared.DataFileHelper.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.engines.Constants.PerformanceProfiles.*; /** diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java index b3859fcac..ed3295b99 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java @@ -26,7 +26,6 @@ import fiftyone.devicedetection.cloud.data.MultiDeviceDataCloud; import fiftyone.devicedetection.shared.Constants; import fiftyone.devicedetection.shared.DeviceData; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; import fiftyone.pipeline.core.data.FlowData; @@ -87,7 +86,7 @@ public static void run(String resourceKey, OutputStream os) throws Exception { // use the one supplied to this method PipelineOptions pipelineOptions = PipelineOptionsFactory.getOptionsFromFile(optionsFile); String fileResourceKey = pipelineOptions.find("CloudRequestEngine", "ResourceKey"); - if (KeyUtils.isInvalidKey(fileResourceKey)) { + if (isInvalidKey(fileResourceKey)) { getOrSetSuperResourceKey(resourceKey, TAC_EXAMPLE_RESOURCE_KEY_NAME); } try (PrintWriter writer = new PrintWriter(os)) { diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index eae7ca9fd..a56b78bbb 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -25,7 +25,7 @@ import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; import fiftyone.devicedetection.examples.shared.DataFileHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; +import fiftyone.devicedetection.examples.shared.KeyHelper; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.services.DataUpdateService; import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; @@ -43,8 +43,8 @@ import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.devicedetection.examples.shared.DataFileHelper.getDatafileMetaData; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.ENTERPRISE_HASH_DATA_FILE_NAME; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.examples.shared.DataFileHelper.ENTERPRISE_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.examples.shared.DataFileHelper.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; /*! @@ -236,9 +236,9 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // try to find a license key if (Objects.isNull(licenseKey)) { - licenseKey = KeyUtils.getNamedKey(UPDATE_EXAMPLE_LICENSE_KEY_NAME); + licenseKey = KeyHelper.getNamedKey(UPDATE_EXAMPLE_LICENSE_KEY_NAME); } - if (Objects.isNull(licenseKey) || KeyUtils.isInvalidKey(licenseKey)) { + if (Objects.isNull(licenseKey) || KeyHelper.isInvalidKey(licenseKey)) { explainLicenseKey(dataFilename, new PrintWriter(System.out, true)); return; } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java index 00f21fca2..1ee1bf5b9 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java @@ -136,6 +136,14 @@ public void performance() throws Exception { assertOffer(runWithoutSettings(PerformanceBenchmark::main)); } + @Test + public void manualDataUpdate() throws Exception { + String output = runWithoutSettings(ManualDataUpdate::main); + assertContains(output, "Hardware Vendor: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "The Enterprise data file adds daily automatic updates."); + assertContains(output, CONTACT_US); + } + @Test public void updateDataFile() throws Exception { String output = runWithoutSettings(UpdateDataFile::main); diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java index 779a70fc2..4584988a4 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java @@ -23,16 +23,16 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.EvidenceHelper; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import org.junit.Test; public class GettingStartedOnPremTest { @Test public void gettingStartedOnPremTest() throws Exception { - GettingStartedOnPrem.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_PATH - : FileUtils.getHashFileName(), + GettingStartedOnPrem.run(TestFileUtils.getHashFileName() == null + ? TestFileUtils.LITE_HASH_DATA_FILE_PATH + : TestFileUtils.getHashFileName(), EvidenceHelper.setUpEvidence(), System.out); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java new file mode 100644 index 000000000..24cead133 --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Runs the example that reloads the data file when its code says so, + * rather than through an automatic update, and checks that it detects + * the same User-Agents before and after the reload. + */ +public class ManualDataUpdateTest { + + private static final String CHECK = "a mobile device?"; + + @Test + public void manualDataUpdate_ReloadsTheDataFile() throws Exception { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + ManualDataUpdate.run(TestFileUtils.getHashFileName() == null + ? TestFileUtils.LITE_HASH_DATA_FILE_PATH + : TestFileUtils.getHashFileName(), + buffer); + String output = buffer.toString("UTF-8"); + System.out.println(output); + + int reloading = output.indexOf("Reloading the data file"); + int reloaded = output.indexOf("Reloaded the data file"); + assertTrue("The example should say it reloads the data file", + reloading >= 0); + assertTrue("The example should say it reloaded the data file", + reloaded > reloading); + assertEquals(2, count(output.substring(0, reloading), CHECK)); + assertEquals(2, count(output.substring(reloaded), CHECK)); + } + + private static int count(String text, String part) { + int count = 0; + for (int i = text.indexOf(part); i >= 0; + i = text.indexOf(part, i + part.length())) { + count++; + } + return count; + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java index a128b1393..ffbc67a2d 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java @@ -22,14 +22,14 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import org.junit.Test; public class MetadataOnPremTest { @Test public void gettingMetaDataOnPremTest() throws Exception { - MetadataOnPrem.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_PATH - : FileUtils.getHashFileName(), System.out); + MetadataOnPrem.run(TestFileUtils.getHashFileName() == null + ? TestFileUtils.LITE_HASH_DATA_FILE_PATH + : TestFileUtils.getHashFileName(), System.out); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java index 4861f14d6..54605a354 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java @@ -22,7 +22,7 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import org.junit.Test; import java.io.FileInputStream; @@ -31,10 +31,10 @@ public class OfflineProcessingTest { @Test public void offlineProcessingTest() throws Exception { - OfflineProcessing.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_PATH - : FileUtils.getHashFileName(), - new FileInputStream(Objects.requireNonNull(FileUtils.getEvidenceFile())), + OfflineProcessing.run(TestFileUtils.getHashFileName() == null + ? TestFileUtils.LITE_HASH_DATA_FILE_PATH + : TestFileUtils.getHashFileName(), + new FileInputStream(Objects.requireNonNull(TestFileUtils.getEvidenceFile())), System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java index 301984e5a..1c37b8d11 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java @@ -22,7 +22,7 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; +import fiftyone.devicedetection.shared.testhelpers.TestKeyUtils; import org.junit.Test; import static fiftyone.devicedetection.examples.console.UpdateDataFile.run; @@ -33,9 +33,9 @@ public class UpdateDataFileTest { @Test public void updateExampleTest () throws Exception { - String licenseKey = KeyUtils.getNamedKey(UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME); + String licenseKey = TestKeyUtils.getNamedKey(UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME); assumeFalse("Skipping test, no license key found", - KeyUtils.isInvalidKey(licenseKey)); + TestKeyUtils.isInvalidKey(licenseKey)); try { run(null, licenseKey, false); } catch (IllegalStateException e) { diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java index 55a9c44c1..e2acf185d 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java @@ -24,7 +24,6 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.data.AspectEngineDataFile; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; @@ -75,6 +74,24 @@ public class DataFileHelper { */ public static final String NEEDS_ENTERPRISE = "needs the Enterprise data file"; + public static final String TAC_HASH_DATA_FILE_NAME = "TAC-HashV41.hash"; + + public static final String ENTERPRISE_HASH_DATA_FILE_NAME = "Enterprise-HashV41.hash"; + + /** + * Where the free Lite data file is in the data repository. A search for + * the path rather than the name does not stop at another file of the + * same name, such as a Git LFS pointer in an old device-detection-data + * folder. + */ + public static final String LITE_HASH_DATA_FILE_PATH = + "device-detection/51Degrees-LiteV4.1.hash"; + + /** + * Where the evidence the examples process is in the data repository. + */ + public static final String EVIDENCE_FILE_PATH = + "device-detection/20000 Evidence Records.yml"; public static class DatafileInfo { FiftyOneDataFile fileInfo; String tier; @@ -210,7 +227,7 @@ public static void writeEnterpriseOffer(PrintWriter output, String contactUs) { @SuppressWarnings("RedundantThrows") public static File getEvidenceFile(String evidenceFilename) throws Exception { if (Objects.isNull(evidenceFilename)) { - evidenceFilename = FileUtils.EVIDENCE_FILE_PATH; + evidenceFilename = EVIDENCE_FILE_PATH; } File evidenceFile; @@ -233,10 +250,10 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { @SuppressWarnings("RedundantThrows") public static String getDataFileLocation(String dataFilename) throws Exception { if (Objects.isNull(dataFilename)) { - dataFilename = FileUtils.getHashFileName(); + dataFilename = findHashFileName(); } if (Objects.isNull(dataFilename)) { - dataFilename = FileUtils.LITE_HASH_DATA_FILE_PATH; + dataFilename = LITE_HASH_DATA_FILE_PATH; } String dataFileLocation; try { @@ -248,6 +265,25 @@ public static String getDataFileLocation(String dataFilename) throws Exception { return dataFileLocation; } + /** + * The first Hash data file found below the project root, being the TAC + * file, then the Enterprise file, then the free Lite file. + * @return the path of the file, or null when there is none + */ + private static String findHashFileName() { + for (String file : new String[]{ + TAC_HASH_DATA_FILE_NAME, + ENTERPRISE_HASH_DATA_FILE_NAME, + LITE_HASH_DATA_FILE_PATH}) { + try { + return getFilePath(file).getPath(); + } catch (Exception e) { + // Not found, so try the next file. + } + } + return null; + } + /** * Finds a file below the project root, as * {@link FileFinder#getFilePath(String)} does, and then in the data diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java index 06ae82900..9f5b6729c 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java @@ -22,7 +22,6 @@ package fiftyone.devicedetection.examples.shared; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +32,47 @@ public class KeyHelper { public static final String TEST_RESOURCE_KEY = "TestResourceKey"; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); + /** + * The shortest a key can be, as text and once decoded. + */ + private static final int MIN_KEY_LENGTH = 19; + + private static final int MIN_DECODED_KEY_LENGTH = 14; + + /** + * Read a key as an environment variable, then as its upper case name, + * then as a system property. + * @param keyName the name of the key + * @return the key, or null when none is set + */ + public static String getNamedKey(String keyName) { + String key = System.getenv(keyName); + if (Objects.isNull(key)) { + key = System.getenv(keyName.toUpperCase()); + if (Objects.isNull(key)) { + key = System.getProperty(keyName); + } + } + return key; + } + + /** + * Whether a value cannot be a key, being missing, too short, or not URL + * safe base 64 of a long enough key. + * @param keyValue the value to check + * @return true if the value cannot be a key + */ + public static boolean isInvalidKey(String keyValue) { + try { + return Objects.isNull(keyValue) || + keyValue.trim().length() < MIN_KEY_LENGTH || + Base64.getUrlDecoder().decode(keyValue).length < + MIN_DECODED_KEY_LENGTH; + } catch (IllegalArgumentException e) { + return true; + } + } + /** * The names every cloud test reads a resource key from first, before * the names its own example reads. @@ -84,7 +124,7 @@ static String requireResourceKey(Function read, names.addAll(exampleNames); for (String name : names) { String value = read.apply(name); - if (KeyUtils.isInvalidKey(value) == false) { + if (isInvalidKey(value) == false) { return value.trim(); } } @@ -141,7 +181,7 @@ private static String readKey(String name) { System.getenv(name), System.getenv(name.toUpperCase()), System.getProperty(name)}) { - if (KeyUtils.isInvalidKey(value) == false) { + if (isInvalidKey(value) == false) { return value; } } @@ -178,10 +218,10 @@ public static String getOrSetResourceKey(String value, String variableName, String errorMessage, boolean shouldThrow) { if (Objects.isNull(value)) { - value = KeyUtils.getNamedKey(variableName); + value = getNamedKey(variableName); } - if (KeyUtils.isInvalidKey(value)) { + if (isInvalidKey(value)) { logger.error("\nTo access Cloud Services you must supply a " + "\"ResourceKey\" in one of the following ways: \n - in the " + "configuration file of an example,\n - as a command line parameter of a " + @@ -196,7 +236,7 @@ public static String getOrSetResourceKey(String value, String variableName, // Keep a valid key for next time. System.setProperty throws on null, // and an invalid key is not worth keeping. - if (KeyUtils.isInvalidKey(value) == false) { + if (isInvalidKey(value) == false) { System.setProperty(variableName, value); } return value; diff --git a/examples/device-detection/web/getting-started.onprem/pom.xml b/examples/device-detection/web/getting-started.onprem/pom.xml index 647fceabb..7f2b23d45 100644 --- a/examples/device-detection/web/getting-started.onprem/pom.xml +++ b/examples/device-detection/web/getting-started.onprem/pom.xml @@ -66,6 +66,13 @@ device-detection-java-examples.shared ${project.version} + + ${project.groupId} + device-detection.shared + ${device-detection.version} + test-jar + test + ch.qos.logback logback-classic diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java index 967773a14..e24c4a0c3 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java @@ -24,7 +24,7 @@ import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.examples.shared.FindOutMore; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -47,10 +47,10 @@ public class GettingStartedWebOnPremTest { public static void startJetty() throws Exception { // With no Enterprise file, the free Lite file is found, or // downloaded, as the example finds it, so a fresh clone runs this. - System.setProperty("TestDataFile", FileUtils.getHashFile() == null + System.setProperty("TestDataFile", TestFileUtils.getHashFile() == null ? DataFileHelper.findDataFile( - FileUtils.LITE_HASH_DATA_FILE_PATH).getAbsolutePath() - : FileUtils.getHashFile().getAbsolutePath()); + TestFileUtils.LITE_HASH_DATA_FILE_PATH).getAbsolutePath() + : TestFileUtils.getHashFile().getAbsolutePath()); SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index 1e4bae6ad..0c21646b5 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -286,7 +286,9 @@ public static void run(String dataFile, // for more detail on this and other configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=automatic-datafile-updates .setPerformanceProfile(profile) - // inhibit sharing usage for this example + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // inhibit auto-update of the data file for this example .setAutoUpdate(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index 771fde5e5..69cba6a13 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -156,6 +156,10 @@ public static void run(String ddDataFile, PipelineOptions pipelineOptions = PipelineOptionsFactory.getOptionsFromFile(optionsFile); try (Pipeline pipeline = new FiftyOnePipelineBuilder() + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. + .setShareUsage(false) .buildFromConfiguration(pipelineOptions)) { // The engines say which properties their data files have. diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index ca39cc6fe..54868dc60 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -156,6 +156,10 @@ public static void run(String dataFile, PipelineOptions pipelineOptions = PipelineOptionsFactory.getOptionsFromFile(optionsFile); // Build a new Pipeline from the configuration. try (Pipeline pipeline = new FiftyOnePipelineBuilder() + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. + .setShareUsage(false) .buildFromConfiguration(pipelineOptions)) { diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 19466fc9a..ecd247768 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -420,7 +420,9 @@ public static void run(String dataFile, // configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=automatic-datafile-updates .setPerformanceProfile(profile) - // inhibit sharing usage for this example + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // inhibit auto-update of the data file for this example .setAutoUpdate(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 87c67d177..79fe0f5dd 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -177,8 +177,9 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws // that the pipeline is disposed when done try (Pipeline pipeline = new IPIntelligencePipelineBuilder() .useOnPremise(dataFileLocation, false) - // inhibit sharing usage for this test, usually - // this should be set "true" + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // inhibit auto-update of the data file for this test .setAutoUpdate(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java index 6a535f868..44ac1c5fb 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java @@ -243,7 +243,9 @@ private void setPipelinePerformanceProperties( builder.setPerformanceProfile(config.profile) // set this to false for testing .setAutoUpdate(false) - // set this to false for testing + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // hint for cache concurrency .setConcurrency(numberOfThreads); diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index 5ad8a3865..fb84c3f1d 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -175,7 +175,9 @@ public static void run(String dataFile, // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=automatic-datafile-updates // https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=usage-sharing .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) - // inhibit sharing usage for this example, usually this should be set to "true" + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // inhibit auto-update of the data file for this example .setAutoUpdate(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java index a396c73c9..d82039833 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java @@ -303,6 +303,10 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // overwritten the pipeline must be configured to copy the supplied // file to a temporary file (createTempDataCopy parameter == true. .useOnPremise(dataFilename, true) + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. + .setShareUsage(false) // Use the low memory profile, because this example // downloads the Enterprise data file, which is over 6 GB, // and MaxPerformance would load all of it into memory. diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UsageSharingCoverageTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UsageSharingCoverageTest.java new file mode 100644 index 000000000..997a71517 --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UsageSharingCoverageTest.java @@ -0,0 +1,166 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertTrue; + +/** + * Checks that no console example adds a Usage Sharing Element to its + * Pipeline, and that each says beside the setting that turns it off that + * usage sharing should be on in production. The builders that add the + * element unless told not to are the 51Degrees Pipeline builder and the + * on-premise IP Intelligence Pipeline builder. + */ +public class UsageSharingCoverageTest { + + /** + * Each builder that adds a Usage Sharing Element unless it is turned + * off. + */ + private static final Pattern SHARING_BUILDER = Pattern.compile( + "new FiftyOnePipelineBuilder\\(|\\.useOnPremise\\("); + + private static final Pattern SHARING_OFF = Pattern.compile( + "\\.setShareUsage\\(false\\)"); + + /** + * The words the comment before each setting that turns usage sharing + * off must hold. + */ + private static final String PRODUCTION_COMMENT = + "In production, leave usage sharing on"; + + /** + * How many lines before the setting the comment may start. + */ + private static final int COMMENT_LINES = 6; + + @Test + public void everyExample_TurnsUsageSharingOff() throws IOException { + List examples = new ArrayList<>(); + List sharing = new ArrayList<>(); + for (Path file : exampleSources()) { + String text = code(read(file)); + examples.add(file.getFileName().toString()); + int builders = count(SHARING_BUILDER, text); + int off = count(SHARING_OFF, text); + if (off < builders) { + sharing.add(file.getFileName() + " (" + builders + + " builders, " + off + " turned off)"); + } + } + assertTrue("No examples were found", examples.isEmpty() == false); + assertTrue("These console examples share usage: " + sharing, + sharing.isEmpty()); + } + + @Test + public void everySettingOff_SaysToShareInProduction() throws IOException { + List missing = new ArrayList<>(); + for (Path file : exampleSources()) { + String[] lines = read(file).split("\n"); + for (int i = 0; i < lines.length; i++) { + if (SHARING_OFF.matcher(lines[i]).find()) { + StringBuilder before = new StringBuilder(); + for (int j = Math.max(0, i - COMMENT_LINES); j < i; j++) { + before.append(lines[j]).append(' '); + } + String comment = before.toString() + .replaceAll("\\s*(//|\\*)\\s*", " ") + .replaceAll("\\s+", " "); + if (comment.contains(PRODUCTION_COMMENT) == false) { + missing.add(file.getFileName() + ":" + (i + 1)); + } + } + } + } + assertTrue("These settings that turn usage sharing off do not say " + + "to leave it on in production: " + missing, missing.isEmpty()); + } + + /** + * The source of every console example, being each file with a main + * method. + */ + private static List exampleSources() throws IOException { + Path sources = Paths.get(System.getProperty("basedir", "."), + "src", "main", "java"); + assertTrue(sources + " should hold the examples", + Files.isDirectory(sources)); + List examples = new ArrayList<>(); + try (Stream walk = Files.walk(sources)) { + for (Path file : walk + .filter(f -> f.toString().endsWith(".java")) + .collect(Collectors.toList())) { + if (read(file).contains("static void main(")) { + examples.add(file); + } + } + } + return examples; + } + + /** + * The lines of a source that are code, leaving out the comment lines, + * which can show a builder in an example of its use. + */ + private static String code(String text) { + StringBuilder code = new StringBuilder(); + for (String line : text.split("\n")) { + String trimmed = line.trim(); + if (trimmed.startsWith("//") == false && + trimmed.startsWith("*") == false && + trimmed.startsWith("/*") == false) { + code.append(line).append('\n'); + } + } + return code.toString(); + } + + private static String read(Path file) throws IOException { + return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } + + private static int count(Pattern pattern, String text) { + Matcher matcher = pattern.matcher(text); + int count = 0; + while (matcher.find()) { + count++; + } + return count; + } +} diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java index 0b0051a0b..c1cfe7073 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java @@ -148,8 +148,19 @@ protected void processEngine(FlowData data, IPIntelligenceDataCloud aspectData) else { CloudRequestData requestData = data.getFromElement(cloudRequestEngine); - String json = ""; - json = requestData.getJsonResponse(); + setCloudRequestFailed(data, aspectData, cloudRequestEngine); + String json = requestData.getJsonResponse(); + + // A failed request can leave no response, which the Cloud + // Request Engine has reported. + if (json == null || json.isEmpty()) { + logger.warn("The " + getClass().getSimpleName() + " did " + + "not process as the JSON response from the " + + "CloudRequestEngine was null or empty. Please refer to " + + "errors generated by the CloudRequestEngine in the logs " + + "as this indicates an error occurred there."); + return; + } // Extract data from json to the aspectData instance. JSONObject jsonObj = new JSONObject(json); diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java new file mode 100644 index 000000000..ed50ea685 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java @@ -0,0 +1,161 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * When the Cloud Request Engine's request for the Flow Data failed, a + * property of the IP Intelligence cloud engine is missing with the + * CloudRequestFailed reason, rather than a reason about the resource key. + */ +public class CloudRequestFailedTests { + + private ILoggerFactory loggerFactory; + + private CloudRequestEngine requestEngine; + + @Before + public void init() { + Logger logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void RequestFailedWithNoResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed(null); + } + + @Test + public void RequestFailedWithAnErrorResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed("{ \"errors\": [ \"an error\" ] }"); + } + + private void assertCloudRequestFailed(String response) throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine()) { + IPIntelligenceDataCloud data = process(engine, response); + try { + data.getRegisteredName(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + e.getMessage(), + "CloudRequestFailed", + e.getReason().name()); + assertEquals( + "Property 'registeredname' not found in data for " + + "element '" + engine.getElementDataKey() + "'. " + + "This is because an upstream cloud request failed, " + + "so no data was available to populate the property.", + e.getMessage()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the IP Intelligence + * product, holding a property other than the one read. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private IPIntelligenceCloudEngine buildEngine() throws Exception { + IPIntelligenceCloudEngine engine = + new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredowner", "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put(engine.getElementDataKey(), product); + + requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + private IPIntelligenceDataCloud process( + IPIntelligenceCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + when(cloudData.getProcessStarted()).thenReturn(true); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getErrors()).thenReturn( + Collections.singletonList(new FlowError.Default( + new Exception("The cloud request failed."), + requestEngine))); + + IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( + loggerFactory.getLogger( + IPIntelligenceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + try { + engine.processEngine(flowData, data); + } catch (RuntimeException e) { + // A failed request's response may not be readable. The + // property is read afterwards either way, as it is through a + // Pipeline that suppresses process exceptions. + } + return data; + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java index 6d1300fd8..6ffe44a61 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java @@ -142,6 +142,11 @@ private ResultsIpiSwig getResultsContainingProperty(String propertyName) { @Override protected boolean propertyIsAvailable(String propertyName) { checkState(); + // A property the engine has a position for is held by its results, + // so that needs no lookup of the name in native code. + if (indexOf(propertyName) != null) { + return resultsList.isEmpty() == false; + } for (ResultsIpiSwig results : resultsList) { if (results.containsProperty(propertyName)) { return true; diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ReadByPositionTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ReadByPositionTests.java new file mode 100644 index 000000000..de73a7ba6 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ReadByPositionTests.java @@ -0,0 +1,86 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.StringValueSwig; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.*; + +/** + * A read of a property the engine has a position for goes by that + * position, and asks the native results nothing by name, not even whether + * they hold the property. + */ +public class ReadByPositionTests extends TestsBase { + + private static final Logger logger = mock(Logger.class); + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.MaxPerformance); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void StringRead_ByPosition_NoLookupByName() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + Integer index = engine.getPropertyIndex("RegisteredName"); + assertNotNull(index); + try (FlowData flowData = + getWrapper().getPipeline().createFlowData()) { + ResultsIpiSwig results = mock(ResultsIpiSwig.class); + when(results.containsProperty(any(String.class))).thenReturn(true); + StringValueSwig value = mock(StringValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn("Example"); + when(results.getValueAsString(index.intValue())).thenReturn(value); + IPIntelligenceDataHashDefault data = + new IPIntelligenceDataHashDefault( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + data.setResults(results); + + assertEquals("Example", data.getRegisteredName().getValue()); + + verify(results).getValueAsString(index.intValue()); + verify(results, never()).containsProperty(any(String.class)); + } + } +} diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java index 984fd29de..0e7c256a2 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java @@ -135,8 +135,20 @@ public GeoLocationCloudEngine( protected void processEngine(FlowData data, CloudGeoData aspectData) { if (cloudRequestEngine != null) { CloudRequestData requestData = data.getFromElement(cloudRequestEngine); + setCloudRequestFailed(data, aspectData, cloudRequestEngine); String json = requestData.getJsonResponse(); + // A failed request can leave no response, which the Cloud + // Request Engine has reported. + if (json == null || json.isEmpty()) { + logger.warn("The " + getClass().getSimpleName() + " did " + + "not process as the JSON response from the " + + "CloudRequestEngine was null or empty. Please refer to " + + "errors generated by the CloudRequestEngine in the logs " + + "as this indicates an error occurred there."); + return; + } + JSONObject jsonObj = new JSONObject(json); JSONObject geoObj = getElementJson(jsonObj); diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudRequestFailedTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudRequestFailedTests.java new file mode 100644 index 000000000..1fae1622a --- /dev/null +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudRequestFailedTests.java @@ -0,0 +1,163 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.flowelements; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.data.CloudGeoData; +import fiftyone.geolocation.data.CloudGeoDataDefault; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * When the Cloud Request Engine's request for the Flow Data failed, a + * property of the location cloud engine is missing with the + * CloudRequestFailed reason, rather than a reason about the resource key. + */ +public class GeoLocationCloudRequestFailedTests { + + private ILoggerFactory loggerFactory; + + private CloudRequestEngine requestEngine; + + @Before + public void init() { + Logger logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void RequestFailedWithNoResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed(null); + } + + @Test + public void RequestFailedWithAnErrorResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed("{ \"errors\": [ \"an error\" ] }"); + } + + private void assertCloudRequestFailed(String response) throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, response); + try { + data.getTown(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + e.getMessage(), + "CloudRequestFailed", + e.getReason().name()); + assertEquals( + "Property 'town' not found in data for element '" + + engine.getElementDataKey() + "'. This is because " + + "an upstream cloud request failed, so no data was " + + "available to populate the property.", + e.getMessage()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the location product, + * holding a property other than the one read. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private GeoLocationCloudEngine buildEngine() throws Exception { + GeoLocationCloudEngine engine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("country", "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put(engine.getElementDataKey(), product); + + requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + private CloudGeoData process( + GeoLocationCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + when(cloudData.getProcessStarted()).thenReturn(true); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getErrors()).thenReturn( + Collections.singletonList(new FlowError.Default( + new Exception("The cloud request failed."), + requestEngine))); + + CloudGeoData data = new CloudGeoDataDefault( + loggerFactory.getLogger(CloudGeoData.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + try { + engine.processEngine(flowData, data); + } catch (RuntimeException e) { + // A failed request's response may not be readable. The + // property is read afterwards either way, as it is through a + // Pipeline that suppresses process exceptions. + } + return data; + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java index 80b39e9a6..b19dc5c34 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java @@ -31,6 +31,7 @@ import fiftyone.pipeline.core.data.ElementPropertyMetaData; import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; import fiftyone.pipeline.core.data.factories.ElementDataFactory; import fiftyone.pipeline.core.data.types.JavaScript; import fiftyone.pipeline.core.flowelements.FlowElement; @@ -397,13 +398,14 @@ protected Map createAPVMap( protected void processEngine(FlowData data, TData aspectData) { CloudRequestData requestData; - + CloudRequestEngine requestEngine; + // Get requestData from CloudRequestEngine. If requestData does not // exist in the element data TypedKeyMap then the CloudRequestEngine either // does not exist in the Pipeline or is not run before this engine. try { - requestData = data.getFromElement( - getRequestEngine().getInstance()); + requestEngine = getRequestEngine().getInstance(); + requestData = data.getFromElement(requestEngine); } catch(Exception ex) { throw new PipelineConfigurationException( @@ -421,7 +423,9 @@ protected void processEngine(FlowData data, TData aspectData) { "before it in the Pipeline. This engine will be unable " + "to produce results until this is corrected."); } - + + setCloudRequestFailed(data, aspectData, requestEngine); + String json = requestData == null ? null : requestData.getJsonResponse(); // If the JSON is empty or null then do not Process the CloudAspectEngine. @@ -440,6 +444,48 @@ protected void processEngine(FlowData data, TData aspectData) { } } + /** + * Where the Cloud Request Engine's request for the Flow Data failed, + * which an error it added to the Flow Data shows, record it in the + * Element Data, so that a property the Element Data does not hold is + * missing with the CloudRequestFailed reason. + * @param data the Flow Data being processed + * @param aspectData the Element Data this engine populates + * @param requestEngine the Cloud Request Engine of the Pipeline + */ + protected void setCloudRequestFailed( + FlowData data, + TData aspectData, + CloudRequestEngine requestEngine) { + if (aspectData instanceof AspectDataBase == false || + data.getErrors() == null) { + return; + } + for (FlowError error : data.getErrors()) { + if (error.getFlowElement() == requestEngine) { + ((AspectDataBase) aspectData).setCloudRequestFailed(); + return; + } + } + } + + /** + * The request for the property list failed where the Cloud Request + * Engine still does not know the accessible properties, because it + * requests them when it is built and again on first use. + * @return true if the request for the property list failed + */ + @Override + public boolean propertyListRequestFailed() { + CloudRequestEngine requestEngine; + try { + requestEngine = getRequestEngine().getInstance(); + } catch (PipelineConfigurationException e) { + return false; + } + return requestEngine.getPublicProperties() == null; + } + /** * A virtual method to be implemented by the derived class which * uses the JsonResponse from the CloudRequestEngine to populate the TData diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestFailedReasonTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestFailedReasonTests.java new file mode 100644 index 000000000..66d70a737 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestFailedReasonTests.java @@ -0,0 +1,271 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.factories.ElementDataFactory; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectDataBase; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.flowelements.AspectEngineBase; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * A Cloud Aspect Engine's property is missing with the CloudRequestFailed + * reason when the Cloud Request Engine's request for the Flow Data failed, + * naming the first engine of the Element Data, and when the engine's + * metadata is not loaded because the request for the property list failed. + */ +public class CloudRequestFailedReasonTests { + + private static final String CLOUD_REQUEST_FAILED = + "This is because an upstream cloud request failed, so no data was " + + "available to populate the property."; + + private final TestLoggerFactory loggerFactory = + new TestLoggerFactory(null); + + public static class TestData extends AspectDataBase { + public TestData( + Logger logger, + FlowData flowData, + AspectEngine engine) { + super( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + } + } + + private class TestAspectEngine extends CloudAspectEngineBase { + TestAspectEngine() { + super( + loggerFactory.getLogger("TestAspectEngine"), + new ElementDataFactory() { + @Override + public TestData create( + FlowData flowData, + FlowElement flowElement) { + return new TestData( + loggerFactory.getLogger("TestData"), + flowData, + (AspectEngine) flowElement); + } + }); + } + + @Override + public String getElementDataKey() { + return "test"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(new ArrayList()); + } + + @Override + protected void unmanagedResourcesCleanup() { + } + + @Override + protected void processCloudEngine( + FlowData data, + TestData aspectData, + String json) { + } + } + + /** + * A Cloud Request Engine whose request fails, or answers with no part + * for the test engine, and which has the given accessible properties. + */ + private class TestRequestEngine + extends AspectEngineBase + implements CloudRequestEngine { + private final boolean fail; + private final Map + products; + + TestRequestEngine( + boolean fail, + Map products) { + super( + loggerFactory.getLogger("TestRequestEngine"), + new ElementDataFactory() { + @Override + public CloudRequestData create( + FlowData flowData, + FlowElement flowElement) { + return new CloudRequestData( + loggerFactory.getLogger("CloudRequestData"), + flowData, + (AspectEngine) flowElement); + } + }); + this.fail = fail; + this.products = products; + } + + @Override + public Map + getPublicProperties() { + return products; + } + + @Override + public String getDataSourceTier() { + return ""; + } + + @Override + public String getElementDataKey() { + return "cloud"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(new ArrayList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void unmanagedResourcesCleanup() { + } + + @Override + protected void processEngine( + FlowData data, + CloudRequestData aspectData) throws IOException { + aspectData.setProcessStarted(true); + if (fail) { + throw new IOException("The cloud service could not be reached."); + } + aspectData.setJsonResponse("{ \"other\": {} }"); + } + } + + @Test + public void RequestFailed_CloudRequestFailed() throws Exception { + PropertyMissingException e = readMissing(true, products()); + + assertEquals("CloudRequestFailed", e.getReason().name(), + e.getMessage()); + assertEquals( + "Property 'ismobile' not found in data for element 'test'. " + + CLOUD_REQUEST_FAILED, + e.getMessage()); + } + + @Test + public void RequestSucceeded_NotCloudRequestFailed() throws Exception { + PropertyMissingException e = readMissing(false, products()); + + assertEquals( + "PropertyNotAccessibleWithResourceKey", + e.getReason().name(), + e.getMessage()); + } + + @Test + public void PropertyListRequestFailed_CloudRequestFailed() + throws Exception { + PropertyMissingException e = readMissing(false, null); + + assertEquals("CloudRequestFailed", e.getReason().name(), + e.getMessage()); + assertEquals( + "Property 'ismobile' not found in data for element 'test'. " + + CLOUD_REQUEST_FAILED, + e.getMessage()); + } + + /** + * Process one Flow Data, with process exceptions suppressed, and read + * the property the response does not hold. + */ + private PropertyMissingException readMissing( + boolean fail, + Map products) + throws Exception { + TestAspectEngine engine = new TestAspectEngine(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(new TestRequestEngine(fail, products)) + .addFlowElement(engine) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + TestData data = flowData.getFromElement(engine); + return assertThrows( + PropertyMissingException.class, + () -> data.get("ismobile")); + } + } + + /** + * Products holding one test property, so that the engine's metadata is + * loaded and does not list the property read. + */ + private static Map + products() { + List properties = + new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData( + "hardwarevendor", "String", null, null)); + AccessiblePropertyMetaData.ProductMetaData product = + new AccessiblePropertyMetaData.ProductMetaData(); + product.properties = properties; + Map products = + new HashMap<>(); + products.put("test", product); + return products; + } +} diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java index 763da2bb3..6b60abfa9 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java @@ -22,9 +22,11 @@ package fiftyone.pipeline.core.data; +import fiftyone.pipeline.core.exceptions.PipelineDataException; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.exceptions.AggregateException; import java.util.Collection; import java.util.List; @@ -40,8 +42,12 @@ public interface FlowData extends AutoCloseable { /** * Process this FlowData via the owning Pipeline. Once this method returns * the FlowData will not be changed further and cannot be processed again. + * @throws IllegalStateException if this FlowData has already been + * processed, or its Pipeline has been closed + * @throws AggregateException if a Flow Element failed and the Pipeline + * does not suppress process exceptions, holding each failure */ - void process(); + void process() throws IllegalStateException, AggregateException; /** * If this flag is set to true, future FlowElements will not process this @@ -129,15 +135,18 @@ public interface FlowData extends AutoCloseable { * @param key the aspect to get * @param type type of the aspect * @return aspect's {@link ElementData} or null if not present + * @throws IllegalStateException if this FlowData has not been processed */ - T get(TypedKey key); + T get(TypedKey key) + throws IllegalStateException; /** * Get the {@link ElementData} for the requested data key. * @param key to the aspect to get * @return aspect's {@link ElementData} or null if not present + * @throws IllegalStateException if this FlowData has not been processed */ - ElementData get(String key); + ElementData get(String key) throws IllegalStateException; /** * Get the {@link ElementData} for the requested {@link ElementData} @@ -157,8 +166,10 @@ public interface FlowData extends AutoCloseable { * @param element {@link FlowElement} to get the data for * @param {@link ElementData} stored by the element * @return element data stored by the element + * @throws IllegalStateException if this FlowData has not been processed */ - T getFromElement(FlowElement element); + T getFromElement(FlowElement element) + throws IllegalStateException; /** * Get the specified property as the specified type. @@ -166,8 +177,12 @@ public interface FlowData extends AutoCloseable { * @param key the name of the property to get * @param type the type to return the property value as * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not of the type given */ - T getAs(String key, Class type); + T getAs(String key, Class type) + throws PipelineDataException, ClassCastException; /** * Check if the flow data contains an item with the specified key name and @@ -183,43 +198,67 @@ public interface FlowData extends AutoCloseable { * Get the specified property as a string. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a string */ - String getAsString(String propertyName); + String getAsString(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a boolean. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a boolean */ - Boolean getAsBool(String propertyName); + Boolean getAsBool(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as an integer. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not an integer */ - Integer getAsInt(String propertyName); + Integer getAsInt(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a long. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a long */ - Long getAsLong(String propertyName); + Long getAsLong(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a float. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a float */ - Float getAsFloat(String propertyName); + Float getAsFloat(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a double. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a double */ - Double getAsDouble(String propertyName); + Double getAsDouble(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get or add the specified element data to the internal map. diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java index f8758e126..1cb9dee48 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java @@ -191,7 +191,7 @@ public T get(TypedKey key) { @Override public ElementData get(String key) { if (processed.get() == false) { - throw new RuntimeException("This instance has not yet been processed"); + throw new IllegalStateException("This instance has not yet been processed"); } return data.get(new TypedKeyDefault(key, ElementData.class)); diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataThrowsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataThrowsTests.java new file mode 100644 index 000000000..316d2a5a4 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataThrowsTests.java @@ -0,0 +1,187 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineDataException; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.core.typed.TypedKeyDefault; +import fiftyone.pipeline.exceptions.AggregateException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.fail; +import static org.mockito.Mockito.*; + +/** + * Each exception a FlowData method throws is declared by that method of the + * FlowData interface, by its own type or a type it extends that is more + * specific than RuntimeException, so that IDEs and callers can see it. + */ +public class FlowDataThrowsTests { + + private PipelineInternal pipeline; + + private FlowDataDefault flowData; + + /** + * Code that is expected to throw. + */ + private interface Call { + void call() throws Exception; + } + + @Before + public void init() { + pipeline = mock(PipelineInternal.class); + flowData = new FlowDataDefault( + mock(Logger.class), + pipeline, + new EvidenceDefault(mock(Logger.class))); + } + + @After + public void cleanUp() throws Exception { + flowData.close(); + } + + @Test + public void process_Twice() throws Exception { + flowData.process(); + assertDeclared(method("process"), () -> flowData.process()); + } + + @Test + public void process_PipelineClosed() throws Exception { + when(pipeline.isClosed()).thenReturn(true); + assertDeclared(method("process"), () -> flowData.process()); + } + + @Test + public void process_ElementsFailed() throws Exception { + doThrow(new AggregateException( + "Exception(s) occurred processing evidence.", + Collections.singletonList(new Exception("failed")))) + .when(pipeline).process(flowData); + assertDeclared(method("process"), () -> flowData.process()); + } + + @Test + public void getByTypedKey_NotProcessed() throws Exception { + TypedKey key = + new TypedKeyDefault<>("test", ElementData.class); + assertDeclared( + method("get", TypedKey.class), + () -> flowData.get(key)); + } + + @Test + public void getByDataKey_NotProcessed() throws Exception { + assertDeclared( + method("get", String.class), + () -> flowData.get("test")); + } + + @Test + @SuppressWarnings("unchecked") + public void getFromElement_NotProcessed() throws Exception { + FlowElement element = mock(FlowElement.class); + when(element.getTypedDataKey()) + .thenReturn(new TypedKeyDefault<>("test", ElementData.class)); + assertDeclared( + method("getFromElement", FlowElement.class), + () -> flowData.getFromElement(element)); + } + + @Test + public void getAs_NotProcessed() throws Exception { + assertDeclared( + method("getAs", String.class, Class.class), + () -> flowData.getAs("test", String.class)); + } + + @Test + public void getAs_PropertyNotKnown() throws Exception { + flowData.process(); + when(pipeline.getMetaDataForProperty("test")) + .thenThrow(new PipelineDataException("Property not found")); + assertDeclared( + method("getAs", String.class, Class.class), + () -> flowData.getAs("test", String.class)); + } + + @Test + public void typedGetters_NotProcessed() throws Exception { + assertDeclared(method("getAsString", String.class), + () -> flowData.getAsString("test")); + assertDeclared(method("getAsBool", String.class), + () -> flowData.getAsBool("test")); + assertDeclared(method("getAsInt", String.class), + () -> flowData.getAsInt("test")); + assertDeclared(method("getAsLong", String.class), + () -> flowData.getAsLong("test")); + assertDeclared(method("getAsFloat", String.class), + () -> flowData.getAsFloat("test")); + assertDeclared(method("getAsDouble", String.class), + () -> flowData.getAsDouble("test")); + } + + private static Method method(String name, Class... parameters) + throws NoSuchMethodException { + return FlowData.class.getMethod(name, parameters); + } + + /** + * Run the call, which must throw, and check that the interface method + * declares what it threw. + */ + private static void assertDeclared(Method method, Call call) { + Throwable thrown = null; + try { + call.call(); + } catch (Throwable e) { + thrown = e; + } + if (thrown == null) { + fail(method.getName() + " should have thrown"); + } + for (Class declared : method.getExceptionTypes()) { + if (declared != RuntimeException.class && + declared != Exception.class && + declared != Throwable.class && + declared.isInstance(thrown)) { + return; + } + } + fail("FlowData." + method.getName() + " throws " + + thrown.getClass().getName() + " but declares " + + Arrays.toString(method.getExceptionTypes())); + } +} diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java index 5cd469993..94f14caaf 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java @@ -73,23 +73,43 @@ public static class MissingPropertyMessages { public static final String PREFIX = "Property '%s' not found in data for element '%s'. "; public static final String DATA_UPGRADE_REQUIRED = - "This is because your license and/or data file " + - "does not include this property. The property is available " + - "with the %s license/data for the %s"; + "This is because your license or data file does not include " + + "this property. It is available with the %s license/data for " + + "the %s. To upgrade, see " + + "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=data_upgrade_required."; + /** + * The upgrade message where the data tiers the property is present + * in are not known. + */ + public static final String DATA_UPGRADE_REQUIRED_TIERS_NOT_KNOWN = + "This is because your license or data file does not include " + + "this property. To see which licenses include it, see " + + "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=data_upgrade_required_tiers_not_known."; public static final String PROPERTY_EXCLUDED = "This is because the property has been excluded when configuring " + "the engine."; public static final String PRODUCT_NOT_IN_CLOUD_RESOURCE = "This is because your resource key does not include access to " + - "any properties under '%s'. For more details on resource keys, " + - "see our explainer: " + - "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=product_not_in_cloud_resource"; + "any properties under '%s'. Check or create a resource key with " + + "the properties you need in the configurator at " + + "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=product_not_in_cloud_resource."; public static final String PROPERTY_NOT_IN_CLOUD_RESOURCE = "This is because your resource key does not include access to " + - "this property. Properties that are included for this key under " + - "'%s' are %s. For more details on resource keys, see our " + - "explainer: " + - "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=property_not_in_cloud_resource"; + "this property. The properties included for your key under " + + "'%s' are %s. Check or change your key in the configurator at " + + "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=property_not_in_cloud_resource."; + /** + * The separator of the names in a list in a message. + */ + public static final String LIST_SEPARATOR = ","; + public static final String CLOUD_REQUEST_FAILED = + "This is because an upstream cloud request failed, so no data " + + "was available to populate the property."; + /** + * The Data Key a CloudRequestFailed message names when no engine + * populates the Element Data. + */ + public static final String UNKNOWN_DATA_KEY = "Unknown"; public static final String UNKNOWN = "The reason for this is unknown. Please check that the aspect " + "and property name are correct."; diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java index 6f1ff15ae..88ced1b32 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java @@ -53,6 +53,11 @@ public abstract class AspectDataBase extends ElementDataBase implements AspectDa private final Map, Future> processFutures; + /** + * Whether the cloud request for the Flow Data failed. + */ + private volatile boolean cloudRequestFailed = false; + /** * Constructs a new instance with a non-thread-safe, case-insensitive * {@link Map} as the underlying storage. @@ -171,6 +176,15 @@ public Object get(long timeout, TimeUnit unit) }; } + /** + * Record that the cloud request for the Flow Data failed, so that a + * property this instance does not hold is missing with the + * CloudRequestFailed reason. + */ + public void setCloudRequestFailed() { + cloudRequestFailed = true; + } + /** * Add an engine to the list of engines which have generated the data within * this instance. @@ -228,8 +242,13 @@ protected T getAs(String key, Class type, Class... parameterisedTypes) missingPropertyService != null) { // If there was no entry for the key then use the missing // property service to find out why. - MissingPropertyResult missingReason = missingPropertyService - .getMissingPropertyReason(key, engines); + MissingPropertyResult missingReason = cloudRequestFailed ? + missingPropertyService.getCloudRequestFailedReason( + key, + engines) : + missingPropertyService.getMissingPropertyReason( + key, + engines); logger.debug("Property '" + key + "' missing from " + "aspect data '" + getClass().getName() + "'-'" + hashCode() + "'. " + missingReason.getReason()); diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudAspectEngine.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudAspectEngine.java index f08018998..ff9caed9d 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudAspectEngine.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudAspectEngine.java @@ -37,4 +37,13 @@ public interface CloudAspectEngine< TData extends AspectData, TProperty extends AspectPropertyMetaData> extends AspectEngine { + + /** + * Whether the request for the list of properties the cloud service + * offers failed, so that this engine's property metadata is not loaded. + * @return true if the request for the property list failed + */ + default boolean propertyListRequestFailed() { + return false; + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyReason.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyReason.java index e9c01f3c1..60697b59e 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyReason.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyReason.java @@ -33,5 +33,10 @@ public enum MissingPropertyReason { PropertyExcludedFromEngineConfiguration, PropertyNotAccessibleWithResourceKey, ProductNotAccessibleWithResourceKey, + /** + * A request to the cloud service failed, so no data was available to + * populate the property. + */ + CloudRequestFailed, Unknown } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyService.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyService.java index af24e0696..359f4c872 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyService.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyService.java @@ -29,6 +29,8 @@ import java.util.List; +import static fiftyone.pipeline.engines.Constants.MissingPropertyMessages; + /** * Service used by {@link AspectEngine}s to report the reason for a property * not being present in the results. @@ -59,4 +61,29 @@ MissingPropertyResult getMissingPropertyReason( MissingPropertyResult getMissingPropertyReason( String propertyName, List> engines); + + /** + * Get the reason for the property not being found when the cloud + * request for the Flow Data failed, which is CloudRequestFailed. The + * message names the Data Key of the first engine, or Unknown when there + * is none. + * + * @param propertyName name of the {@link AspectPropertyMetaData} + * @param engines that populate the Element Data + * @return reason the property was not found + */ + default MissingPropertyResult getCloudRequestFailedReason( + String propertyName, + List> engines) { + String dataKey = engines == null || engines.isEmpty() ? + MissingPropertyMessages.UNKNOWN_DATA_KEY : + engines.get(0).getElementDataKey(); + return new MissingPropertyResult( + MissingPropertyReason.CloudRequestFailed, + String.format( + MissingPropertyMessages.PREFIX, + propertyName, + dataKey) + + MissingPropertyMessages.CLOUD_REQUEST_FAILED); + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyServiceDefault.java index 69f94920e..1c9206075 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyServiceDefault.java @@ -23,11 +23,15 @@ package fiftyone.pipeline.engines.services; import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.typed.TypedKey; import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import java.lang.reflect.Method; import java.util.List; import java.util.stream.Collectors; @@ -46,6 +50,11 @@ public class MissingPropertyServiceDefault implements MissingPropertyService { private static final Object lock = new Object(); + /** + * The start of the name of a typed accessor for a property. + */ + private static final String ACCESSOR_PREFIX = "get"; + /** * Private constructor to prevent new copies */ @@ -76,48 +85,56 @@ public MissingPropertyResult getMissingPropertyReason( MissingPropertyReason reason = MissingPropertyReason.Unknown; - AspectPropertyMetaData property = null; + boolean cloud = engine instanceof CloudAspectEngine; + boolean onPremise = engine instanceof OnPremiseAspectEngine; + // An On-premise Engine loads its metadata when it is built. + boolean loaded = cloud == false || engine.hasLoadedProperties(); - // We know the property has extends AspectPropertyMetaData as it - // is a constraint of the AspectEngine interface, so don't check this - // cast - for (Object currentPropertyObject : engine.getProperties()) { - AspectPropertyMetaData currentProperty = - (AspectPropertyMetaData)currentPropertyObject; - if (currentProperty.getName().equalsIgnoreCase(propertyName)) { - property = currentProperty; - break; - } - } - - if (property != null) { - // Check if the property is available in the data file that is - // being used by the engine. - boolean containsDataTier = false; - for (String tier : property.getDataTiersWherePresent()) { - if (tier.equals(engine.getDataSourceTier())) { - containsDataTier = true; + AspectPropertyMetaData property = null; + if (loaded) { + // We know the property has extends AspectPropertyMetaData as it + // is a constraint of the AspectEngine interface, so don't check + // this cast + for (Object currentPropertyObject : engine.getProperties()) { + AspectPropertyMetaData currentProperty = + (AspectPropertyMetaData)currentPropertyObject; + if (currentProperty.getName().equalsIgnoreCase(propertyName)) { + property = currentProperty; break; } } + } - if (containsDataTier == false) { + // The metadata lists the property. Only an On-premise Engine has a + // data file tier to compare. + if (property != null) { + if (onPremise && hasDataTier(property, engine) == false) { reason = MissingPropertyReason.DataFileUpgradeRequired; } - // Check if the property is excluded from the results. else if (property.isAvailable() == false) { reason = MissingPropertyReason.PropertyExcludedFromEngineConfiguration; } } - else { - if (CloudAspectEngine.class.isAssignableFrom(engine.getClass())) { - if (engine.getProperties().size() == 0) { - reason = MissingPropertyReason.ProductNotAccessibleWithResourceKey; - } - else { - reason = MissingPropertyReason.PropertyNotAccessibleWithResourceKey; - } + + if (reason == MissingPropertyReason.Unknown && cloud) { + if (loaded) { + reason = engine.getProperties().isEmpty() ? + MissingPropertyReason.ProductNotAccessibleWithResourceKey : + MissingPropertyReason.PropertyNotAccessibleWithResourceKey; } + else if (((CloudAspectEngine) engine) + .propertyListRequestFailed()) { + reason = MissingPropertyReason.CloudRequestFailed; + } + } + + // The metadata does not list the property, but the Element Data has + // a typed accessor for it, so another data file has it. + if (reason == MissingPropertyReason.Unknown && + onPremise && + property == null && + hasTypedAccessor(engine, propertyName)) { + reason = MissingPropertyReason.DataFileUpgradeRequired; } // Build the message string to return to the caller. @@ -128,12 +145,20 @@ else if (property.isAvailable() == false) { engine.getElementDataKey())); switch (reason) { case DataFileUpgradeRequired: - message.append(String.format( - MissingPropertyMessages.DATA_UPGRADE_REQUIRED, - stringJoin( - property.getDataTiersWherePresent(), - ","), - engine.getClass().getSimpleName())); + List tiers = property == null ? + null : + property.getDataTiersWherePresent(); + if (tiers == null || tiers.isEmpty()) { + message.append( + MissingPropertyMessages.DATA_UPGRADE_REQUIRED_TIERS_NOT_KNOWN); + } else { + message.append(String.format( + MissingPropertyMessages.DATA_UPGRADE_REQUIRED, + stringJoin( + tiers, + MissingPropertyMessages.LIST_SEPARATOR), + engine.getClass().getSimpleName())); + } break; case PropertyExcludedFromEngineConfiguration: message.append(MissingPropertyMessages.PROPERTY_EXCLUDED); @@ -151,7 +176,12 @@ else if (property.isAvailable() == false) { message.append(String.format( MissingPropertyMessages.PROPERTY_NOT_IN_CLOUD_RESOURCE, engine.getElementDataKey(), - stringJoin(available, ", "))); + stringJoin( + available, + MissingPropertyMessages.LIST_SEPARATOR))); + break; + case CloudRequestFailed: + message.append(MissingPropertyMessages.CLOUD_REQUEST_FAILED); break; case Unknown: message.append(MissingPropertyMessages.UNKNOWN); @@ -163,6 +193,43 @@ else if (property.isAvailable() == false) { return new MissingPropertyResult(reason, message.toString()); } + /** + * Whether the engine's data file tier is among the tiers the property + * is present in. + */ + private static boolean hasDataTier( + AspectPropertyMetaData property, + AspectEngine engine) { + List tiers = property.getDataTiersWherePresent(); + return tiers != null && tiers.contains(engine.getDataSourceTier()); + } + + /** + * Whether the type of the engine's Element Data, counting the types it + * inherits, has a typed accessor for the property, being a public + * method with no parameters named 'get' and the property name, compared + * ignoring case, that returns an {@link AspectPropertyValue}. + */ + private static boolean hasTypedAccessor( + AspectEngine engine, + String propertyName) { + TypedKey key = engine.getTypedDataKey(); + Class type = key == null ? null : key.getType(); + if (type == null) { + return false; + } + String accessor = ACCESSOR_PREFIX + propertyName; + for (Method method : type.getMethods()) { + if (method.getParameterCount() == 0 && + method.getName().equalsIgnoreCase(accessor) && + AspectPropertyValue.class.isAssignableFrom( + method.getReturnType())) { + return true; + } + } + return false; + } + @Override public MissingPropertyResult getMissingPropertyReason( String propertyName, diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyMessageTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyMessageTests.java new file mode 100644 index 000000000..8c902117a --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyMessageTests.java @@ -0,0 +1,211 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Each missing property message is the text the specification's table of + * reasons gives, with campaign parameters allowed on each link. Lists in a + * message are joined with a comma and no space. + */ +public class MissingPropertyMessageTests { + + private static final String DOMAIN = "51degrees.com"; + + private static final String PRICING = "https://" + DOMAIN + "/pricing"; + + private static final String CONFIGURATOR = "https://configure." + DOMAIN; + + /** + * The campaign parameters on a link, up to the end of the link, which + * is a space, the end of the message, or a full stop that ends it. + */ + private static final Pattern CAMPAIGN = + Pattern.compile("\\?utm_\\S*?(?=\\.(\\s|$)|\\s|$)"); + + private static final String PROPERTY = "HardwareVendor"; + + private static final String DATA_KEY = "device"; + + private static final String PREFIX = "Property '" + PROPERTY + + "' not found in data for element '" + DATA_KEY + "'. "; + + private final MissingPropertyService service = + MissingPropertyServiceDefault.getInstance(); + + @Test + public void DataFileUpgradeRequired_TiersKnown() { + AspectEngine engine = + onPremise("Lite"); + listed(engine, Arrays.asList("Premium", "Enterprise"), true); + + assertMessage( + PREFIX + "This is because your license or data file does not " + + "include this property. It is available with the " + + "Premium,Enterprise license/data for the " + + engine.getClass().getSimpleName() + ". To upgrade, see " + + PRICING + ".", + engine); + } + + @Test + public void DataFileUpgradeRequired_TiersNotKnown() { + AspectEngine engine = + onPremise("Lite"); + listed(engine, Collections.emptyList(), true); + + assertMessage( + PREFIX + "This is because your license or data file does not " + + "include this property. To see which licenses include it, " + + "see " + PRICING + ".", + engine); + } + + @Test + public void PropertyExcludedFromEngineConfiguration() { + AspectEngine engine = + onPremise("Premium"); + listed(engine, Arrays.asList("Premium", "Enterprise"), false); + + assertMessage( + PREFIX + "This is because the property has been excluded when " + + "configuring the engine.", + engine); + } + + @Test + public void ProductNotAccessibleWithResourceKey() { + AspectEngine engine = + cloud(Collections.emptyList()); + + assertMessage( + PREFIX + "This is because your resource key does not include " + + "access to any properties under '" + DATA_KEY + "'. Check " + + "or create a resource key with the properties you need in " + + "the configurator at " + CONFIGURATOR + ".", + engine); + } + + @Test + public void PropertyNotAccessibleWithResourceKey() { + AspectEngine engine = + cloud(Arrays.asList("IsMobile", "IsTablet")); + + assertMessage( + PREFIX + "This is because your resource key does not include " + + "access to this property. The properties included for your " + + "key under '" + DATA_KEY + "' are IsMobile,IsTablet. Check " + + "or change your key in the configurator at " + + CONFIGURATOR + ".", + engine); + } + + @Test + public void Unknown() { + AspectEngine engine = + onPremise("Premium"); + when(engine.getProperties()).thenReturn( + Collections.emptyList()); + + assertMessage( + PREFIX + "The reason for this is unknown. Please check that the " + + "aspect and property name are correct.", + engine); + } + + private void assertMessage( + String expected, + AspectEngine engine) { + String message = service.getMissingPropertyReason(PROPERTY, engine) + .getDescription(); + assertEquals(expected, CAMPAIGN.matcher(message).replaceAll("")); + } + + @SuppressWarnings("unchecked") + private static AspectEngine + onPremise(String tier) { + OnPremiseAspectEngine + engine = mock(OnPremiseAspectEngine.class); + when(engine.getDataSourceTier()).thenReturn(tier); + when(engine.getElementDataKey()).thenReturn(DATA_KEY); + return engine; + } + + /** + * A Cloud Engine whose metadata is loaded and lists the given names. + */ + @SuppressWarnings("unchecked") + private static AspectEngine + cloud(List names) { + CloudAspectEngine + engine = mock(CloudAspectEngine.class); + when(engine.getElementDataKey()).thenReturn(DATA_KEY); + when(engine.hasLoadedProperties()).thenReturn(true); + List properties = new ArrayList<>(); + for (String name : names) { + properties.add(new AspectPropertyMetaDataDefault( + name, + engine, + "", + String.class, + Collections.emptyList(), + true)); + } + when(engine.getProperties()).thenReturn(properties); + return engine; + } + + /** + * The engine's metadata lists the property, in the given data tiers. + */ + private static void listed( + AspectEngine engine, + List tiers, + boolean available) { + AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( + PROPERTY, + engine, + "", + String.class, + tiers, + available); + when(engine.getProperties()).thenReturn(Arrays.asList(property)); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyReasonTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyReasonTests.java new file mode 100644 index 000000000..4f1123c9d --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyReasonTests.java @@ -0,0 +1,286 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.pipeline.core.typed.TypedKeyDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The reason a property is missing, decided for each engine in the order + * the specification gives. The data tier is compared for an On-premise + * Engine only. A Cloud Engine whose metadata is not loaded gives + * CloudRequestFailed when its request for the property list failed, and + * Unknown otherwise. An On-premise Engine whose metadata does not list the + * property, but whose Element Data has a typed accessor for it, gives + * DataFileUpgradeRequired with no data tiers. When every engine gives + * Unknown, the message names the last engine. + */ +public class MissingPropertyReasonTests { + + /** + * A Cloud Engine that says whether its request for the property list + * failed. + */ + public abstract static class PropertyListCloudEngine + implements CloudAspectEngine { + public abstract boolean propertyListRequestFailed(); + } + + /** + * Element Data whose typed accessor for HardwareVendor is inherited. + */ + public interface BaseDeviceData extends AspectData { + AspectPropertyValue getHardwareVendor(); + } + + public interface DeviceData extends BaseDeviceData { + } + + private static final String PREFIX = + "Property '%s' not found in data for element '%s'. "; + + private static final String CLOUD_REQUEST_FAILED = + "This is because an upstream cloud request failed, so no data was " + + "available to populate the property."; + + private static final String UNKNOWN = + "The reason for this is unknown. Please check that the aspect and " + + "property name are correct."; + + private static final Pattern CAMPAIGN = + Pattern.compile("\\?utm_\\S*?(?=\\.(\\s|$)|\\s|$)"); + + private final MissingPropertyService service = + MissingPropertyServiceDefault.getInstance(); + + @Test + public void CloudEngine_PropertyListed_NoDataTierCheck() { + CloudAspectEngine engine = cloud(); + listed(engine, "Premium", true); + + assertReason( + "PropertyNotAccessibleWithResourceKey", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void OtherEngine_PropertyListed_NoDataTierCheck() { + @SuppressWarnings("unchecked") + AspectEngine engine = + mock(AspectEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + when(engine.getDataSourceTier()).thenReturn("Lite"); + listed(engine, "Premium", true); + + assertReason( + "Unknown", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void OnPremiseEngine_PropertyListed_DataTierChecked() { + AspectEngine engine = onPremise(null); + listed(engine, "Premium", true); + + assertReason( + "DataFileUpgradeRequired", + service.getMissingPropertyReason("hardwarevendor", engine)); + } + + @Test + public void CloudEngine_PropertyListedNotAvailable_Excluded() { + CloudAspectEngine engine = cloud(); + listed(engine, "Premium", false); + + assertReason( + "PropertyExcludedFromEngineConfiguration", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void CloudEngine_MetadataNotLoaded_ListRequestFailed() { + PropertyListCloudEngine engine = notLoaded(true); + + MissingPropertyResult result = + service.getMissingPropertyReason("HardwareVendor", engine); + + assertReason("CloudRequestFailed", result); + assertEquals( + String.format(PREFIX, "HardwareVendor", "device") + + CLOUD_REQUEST_FAILED, + result.getDescription()); + } + + @Test + public void CloudEngine_MetadataNotLoaded_ListRequestNotFailed() { + PropertyListCloudEngine engine = notLoaded(false); + + assertReason( + "Unknown", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void OnPremiseEngine_NotListed_TypedAccessor_TiersNotKnown() { + AspectEngine engine = onPremise(DeviceData.class); + listed(engine, "Premium", true, "IsMobile"); + + MissingPropertyResult result = + service.getMissingPropertyReason("hardwarevendor", engine); + + assertReason("DataFileUpgradeRequired", result); + assertEquals( + String.format(PREFIX, "hardwarevendor", "device") + + "This is because your license or data file does not " + + "include this property. To see which licenses include it, " + + "see https://" + "51degrees.com/pricing.", + CAMPAIGN.matcher(result.getDescription()).replaceAll("")); + } + + @Test + public void OnPremiseEngine_NotListed_NoTypedAccessor_Unknown() { + AspectEngine engine = onPremise(DeviceData.class); + listed(engine, "Premium", true, "IsMobile"); + + assertReason( + "Unknown", + service.getMissingPropertyReason("PlatformName", engine)); + } + + @Test + public void CloudEngine_NotListed_TypedAccessor_NotAnUpgrade() { + CloudAspectEngine engine = cloud(); + doReturn(new TypedKeyDefault<>("device", DeviceData.class)) + .when(engine).getTypedDataKey(); + listed(engine, "Premium", true, "IsMobile"); + + assertReason( + "PropertyNotAccessibleWithResourceKey", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void EveryEngineUnknown_MessageNamesTheLastEngine() { + AspectEngine first = onPremise(null); + when(first.getElementDataKey()).thenReturn("first"); + listed(first, "Premium", true, "IsMobile"); + AspectEngine last = onPremise(null); + when(last.getElementDataKey()).thenReturn("last"); + listed(last, "Premium", true, "IsMobile"); + List> + engines = new ArrayList<>(); + engines.add(first); + engines.add(last); + + MissingPropertyResult result = + service.getMissingPropertyReason("PlatformName", engines); + + assertReason("Unknown", result); + assertEquals( + String.format(PREFIX, "PlatformName", "last") + UNKNOWN, + result.getDescription()); + } + + private static void assertReason( + String expected, + MissingPropertyResult result) { + assertEquals( + result.getDescription(), + expected, + result.getReason().name()); + } + + @SuppressWarnings("unchecked") + private static CloudAspectEngine cloud() { + CloudAspectEngine engine = + mock(CloudAspectEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + when(engine.getDataSourceTier()).thenReturn("Lite"); + when(engine.hasLoadedProperties()).thenReturn(true); + return engine; + } + + private static PropertyListCloudEngine notLoaded(boolean failed) { + PropertyListCloudEngine engine = mock(PropertyListCloudEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + when(engine.hasLoadedProperties()).thenReturn(false); + when(engine.propertyListRequestFailed()).thenReturn(failed); + when(engine.getProperties()).thenReturn( + Collections.emptyList()); + return engine; + } + + @SuppressWarnings("unchecked") + private static AspectEngine onPremise(Class dataType) { + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + when(engine.getDataSourceTier()).thenReturn("Lite"); + when(engine.hasLoadedProperties()).thenReturn(true); + if (dataType != null) { + doReturn(new TypedKeyDefault<>("device", dataType)) + .when(engine).getTypedDataKey(); + } + return engine; + } + + private static void listed( + AspectEngine engine, + String tier, + boolean available) { + listed(engine, tier, available, "HardwareVendor"); + } + + private static void listed( + AspectEngine engine, + String tier, + boolean available, + String name) { + AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( + name, + engine, + "", + String.class, + Arrays.asList(tier), + available); + doReturn(Arrays.asList(property)).when(engine).getProperties(); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyServiceTests.java index 5c81ab7f8..f63f33e83 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyServiceTests.java @@ -28,6 +28,7 @@ import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; import org.junit.Before; import org.junit.Test; @@ -51,13 +52,14 @@ public void Initialise() { /** * Check that an "upgrade required" reason is returned when a property is - * available, but in another data tier. + * available, but in another data tier, which only an on-premise engine + * has. */ @SuppressWarnings("unchecked") @Test public void MissingPropertyService_GetReason_Upgrade() { // Arrange - AspectEngine engine = mock(AspectEngine.class); + AspectEngine engine = mock(OnPremiseAspectEngine.class); when(engine.getDataSourceTier()).thenReturn("lite"); when(engine.getElementDataKey()).thenReturn("testElement"); configureProperty(engine); @@ -156,6 +158,7 @@ public void MissingPropertyService_GetReason_ProductNotInResource() { // Arrange CloudAspectEngine engine = mock(CloudAspectEngine.class); when(engine.getElementDataKey()).thenReturn("testElement"); + when(engine.hasLoadedProperties()).thenReturn(true); when(engine.getProperties()).thenReturn(Collections.emptyList()); // Act @@ -188,6 +191,7 @@ public void MissingPropertyService_GetReason_PropertyNotInResource() { // Arrange CloudAspectEngine engine = mock(CloudAspectEngine.class); when(engine.getElementDataKey()).thenReturn("testElement"); + when(engine.hasLoadedProperties()).thenReturn(true); configureProperty(engine); // Act From 343baac313c0f1ec30c96a4bb9d823b1ac1a2553 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 13:30:03 +0000 Subject: [PATCH 33/42] Build the Device Detection example tests from a fresh clone (#187) [device-detection-specification/required-examples.md lines 90 to 96 at dbaf099c](https://github.com/51Degrees/specifications/blob/dbaf099c712e2bd3a2a80bfb4eaf284b079f05eb/device-detection-specification/required-examples.md?plain=1#L90-L96) says --- .../examples/console/GettingStartedOnPremTest.java | 6 ++---- .../examples/console/ManualDataUpdateTest.java | 8 +++----- .../examples/console/MetadataOnPremTest.java | 7 +++---- .../examples/console/OfflineProcessingTest.java | 9 +++------ .../examples/console/UpdateDataFileTest.java | 6 +++--- .../examples/web/GettingStartedWebOnPremTest.java | 7 ++----- 6 files changed, 16 insertions(+), 27 deletions(-) diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java index 4584988a4..7ceb26b14 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java @@ -22,17 +22,15 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.examples.shared.EvidenceHelper; -import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import org.junit.Test; public class GettingStartedOnPremTest { @Test public void gettingStartedOnPremTest() throws Exception { - GettingStartedOnPrem.run(TestFileUtils.getHashFileName() == null - ? TestFileUtils.LITE_HASH_DATA_FILE_PATH - : TestFileUtils.getHashFileName(), + GettingStartedOnPrem.run(DataFileHelper.getDataFileLocation(null), EvidenceHelper.setUpEvidence(), System.out); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java index 24cead133..b33d91b5a 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java @@ -22,7 +22,7 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -42,10 +42,8 @@ public class ManualDataUpdateTest { @Test public void manualDataUpdate_ReloadsTheDataFile() throws Exception { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - ManualDataUpdate.run(TestFileUtils.getHashFileName() == null - ? TestFileUtils.LITE_HASH_DATA_FILE_PATH - : TestFileUtils.getHashFileName(), - buffer); + ManualDataUpdate.run( + DataFileHelper.getDataFileLocation(null), buffer); String output = buffer.toString("UTF-8"); System.out.println(output); diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java index ffbc67a2d..cc2fa6573 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java @@ -22,14 +22,13 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import org.junit.Test; public class MetadataOnPremTest { @Test public void gettingMetaDataOnPremTest() throws Exception { - MetadataOnPrem.run(TestFileUtils.getHashFileName() == null - ? TestFileUtils.LITE_HASH_DATA_FILE_PATH - : TestFileUtils.getHashFileName(), System.out); + MetadataOnPrem.run( + DataFileHelper.getDataFileLocation(null), System.out); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java index 54605a354..15d517f04 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java @@ -22,19 +22,16 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import org.junit.Test; import java.io.FileInputStream; -import java.util.Objects; public class OfflineProcessingTest { @Test public void offlineProcessingTest() throws Exception { - OfflineProcessing.run(TestFileUtils.getHashFileName() == null - ? TestFileUtils.LITE_HASH_DATA_FILE_PATH - : TestFileUtils.getHashFileName(), - new FileInputStream(Objects.requireNonNull(TestFileUtils.getEvidenceFile())), + OfflineProcessing.run(DataFileHelper.getDataFileLocation(null), + new FileInputStream(DataFileHelper.getEvidenceFile(null)), System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java index 1c37b8d11..d05d8061d 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java @@ -22,7 +22,7 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.TestKeyUtils; +import fiftyone.devicedetection.examples.shared.KeyHelper; import org.junit.Test; import static fiftyone.devicedetection.examples.console.UpdateDataFile.run; @@ -33,9 +33,9 @@ public class UpdateDataFileTest { @Test public void updateExampleTest () throws Exception { - String licenseKey = TestKeyUtils.getNamedKey(UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME); + String licenseKey = KeyHelper.getNamedKey(UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME); assumeFalse("Skipping test, no license key found", - TestKeyUtils.isInvalidKey(licenseKey)); + KeyHelper.isInvalidKey(licenseKey)); try { run(null, licenseKey, false); } catch (IllegalStateException e) { diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java index e24c4a0c3..8de867e3f 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java @@ -24,7 +24,6 @@ import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.examples.shared.FindOutMore; -import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -47,10 +46,8 @@ public class GettingStartedWebOnPremTest { public static void startJetty() throws Exception { // With no Enterprise file, the free Lite file is found, or // downloaded, as the example finds it, so a fresh clone runs this. - System.setProperty("TestDataFile", TestFileUtils.getHashFile() == null - ? DataFileHelper.findDataFile( - TestFileUtils.LITE_HASH_DATA_FILE_PATH).getAbsolutePath() - : TestFileUtils.getHashFile().getAbsolutePath()); + System.setProperty( + "TestDataFile", DataFileHelper.getDataFileLocation(null)); SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); } From 402ab9c3b17360431b3ab2758aa90cbcd4b76749 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 13:30:19 +0000 Subject: [PATCH 34/42] Run the usage sharing builder tests, which JUnit 5 passed over (#142) `ShareUsageBuilderBaseTest` holds 32 tests of the usage sharing builder's setters, and none of them has ever run. The class uses JUnit 4 (`org.junit.Test`, `org.junit.Before`, `@Test(expected = ...)`), while its module, `pipeline/pipeline.engines.fiftyone`, depends on JUnit 5 alone ([pom.xml lines 61 to 64 at rc 23e0fc64](https://github.com/51Degrees/51degrees-java/blob/23e0fc649e97b290436f22eb0f6894002a9375f6/pipeline/pipeline.engines.fiftyone/pom.xml#L61-L64)). Surefire then runs the JUnit 5 engine only, and a JUnit 4 class there is passed over without a word. [#62](https://github.com/51Degrees/51degrees-java/pull/62) noted that the class "did not run in any of the runs above". --- .../flowelements/ShareUsageBuilderBase.java | 3 +- .../ShareUsageBuilderBaseTest.java | 111 ++++++++++++------ 2 files changed, 75 insertions(+), 39 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index 1ab56e3f9..6481ad1f5 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -447,7 +447,8 @@ public ShareUsageBuilderBase setSessionCookieName(String cookieName) { * If exactly the same evidence values are seen multiple times within this * time limit then they will only be shared once. *

- * The default value for this is 0 - meaning always share. + * The default is 20 minutes. An interval of 0, or a negative one, + * shares every repeat. * @param interval the interval in minutes * @return this builder */ diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBaseTest.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBaseTest.java index b91149c58..01a190e0a 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBaseTest.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBaseTest.java @@ -22,19 +22,19 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Arrays; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class ShareUsageBuilderBaseTest { ShareUsageBuilderBase base; - @Before + @BeforeEach public void setUp(){ base = new ShareUsageBuilderBase(null, null) { @Override @@ -78,9 +78,14 @@ public void testSetIncludedQueryStringParameters() { @Test public void setBlockedHttpHeaders() { + // The text setter adds to the list, as its Javadoc says, where the + // list setter replaces it. base.setBlockedHttpHeaders("g, h, i"); base.setBlockedHttpHeaders("a, b, c"); - assertEquals(3, base.blockedHttpHeaders.size()); + assertEquals(6, base.blockedHttpHeaders.size()); + assertTrue(base.blockedHttpHeaders.contains("g")); + assertTrue(base.blockedHttpHeaders.contains("h")); + assertTrue(base.blockedHttpHeaders.contains("i")); assertTrue(base.blockedHttpHeaders.contains("a")); assertTrue(base.blockedHttpHeaders.contains("b")); assertTrue(base.blockedHttpHeaders.contains("c")); @@ -111,22 +116,30 @@ public void testSetBlockedHttpHeaders() { } - @Test(expected = IllegalArgumentException.class) + @Test public void setIgnoreFlowDataEvidenceFilter1() { - base.setIgnoreFlowDataEvidenceFilter(""); + assertThrows(IllegalArgumentException.class, () -> { + base.setIgnoreFlowDataEvidenceFilter(""); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setIgnoreFlowDataEvidenceFilter2() { - base.setIgnoreFlowDataEvidenceFilter("hello"); + assertThrows(IllegalArgumentException.class, () -> { + base.setIgnoreFlowDataEvidenceFilter("hello"); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setIgnoreFlowDataEvidenceFilter2a() { - base.setIgnoreFlowDataEvidenceFilter("hello, one: two"); + assertThrows(IllegalArgumentException.class, () -> { + base.setIgnoreFlowDataEvidenceFilter("hello, one: two"); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setIgnoreFlowDataEvidenceFilter2b() { - base.setIgnoreFlowDataEvidenceFilter("hello : world, one"); + assertThrows(IllegalArgumentException.class, () -> { + base.setIgnoreFlowDataEvidenceFilter("hello : world, one"); + }); } @Test public void setIgnoreFlowDataEvidenceFilter3() { @@ -143,13 +156,17 @@ public void setSharePercentage1() { base.setSharePercentage(0.1); assertEquals(0.1, base.sharePercentage, 0.00001); } - @Test(expected = IllegalArgumentException.class) + @Test public void setSharePercentage2() { - base.setSharePercentage(-0.1); + assertThrows(IllegalArgumentException.class, () -> { + base.setSharePercentage(-0.1); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setSharePercentage3() { - base.setSharePercentage(1.1); + assertThrows(IllegalArgumentException.class, () -> { + base.setSharePercentage(1.1); + }); } @Test @@ -157,26 +174,34 @@ public void setMinimumEntriesPerMessage1() { base.setMinimumEntriesPerMessage(50); assertEquals(50, base.minimumEntriesPerMessage); } - @Test(expected = IllegalArgumentException.class) + @Test public void setMinimumEntriesPerMessage2() { - base.setMinimumEntriesPerMessage(0); + assertThrows(IllegalArgumentException.class, () -> { + base.setMinimumEntriesPerMessage(0); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setMinimumEntriesPerMessage3() { - base.setMinimumEntriesPerMessage(-1); + assertThrows(IllegalArgumentException.class, () -> { + base.setMinimumEntriesPerMessage(-1); + }); } @Test public void setMaximumQueueSize() { base.setMaximumQueueSize(200); assertEquals(200, base.maximumQueueSize); } - @Test(expected = IllegalArgumentException.class) + @Test public void setMaximumQueueSize1() { - base.setMaximumQueueSize(0); + assertThrows(IllegalArgumentException.class, () -> { + base.setMaximumQueueSize(0); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setMaximumQueueSize2() { - base.setMaximumQueueSize(-1); + assertThrows(IllegalArgumentException.class, () -> { + base.setMaximumQueueSize(-1); + }); } @Test @@ -190,9 +215,11 @@ public void setAddTimeoutMillis1() { base.setAddTimeoutMillis(1000); assertEquals(1000, base.addTimeout); } - @Test(expected = IllegalArgumentException.class) + @Test public void setAddTimeoutMillis2() { - base.setAddTimeoutMillis(-1); + assertThrows(IllegalArgumentException.class, () -> { + base.setAddTimeoutMillis(-1); + }); } @Test @@ -200,20 +227,24 @@ public void setTakeTimeoutMillis1() { base.setTakeTimeoutMillis(1000); assertEquals(1000, base.takeTimeout); } - @Test(expected = IllegalArgumentException.class) + @Test public void setTakeTimeoutMillis2() { - base.setTakeTimeoutMillis(-1); + assertThrows(IllegalArgumentException.class, () -> { + base.setTakeTimeoutMillis(-1); + }); } @Test public void setShareUsageUrl() { base.setShareUsageUrl("https://dog-food.com"); - assertEquals(base.shareUsageUrl, "https://dog-food.com"); + assertEquals("https://dog-food.com", base.shareUsageUrl); } - @Test(expected = IllegalArgumentException.class) + @Test public void setShareUsageUrl1() { - base.setShareUsageUrl("dog food"); + assertThrows(IllegalArgumentException.class, () -> { + base.setShareUsageUrl("dog food"); + }); } @Test @@ -221,11 +252,12 @@ public void setSessionCookieName() { base.setSessionCookieName("test"); assertEquals("test", base.sessionCookieName); } - @Test (expected = IllegalArgumentException.class) + @Test public void setSessionCookieName1() { - // illegal name - base.setSessionCookieName("te st"); - assertEquals("test", base.sessionCookieName); + assertThrows(IllegalArgumentException.class, () -> { + // illegal name + base.setSessionCookieName("te st"); + }); } @Test @@ -233,9 +265,12 @@ public void setRepeatEvidenceIntervalMinutes() { base.setRepeatEvidenceIntervalMinutes(1); assertEquals(1, base.repeatEvidenceInterval); } - @Test (expected = IllegalArgumentException.class) - public void setRepeatEvidenceIntervalMinutes1() { + @Test + public void setRepeatEvidenceIntervalMinutes_Negative_Kept() { + // A negative interval is not refused. It shares every repeat, as 0 + // does. base.setRepeatEvidenceIntervalMinutes(-1); + assertEquals(-1, base.repeatEvidenceInterval); } @Test From 81e3d4ad4bf9d872dac36296943937e3d0e43ee8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 13:33:17 +0000 Subject: [PATCH 35/42] Describe the examples as they run today in AGENTS.md and the example READMEs (#162) Documentation only. AGENTS.md and the two example READMEs described how the examples behaved before the free data file and Find out more changes, so an agent or a person following them was told to pass the Lite file as an argument, to run from the repository root only, and to expect a pricing link. --- AGENTS.md | 88 ++++++++++++++--------------- examples/device-detection/README.md | 9 +-- examples/ip-intelligence/README.md | 11 ++-- 3 files changed, 56 insertions(+), 52 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c5ab041b0..2e9a7a210 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,29 +106,33 @@ where `` is the `` in that examples folder's `pom.xml`, ### Run -Run the examples from the repository root. They find `logback.xml`, the data -file and their options file with `FileFinder`, which searches only below the -folder Java runs in (`pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/FileFinder.java`). -When a free data file is not there, the example finds its `.link` the same way -and runs the download script for it (`findDataFile` in each examples folder's -`shared` module, `DataFileHelper.java`), with Windows PowerShell on Windows and -`sh` on Linux and macOS. The same commands work on Windows, Linux and macOS. +Run the examples from the repository root, or from a product's examples +folder as its README does. They look for the data repository, `cxx/data`, in +the folder Java runs in and then in each folder above it, and find a data file +or its `.link` there (`findDataFile` in each examples folder's `shared` +module, `DataFileHelper.java`). When a free data file is not there, the +example runs the download script for its `.link`, with Windows PowerShell on +Windows and `sh` on Linux and macOS. They find `logback.xml` with +`FileFinder`, which searches below the folder Java runs in +(`pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/FileFinder.java`), +and the root and both examples folders have one. The same commands work on +Windows, Linux and macOS. ``` java -cp examples/device-detection/console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.GettingStartedOnPrem ``` -The IP Intelligence example looks for the paid Enterprise file, -`ip-intelligence/51Degrees-EnterpriseIpiV41.ipi`, so give it the Lite file as -its first argument. +The IP Intelligence example uses the free Lite file, +`ip-intelligence/51Degrees-LiteV41.ipi`, when it is given no argument, and +downloads it first when it is not there. ``` -java -cp examples/ip-intelligence/console/target/ip-intelligence-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.GettingStartedOnPrem cxx/data/ip-intelligence/51Degrees-LiteV41.ipi +java -cp examples/ip-intelligence/console/target/ip-intelligence-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.GettingStartedOnPrem ``` -The example downloads the Lite file first when it is not there. In place of -the argument, the `_51DEGREES_IPI_PATH` environment variable or system property -can give the absolute path of a data file that is already there. +An absolute path as the first argument names another data file. Otherwise the +`_51DEGREES_IPI_PATH` environment variable or system property, when it names +a file that is there, comes before the argument or the Lite file. On Java 24 or later, add `--enable-native-access=ALL-UNNAMED` before the class name, as @@ -143,32 +147,30 @@ and the script's `Downloading` and `Saved` lines follow. The Device Detection example processes five sets of evidence. For each it prints `Input values:` and the evidence, then `Results:` with -`Mobile Device`, `Platform Name`, `Platform Version`, `Browser Name` and -`Browser Version`. Logback lines come before and after. The last ones report -the data file and end the run, with these messages. - -``` -Used a 'Lite' data file, created , days ago, from location '' -This example is using the 'Lite ' data file. This is used for illustration, ... -All done -``` - -A warning that the data file is more than 28 days old may come too, which is -expected with the Lite file. The exit code is 0. +`Mobile Device`, `Platform Name`, `Platform Version`, `Browser Name`, +`Browser Version`, `Hardware Vendor`, `Hardware Name` and `Device Type`. The +Lite file does not have the last three, so each of them reads +`needs the Enterprise data file`. The IP Intelligence example looks up `116.154.188.222`, `45.236.48.61` and an IPv6 address. For each it prints `Input values:` with `query.client-ip`, then `Results:` with a line for each property, from `RegisteredName` to -`TimeZoneOffset`. The Lite file has RegisteredCountry, RegisteredName and -RegisteredOwner, so a line for a property it does not have reads -`Unknown. The property '' is not available in this data file.` followed -by a link, and the first such line also logs a warning that some properties -need a paid subscription. It then logs the data file in the same way, and -`All done`. +`TimeZoneOffset`. The Lite file has RegisteredName, RegisteredOwner and +RegisteredCountry, so every other line reads `needs the Enterprise data file`. -The paid data files have many more properties, as the -[pricing page](https://51degrees.com/pricing?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=correct-output) -describes. +Both then end the same way, where `` is `Lite` for Device Detection and +`IPIV4Lite` for IP Intelligence. + +``` +Some of these properties need the Enterprise data file. Contact us for a copy at +INFO ... Used a '' data file, created , days ago, from location '' +INFO ... All done +``` + +Then comes the "Find out more" section, between two rules, with the +product's pages on 51degrees.com and the repositories. A warning that the +data file is more than 28 days old may come before `All done`, which is +expected with the Lite file. The exit code is 0. ### Cloud examples @@ -198,16 +200,14 @@ example. The IP Intelligence cloud examples read their key from 'device-detection/51Degrees-LiteV4.1.hash' which must be somewhere in the project space to be found.` Run `git submodule update --init --recursive`. A zip download of the repository has no submodules, so clone with Git. -- **Running from another folder.** `FileFinder` looks only below the folder - Java runs in, so `java` started in `examples/device-detection`, as the - examples' README shows, cannot find the data file or its link in `cxx`. - Start it from the repository root. The same applies to `mvn exec:java`, +- **Running from outside the repository.** An example started in a folder + outside the repository, such as one the jar was copied to, finds no + `cxx/data` above it and no `logback.xml` below it. Start it in the + repository root or a folder inside it. The same applies to `mvn exec:java`, which runs in the folder Maven starts in. -- **The version in the jar name.** The examples' READMEs show jar names with - `4.4.20` and `4.4.19`, which are out of date. Use the name in `target`. -- **The IP Intelligence example without an argument.** It searches for the - Enterprise file, logs `Failed to find IP Intelligence data file` and stops, - even when the Lite file is in place. +- **The version in the jar name.** A jar carries the examples folder's own + ``, `4.5.127-SNAPSHOT` on `rc` today, not the version of the + packages the examples use, `4.5.130`. Use the name in `target`. - **The `dev` profile.** `-Pdev`, which CI uses, builds the examples against the libraries here, which needs `mvn install` at the root first and so the native build, with CMake 3.10 or later and a C++ toolchain. It is not needed diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index bdbf5889d..2aaf3f3de 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -104,12 +104,13 @@ Running mvn package ``` -will produce "fat" JARs inside `target` subfolders. +will produce "fat" JARs inside `target` subfolders. The version in each name is the one in this folder's `pom.xml`, which is not +the version of the packages the examples use. Use them with relevant example class entrypoints like: ```bash -java -cp ./console/target/device-detection-java-examples.console-4.4.20-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.OfflineProcessing +java -cp ./console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.OfflineProcessing ``` ### Web examples @@ -120,7 +121,7 @@ is pressed. Set the `PORT` environment variable to listen on another port, in wh case the example runs until the process is stopped: ```bash -PORT=8100 java -jar ./web/getting-started.onprem/target/device-detection-java-examples.web.getting-started.onprem-4.4.20-jar-with-dependencies.jar +PORT=8100 java -jar ./web/getting-started.onprem/target/device-detection-java-examples.web.getting-started.onprem-4.5.127-SNAPSHOT-jar-with-dependencies.jar ``` The on-premise web example takes its data file from the `_51DEGREES_DD_PATH` @@ -138,7 +139,7 @@ The fat JARs above bundle everything into a single jar on the classpath, so the permission to use is `ALL-UNNAMED`: ```bash -java -cp ./console/target/device-detection-java-examples.console-4.4.20-jar-with-dependencies.jar --enable-native-access=ALL-UNNAMED fiftyone.devicedetection.examples.console.OfflineProcessing +java -cp ./console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar --enable-native-access=ALL-UNNAMED fiftyone.devicedetection.examples.console.OfflineProcessing ``` With the Maven exec plugin, use the `exec:exec` goal. `exec:java` runs in the Maven JVM diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index ae3cd8d52..3875b4a26 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -90,8 +90,9 @@ The tables below describe the examples available in this repository. ### Cloud -Cloud examples require a resource key, supplied via the `TestResourceKey` -environment variable, system property, or command line argument. Resource keys +Cloud examples require a resource key, supplied via the `_51DEGREES_RESOURCE_KEY` +environment variable or system property, the legacy `TestResourceKey` name, or a +command line argument. Resource keys for IP Intelligence are not yet generally available, so these examples can also be pointed at a custom hosted cloud endpoint using the `TestCloudEndPoint` environment variable or system property. @@ -199,8 +200,10 @@ mvn package This will produce JARs with dependencies inside `target` subfolders. -Then run examples using the JAR: +Then run examples using the JAR, from this folder. The version in the name is +the one in this folder's `pom.xml`, which is not the version of the packages +the examples use. ```bash -java -cp .\console\target\ip-intelligence-java-examples.console-4.4.19-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.OfflineProcessing +java -cp ./console/target/ip-intelligence-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.OfflineProcessing ``` From 6d4454c827b51068860704a0c30fa46c2ded2fbd Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 18:37:22 +0000 Subject: [PATCH 36/42] Read cloud Device Detection values when the Property metadata is empty, and log a cloud response's warnings (#172) The Cloud Request Engine and the Device Detection cloud engine differed from the specification in two places, and two more rules had no test. --- .../DeviceDetectionCloudEngine.java | 113 ++++++++ .../flowelements/EmptyMetadataTests.java | 247 ++++++++++++++++++ .../CloudRequestEngineDefault.java | 11 + .../CloudRequestEngineCloudErrorsTests.java | 112 ++++++++ .../CloudRequestEngineLicenseKeyTests.java | 41 +++ .../CloudRequestEngineWarningsTests.java | 160 ++++++++++++ 6 files changed, 684 insertions(+) create mode 100644 device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/EmptyMetadataTests.java create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCloudErrorsTests.java create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineWarningsTests.java diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java index f2e429217..3a0a07de6 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java @@ -41,7 +41,13 @@ import org.json.JSONObject; import org.slf4j.Logger; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -56,6 +62,13 @@ */ public class DeviceDetectionCloudEngine extends CloudAspectEngineBase { + /** + * The type of value each typed accessor of the device data gives, keyed + * by the property name in lower case, so that a value the metadata does + * not describe can be given as the type its accessor declares. + */ + private static final Map DECLARED_TYPES = declaredTypes(); + volatile List aspectProperties; private String dataSourceTier; CloudRequestEngine cloudRequestEngine; @@ -204,6 +217,18 @@ protected void processEngine(FlowData data, DeviceDataCloud aspectData) { } } + // A value the metadata does not describe, as when the metadata + // is empty before discovery has completed, takes its type from + // the value itself. + for (String key : deviceObj.keySet()) { + if (key.endsWith("nullreason") == false && + deviceMap.containsKey(key) == false) { + deviceMap.put( + key, + getInferredAspectPropertyValue(deviceObj, key)); + } + } + // Add no value messages for any properties which have not yet been // populated. for (String key : deviceObj.keySet()) { @@ -407,6 +432,94 @@ private AspectPropertyValue getStringAspectPropertyValue( return stringValue; } + /** + * Get a value from the cloud engine's JSON response that the metadata + * does not describe, with its type taken from the value: a string, a + * boolean, a number, a list of strings, or an object as a map. A + * number is an Integer where the typed accessor of + * {@link DeviceDataCloud} declares one and the number is a whole + * number an Integer can hold, and a Double where the accessor declares + * one or the number has a decimal point. A null is no value, with the + * reason the response gives. + * @param deviceObj to get the value from + * @param key to get the value of + * @return {@link AspectPropertyValue} with the value, or the reason for + * the value not being present + */ + private AspectPropertyValue getInferredAspectPropertyValue( + JSONObject deviceObj, + String key) { + Object value = deviceObj.opt(key); + if (value == null || value == JSONObject.NULL) { + AspectPropertyValue noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage(getNoValueReason(deviceObj, key)); + return noValue; + } + if (value instanceof JSONArray) { + JSONArray jsonArray = (JSONArray) value; + List strings = new ArrayList<>(jsonArray.length()); + for (Object item : jsonArray.toList()) { + strings.add(Objects.toString(item, null)); + } + return new AspectPropertyValueDefault<>(strings); + } + if (value instanceof JSONObject) { + return new AspectPropertyValueDefault<>( + ((JSONObject) value).toMap()); + } + if (value instanceof Number) { + Number number = (Number) value; + Type declared = DECLARED_TYPES.get(key.toLowerCase(Locale.ROOT)); + if (Integer.class.equals(declared) && isWholeInteger(number)) { + return new AspectPropertyValueDefault<>(number.intValue()); + } + if (Double.class.equals(declared) || + value instanceof BigDecimal || + value instanceof Double || + value instanceof Float) { + return new AspectPropertyValueDefault<>( + number.doubleValue()); + } + } + return new AspectPropertyValueDefault<>(value); + } + + /** + * True when the number is a whole number an Integer can hold, such as + * 24 or 24.0, and false for 24.5 or a number beyond an Integer's range. + * @param number the number + * @return true if the number can be an Integer without changing it + */ + private static boolean isWholeInteger(Number number) { + double value = number.doubleValue(); + return value == Math.rint(value) && + value >= Integer.MIN_VALUE && + value <= Integer.MAX_VALUE; + } + + /** + * The type of value each typed accessor of {@link DeviceDataCloud} + * gives, being the type argument of the {@link AspectPropertyValue} it + * returns, keyed by the property name in lower case. + * @return the declared types + */ + private static Map declaredTypes() { + Map types = new HashMap<>(); + for (Method method : DeviceDataCloud.class.getMethods()) { + Type returned = method.getGenericReturnType(); + if (method.getParameterCount() == 0 && + method.getName().startsWith("get") && + AspectPropertyValue.class.equals(method.getReturnType()) && + returned instanceof ParameterizedType) { + types.put( + method.getName().substring(3).toLowerCase(Locale.ROOT), + ((ParameterizedType) returned).getActualTypeArguments()[0]); + } + } + return Collections.unmodifiableMap(types); + } + /** * The message for a property with no value when the response gives no * reason. diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/EmptyMetadataTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/EmptyMetadataTests.java new file mode 100644 index 000000000..c7d121c93 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/EmptyMetadataTests.java @@ -0,0 +1,247 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Before the Cloud Request Engine's discovery has completed, the engine has + * no property metadata. It still reads the values in the response, taking + * the type of each from the value itself. + */ +public class EmptyMetadataTests { + + private static final String RESPONSE = + "{ 'device': { " + + "'hardwarevendor': 'Apple', " + + "'ismobile': true, " + + "'screenpixelswidth': 1170, " + + "'pixelratio': 3.5, " + + "'hardwarename': [ 'iPhone 13', 'iPhone 14' ], " + + "'hardwaremodel': null, " + + "'platformname': null, " + + "'platformnamenullreason': 'No value on this request' }, " + + "'javascriptProperties': [] }"; + + private ILoggerFactory loggerFactory; + + @Before + public void init() { + Logger logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void EmptyMetadata_MetadataIsAnEmptyList() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + assertTrue(engine.getProperties().isEmpty()); + assertFalse(engine.hasLoadedProperties()); + } + } + + @Test + public void EmptyMetadata_StringIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals("Apple", data.getHardwareVendor().getValue()); + } + } + + @Test + public void EmptyMetadata_BooleanIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals(Boolean.TRUE, data.getIsMobile().getValue()); + } + } + + @Test + public void EmptyMetadata_IntegerIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals( + Integer.valueOf(1170), + data.getScreenPixelsWidth().getValue()); + } + } + + @Test + public void EmptyMetadata_DoubleIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals( + Double.valueOf(3.5), + data.getPixelRatio().getValue()); + } + } + + /** + * A whole number for a property whose typed accessor gives a Double is + * read as a Double, as the cloud service can give 3 for a pixel ratio + * of 3.0. + */ + @Test + public void EmptyMetadata_WholeNumberDeclaredDouble_IsDouble() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine, + "{ 'device': { 'pixelratio': 3 }, " + + "'javascriptProperties': [] }"); + Object value = data.getPixelRatio().getValue(); + assertEquals(Double.class, value.getClass()); + assertEquals(3.0, (Double) value, 0); + } + } + + /** + * A decimal with no fractional part for a property whose typed + * accessor gives an Integer is read as an Integer, so the accessor + * works if the cloud service gives 24.0 for a bits per pixel of 24. + */ + @Test + public void EmptyMetadata_WholeDecimalDeclaredInteger_IsInteger() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine, + "{ 'device': { 'bitsperpixel': 24.0 }, " + + "'javascriptProperties': [] }"); + Object value = data.getBitsPerPixel().getValue(); + assertEquals(Integer.class, value.getClass()); + assertEquals(24, ((Integer) value).intValue()); + } + } + + /** + * A decimal with a fractional part for a property whose typed accessor + * gives an Integer is not made a whole number, because that would + * change the value the service gave. + */ + @Test + public void EmptyMetadata_FractionDeclaredInteger_IsNotRounded() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine, + "{ 'device': { 'bitsperpixel': 24.5 }, " + + "'javascriptProperties': [] }"); + Object value = data.get("bitsperpixel"); + assertTrue("The value is not an AspectPropertyValue.", + value instanceof AspectPropertyValue); + assertEquals(24.5, + ((Number) ((AspectPropertyValue) value).getValue()) + .doubleValue(), 0); + } + } + + @Test + public void EmptyMetadata_ListIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals( + Arrays.asList("iPhone 13", "iPhone 14"), + data.getHardwareName().getValue()); + } + } + + @Test + public void EmptyMetadata_NullWithReason_HasThatReason() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertFalse(data.getPlatformName().hasValue()); + assertEquals( + "No value on this request", + data.getPlatformName().getNoValueMessage()); + } + } + + @Test + public void EmptyMetadata_NullWithoutReason_IsUnknown() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertFalse(data.getHardwareModel().hasValue()); + assertEquals( + "Unknown", + data.getHardwareModel().getNoValueMessage()); + } + } + + /** + * An engine whose Cloud Request Engine does not yet know the products, + * so the engine has no property metadata. + */ + private DeviceDetectionCloudEngine buildEngine() throws Exception { + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(null); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); + engine.addPipeline(pipeline); + return engine; + } + + private DeviceDataCloud process(DeviceDetectionCloudEngine engine) { + return process(engine, RESPONSE); + } + + private DeviceDataCloud process( + DeviceDetectionCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + DeviceDataCloud device = new DeviceDataCloudInternal( + loggerFactory.getLogger(DeviceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, device); + return device; + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index b95d7ec5d..8cca7a9d8 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -976,6 +976,17 @@ private void validateResponse(String jsonResult, .map(Object::toString) .collect(Collectors.toList())); } + + // Each warning is logged, whether or not the response also + // holds errors. The cloud's warnings can name the license key + // it was sent. + JSONArray warnings = jObj.optJSONArray("warnings"); + if (warnings != null) { + for (Object warning : warnings.toList()) { + logger.warn(LicenseKeyMask.maskIn( + String.valueOf(warning), getLicenseKeys())); + } + } } // If there were no errors but there was also no other data diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCloudErrorsTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCloudErrorsTests.java new file mode 100644 index 000000000..b2934284f --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCloudErrorsTests.java @@ -0,0 +1,112 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.CloudRequestException; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.exceptions.AggregateException; +import org.junit.jupiter.api.Test; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ExceptionCloudErrorsMultiple; +import static org.junit.jupiter.api.Assertions.*; + +/** + * The entries of the errors array in a response from the cloud service are + * thrown, one entry as a CloudRequestException carrying its text, and + * several as one AggregateException holding a CloudRequestException for + * each. + */ +public class CloudRequestEngineCloudErrorsTests + extends CloudRequestEngineTestsBase { + + public CloudRequestEngineCloudErrorsTests() throws MalformedURLException { + super(); + } + + @Test + public void Process_SeveralErrors_OneAggregateException() + throws Exception { + jsonResponse = "{\"errors\":[\"first error\",\"second error\"]}"; + configureMockedClient(); + + Throwable thrown = processOnce(); + + assertInstanceOf(AggregateException.class, thrown); + assertEquals(ExceptionCloudErrorsMultiple, thrown.getMessage()); + List messages = new ArrayList<>(); + for (Throwable inner : thrown.getSuppressed()) { + assertInstanceOf(CloudRequestException.class, inner); + assertEquals( + 200, + ((CloudRequestException) inner).getHttpStatusCode()); + messages.add(inner.getMessage()); + } + List expected = new ArrayList<>(); + expected.add("first error"); + expected.add("second error"); + assertEquals(expected, messages); + } + + @Test + public void Process_OneError_CloudRequestException() throws Exception { + jsonResponse = "{\"errors\":[\"only error\"]}"; + configureMockedClient(); + + Throwable thrown = processOnce(); + + assertInstanceOf(CloudRequestException.class, thrown); + assertTrue( + thrown.getMessage().contains("only error"), + thrown.getMessage()); + assertEquals(0, thrown.getSuppressed().length); + } + + /** + * Process one request, and give the one error it raised. + */ + private Throwable processOnce() throws Exception { + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + Collection errors = data.getErrors(); + assertEquals(1, errors.size(), errors.toString()); + return errors.iterator().next().getThrowable(); + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java index e05e9bc09..dbf5c5662 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java @@ -46,6 +46,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import static org.junit.jupiter.api.Assertions.*; @@ -470,6 +471,46 @@ public void Discovery_LicenseKeyAlone_PostsCredentialsInTheBody() assertNull(evidence.get(0).url.getQuery()); } + /** + * The property metadata is the accessibleproperties answer, never one + * made from the requested properties. A requested name the answer does + * not list has none, a product the answer lists but no name requested + * is kept, and each type is the answer's. + */ + @Test + public void Discovery_LicenseKeyAlone_MetadataIsTheAnswerNotTheRequest() + throws Exception { + cloud.propertiesResponse = + "{\"Products\":{" + + "\"device\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"IsMobile\",\"Type\":\"Boolean\"," + + "\"Category\":\"Device\"}]}," + + "\"location\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"Town\",\"Type\":\"String\"," + + "\"Category\":\"Location\"}]}}}"; + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList( + "device.ismobile", + "device.iscrawler", + "device.hardwarevendor"))); + + Map products = + engine.getPublicProperties(); + assertEquals( + new TreeSet<>(Arrays.asList("device", "location")), + new TreeSet<>(products.keySet())); + List device = + products.get("device").properties; + assertEquals(1, device.size()); + assertEquals("IsMobile", device.get(0).name); + assertEquals(boolean.class, device.get(0).getPropertyType()); + List location = + products.get("location").properties; + assertEquals(1, location.size()); + assertEquals("Town", location.get(0).name); + } + @Test public void Discovery_ResourceKeyAndProperties_GetsWithTheKeyInTheQuery() throws Exception { diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineWarningsTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineWarningsTests.java new file mode 100644 index 000000000..7fa462c8b --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineWarningsTests.java @@ -0,0 +1,160 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.common.testhelpers.TestLogger; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Each entry in the warnings array of a response from the cloud service is + * logged as a warning, for the requests made when the engine is built and + * for those made when it processes. + */ +public class CloudRequestEngineWarningsTests + extends CloudRequestEngineTestsBase { + + private static final String LICENSE_KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + + private static final String MASKED = "************************1234"; + + public CloudRequestEngineWarningsTests() throws MalformedURLException { + super(); + } + + @Test + public void Process_EachWarningLoggedAsAWarning() throws Exception { + jsonResponse = "{\"device\":{\"value\":\"1\"}," + + "\"warnings\":[\"first warning\",\"second warning\"]}"; + configureMockedClient(); + + process(new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build()); + + assertLoggedOnce("first warning"); + assertLoggedOnce("second warning"); + } + + @Test + public void Build_WarningInAccessibleProperties_LoggedAsAWarning() + throws Exception { + accessiblePropertiesResponse = + "{\"Products\": {\"device\": {\"DataTier\": \"tier\"," + + "\"Properties\": [{\"Name\": \"value\",\"Type\": \"String\"," + + "\"Category\": \"Device\"}]}}," + + "\"warnings\":[\"start-up warning\"]}"; + configureMockedClient(); + + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + + assertNotNull(engine.getPublicProperties()); + assertLoggedOnce("start-up warning"); + } + + /** + * The warnings are logged when the response also holds errors, which + * are thrown as before. + */ + @Test + public void Process_WarningBesideAnError_LoggedAndErrorRaised() + throws Exception { + jsonResponse = "{\"errors\":[\"an error\"]," + + "\"warnings\":[\"a warning\"]}"; + configureMockedClient(); + + String errors = process( + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build()); + + assertTrue(errors.contains("an error"), errors); + assertLoggedOnce("a warning"); + } + + /** + * A warning that names the license key shows it only by its last 4 + * characters, as an error does. + */ + @Test + public void Process_LicenseKeyInWarning_Masked() throws Exception { + jsonResponse = "{\"device\":{\"value\":\"1\"}," + + "\"warnings\":[\"License key " + LICENSE_KEY + + " expires soon\"]}"; + configureMockedClient(); + + process(new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .setLicenseKey(LICENSE_KEY) + .build()); + + for (String warning : warnings()) { + assertFalse(warning.contains(LICENSE_KEY), warning); + } + assertLoggedOnce("License key " + MASKED + " expires soon"); + } + + /** + * Process one request, and give the text of each error it raised. + */ + private String process(CloudRequestEngine engine) throws Exception { + StringBuilder errors = new StringBuilder(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + data.getErrors().forEach(e -> + errors.append(e.getThrowable().getMessage()).append('\n')); + } + return errors.toString(); + } + + private List warnings() { + List warnings = new ArrayList<>(); + for (TestLogger logger : loggerFactory.loggers) { + warnings.addAll(logger.warningsLogged); + } + return warnings; + } + + private void assertLoggedOnce(String expected) { + List warnings = warnings(); + long count = warnings.stream().filter(expected::equals).count(); + assertEquals(1, count, + "Expected the warning '" + expected + "' once in " + warnings); + } +} From 820a544d47cd1053932e650562b6760ba612bba4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 18:37:39 +0000 Subject: [PATCH 37/42] Minify the JavaScript with a third-party library, and test the JSON endpoint's form (#178) [javascript-builder.md lines 105 to 114 at dbaf099c](https://github.com/51Degrees/specifications/blob/dbaf099c712e2bd3a2a80bfb4eaf284b079f05eb/pipeline-specification/pipeline-elements/javascript-builder.md?plain=1#L105-L114) says --- pipeline/pipeline.javascriptbuilder/pom.xml | 15 + .../JavaScriptBuilderElement.java | 171 +++++++- .../JavaScriptBuilderElementBuilder.java | 26 +- .../JavaScriptBuilderTests.java | 6 + .../JavaScriptMinifyPerformanceTests.java | 140 ++++++ .../JavaScriptMinifyTests.java | 412 ++++++++++++++++++ .../JavaScriptObjectNameTests.java | 48 +- .../javascriptbuilder/JavaScriptUrlTests.java | 9 +- .../SessionIdAndSequenceTests.java | 7 +- .../web/ClientsideJsonEndpointTests.java | 212 +++++++++ pipeline/pom.xml | 1 + 11 files changed, 1017 insertions(+), 30 deletions(-) create mode 100644 pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyPerformanceTests.java create mode 100644 pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsideJsonEndpointTests.java diff --git a/pipeline/pipeline.javascriptbuilder/pom.xml b/pipeline/pipeline.javascriptbuilder/pom.xml index 7a7a03bf8..73a9e1f19 100644 --- a/pipeline/pipeline.javascriptbuilder/pom.xml +++ b/pipeline/pipeline.javascriptbuilder/pom.xml @@ -53,6 +53,21 @@ compiler ${mustache-compiler.version} + + + net.logicsquad + minifier + ${minifier.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.graalvm.js js-scriptengine diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index 30e7ffbf2..da5e9c6a2 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -38,13 +38,16 @@ import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; import fiftyone.pipeline.javascriptbuilder.templates.JavaScriptResource; import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; +import net.logicsquad.minifier.js.JSMinifier; import org.json.JSONObject; import org.slf4j.Logger; import java.io.*; import java.net.URLEncoder; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Matcher; import java.util.regex.Pattern; import static fiftyone.pipeline.core.Constants.*; @@ -69,8 +72,50 @@ public class JavaScriptBuilderElement protected String contextRoot; protected final String objName; protected final boolean enableCookies; + protected final boolean minify; private final Mustache mustache; + /** + * The minified forms of the template, one for each combination of its + * sections, with a placeholder where each value goes. Minifying is done + * once for each form and not for each request, so minification costs + * almost nothing per request, and the values the script carries are + * never passed through the minifier, so it cannot change them. + */ + private final ConcurrentHashMap minifiedForms = + new ConcurrentHashMap<>(); + + /** + * Set once minification has failed and been logged, after which the + * script is not minified. + */ + private final AtomicBoolean minifyFailed = new AtomicBoolean(false); + + /** + * The text each value is given when the template is minified. Each is + * an identifier, so the minifier leaves it as it is, whether it is in + * code or in a string. + */ + private static final String PLACEHOLDER_PREFIX = + "fiftyoneDegreesMinifyPlaceholder"; + + /** + * The values of the template that differ for each request, in the + * order of {@link #PLACEHOLDER_NAMES}. + */ + private static final String[] VALUE_KEYS = { + "_objName", "_jsonObject", "_sessionId", "_sequence", "_url", + "_parameters"}; + + /** + * The placeholder for each of {@link #VALUE_KEYS}. + */ + private static final String[] PLACEHOLDER_NAMES = { + "ObjName", "Json", "SessionId", "Sequence", "Url", "Parameters"}; + + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile( + PLACEHOLDER_PREFIX + "(" + String.join("|", PLACEHOLDER_NAMES) + ")"); + /** * Set once a callback URL that is not valid has been logged, because a * misconfigured element would otherwise log on every request. @@ -205,6 +250,48 @@ public JavaScriptBuilderElement( String host, String protocol, String contextRoot) { + this(logger, elementDataFactory, endpoint, objName, enableCookies, + host, protocol, contextRoot, true); + } + //! [constructor] + + //! [constructor] + /** + * Constructor. + * @param logger The logger. + * @param elementDataFactory The element data factory. + * @param endpoint Set the endpoint which will be queried on the host. + * e.g /api/v4/json + * @param objName The default name of the object instantiated by the client + * JavaScript. + * @param enableCookies Set whether the client JavaScript stored results of + * client side processing in cookies. + * @param host The host that the client JavaScript should query for updates. + * If null or blank then the host from the request will be used + * @param protocol The protocol (HTTP or HTTPS) that the client JavaScript + * will use when querying for updates. If null or empty + * then the protocol from the request will be used + * @param contextRoot The <context-root> setting from the web.xml. + * This is needed when creating the callback URL. + * @param minify True to minify the JavaScript that is produced. The other + * constructors minify it. + * @throws PipelineConfigurationException if protocol is not null or empty + * and is neither http nor https in any case of its ASCII letters, white + * space included, or if objName is not null and is + * not a valid JavaScript identifier, see {@link #isValidObjectName}. + * A null name means the default name is used, as no name was + * configured, and an empty name is refused + */ + public JavaScriptBuilderElement( + Logger logger, + ElementDataFactory elementDataFactory, + String endpoint, + String objName, + boolean enableCookies, + String host, + String protocol, + String contextRoot, + boolean minify) { super(logger, elementDataFactory); MustacheFactory mf = new DefaultMustacheFactory(); @@ -232,6 +319,7 @@ public JavaScriptBuilderElement( } this.enableCookies = enableCookies; this.contextRoot = contextRoot; + this.minify = minify; } //! [constructor] @@ -619,13 +707,88 @@ private void buildJavaScript( cookies, updateEnabled, hasDelayedProperties); - - StringWriter stringWriter = new StringWriter(); - mustache.execute(stringWriter, javaScriptObj.asMap()); - String content = stringWriter.toString(); + Map values = javaScriptObj.asMap(); + String content = null; + if (minify) { + content = renderMinified(values); + } + if (content == null) { + content = render(values); + } elementData.setJavaScript(content); } + + /** + * Renders the template as written. + */ + private String render(Map values) { + StringWriter stringWriter = new StringWriter(); + mustache.execute(stringWriter, values); + return stringWriter.toString(); + } + + /** + * Renders the minified template. The minified form of the template for + * the request's sections is made once and kept, and the request's values + * are put in place of its placeholders. + * @return the minified script, or null if minification failed + */ + private String renderMinified(Map values) { + if (minifyFailed.get()) { + return null; + } + // The form depends only on the template's sections, being every + // value that is not one of the placeholders. The URL is a value, and + // whether it is present is the section _updateEnabled. + int key = 0; + for (String section : new String[] { + "_supportsPromises", "_enableCookies", "_updateEnabled", + "_hasDelayedProperties"}) { + key = (key << 1) | (Boolean.TRUE.equals(values.get(section)) ? 1 : 0); + } + String form = minifiedForms.computeIfAbsent(key, k -> minifyForm(values)); + if (form.isEmpty()) { + return null; + } + + Matcher matcher = PLACEHOLDER_PATTERN.matcher(form); + StringBuffer script = new StringBuffer(form.length() + + String.valueOf(values.get("_jsonObject")).length()); + while (matcher.find()) { + String valueKey = VALUE_KEYS[ + Arrays.asList(PLACEHOLDER_NAMES).indexOf(matcher.group(1))]; + matcher.appendReplacement(script, + Matcher.quoteReplacement(String.valueOf(values.get(valueKey)))); + } + matcher.appendTail(script); + return script.toString(); + } + + /** + * Renders the template with the given sections and a placeholder for + * each value, and minifies it. + * @return the minified form, or an empty string if minification failed + */ + private String minifyForm(Map values) { + Map placeholders = new HashMap<>(values); + for (int i = 0; i < VALUE_KEYS.length; i++) { + placeholders.put(VALUE_KEYS[i], + PLACEHOLDER_PREFIX + PLACEHOLDER_NAMES[i]); + } + try { + StringWriter minified = new StringWriter(); + new JSMinifier(new StringReader(render(placeholders))) + .minify(minified); + return minified.toString(); + } catch (Exception e) { + if (minifyFailed.compareAndSet(false, true)) { + logger.error("The JavaScript could not be minified, so it " + + "is sent as it is written.", e); + } + return ""; + } + } } //! [class] diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java index 47a449373..89d3bcb8a 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java @@ -40,6 +40,10 @@ @ElementBuilder public class JavaScriptBuilderElementBuilder { public static final boolean ENABLE_COOKIES = true; + /** + * Whether the script is minified when nothing else is configured. + */ + public static final boolean MINIFY = true; private final ILoggerFactory loggerFactory; private final Logger logger; @@ -49,6 +53,7 @@ public class JavaScriptBuilderElementBuilder { protected String contextRoot = ""; protected String objName = Constants.DEFAULT_OBJECT_NAME; private boolean enableCookies = ENABLE_COOKIES; + private boolean minify = MINIFY; /** * Construct a new instance. @@ -171,6 +176,22 @@ public JavaScriptBuilderElementBuilder setEnableCookies( return this; } + /** + * Set whether the JavaScript that is produced is minified. Minifying + * takes out the template's comments and white space, so the script is + * smaller to send. The values the script carries, such as the JSON, are + * not changed. + *

+ * Default is true + * @param minify true to minify the JavaScript + * @return this builder + */ + @DefaultValue(booleanValue = MINIFY) + public JavaScriptBuilderElementBuilder setMinify(boolean minify) { + this.minify = minify; + return this; + } + /** * Build the {@link JavaScriptBuilderElement} * @return new {@link JavaScriptBuilderElement} instance @@ -194,7 +215,8 @@ public JavaScriptBuilderData create( enableCookies, host, protocol, - contextRoot); - } + contextRoot, + minify); + } } //! [class] diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java index 7ec0c95e1..05fc697cc 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java @@ -256,8 +256,10 @@ public void JavaScriptBuilderElement_DelayExecution() throws Exception { public void JavaScriptBuilderElement_VerifySession() throws Exception { configureMocks(); + // Not minified, so the declarations read as the template writes them. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); @@ -326,8 +328,10 @@ public void JavaScriptBuilderElement_ParametersExcludeSessionAndSequence() throws Exception { configureMocks(); + // Not minified, so the script is read line by line as written. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); String javaScript = result.getJavaScript(); @@ -378,9 +382,11 @@ public void JavaScriptBuilderElement_NoParameters() throws Exception { evidence.remove("query.latitude"); evidence.remove("query.longitude"); + // Not minified, so the script is read line by line as written. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) .setEndpoint("/json") + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyPerformanceTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyPerformanceTests.java new file mode 100644 index 000000000..cdece3b1c --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyPerformanceTests.java @@ -0,0 +1,140 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Measures what minification costs, as the JavaScript Builder specification + * asks of the feature, because a slow minification library once caused a + * customer problems. + *

+ * The element with minification on and the element with it off render in + * turn, many times, so a busy machine slows both alike, and the test fails + * only when minifying makes rendering much slower. The times are printed so + * a change in the cost can be seen. + * @see Specification + */ +public class JavaScriptMinifyPerformanceTests { + + /** + * Renders of each element before measuring, so the first render, which + * minifies the template's form, and the JIT compiler's work are not + * measured. + */ + private static final int WARM_UP = 200; + + /** + * Renders of each element that are measured. + */ + private static final int MEASURED = 2000; + + /** + * How many times slower a minified render may be than one not + * minified. Minifying is done once for each form of the template and + * not for each request, so a minified render costs about the same as + * one that is not minified, and a minifier run on every request is many + * times slower. The margin is wide so that timing on a busy machine + * does not decide the result. + */ + private static final double MAX_RATIO = 2.0; + + private final TestLoggerFactory loggerFactory = new TestLoggerFactory(null); + + @Test + public void Minify_PerformanceImpact_Small() throws Exception { + JavaScriptBuilderElement minified = element(true); + JavaScriptBuilderElement asWritten = element(false); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.user-agent", "iPhone"); + data.process(); + + long first = System.nanoTime(); + minified.process(data); + long firstMinified = System.nanoTime() - first; + + for (int i = 0; i < WARM_UP; i++) { + minified.process(data); + asWritten.process(data); + } + + long minifiedTime = 0; + long asWrittenTime = 0; + for (int i = 0; i < MEASURED; i++) { + // Each goes first in turn, so neither always follows the + // other. + if (i % 2 == 0) { + minifiedTime += time(minified, data); + asWrittenTime += time(asWritten, data); + } else { + asWrittenTime += time(asWritten, data); + minifiedTime += time(minified, data); + } + } + + double ratio = (double) minifiedTime / asWrittenTime; + System.out.printf( + "JavaScript render: first minified %.3f ms, then %.4f ms " + + "minified and %.4f ms not minified on average over %d, " + + "ratio %.2f%n", + firstMinified / 1e6, + minifiedTime / 1e6 / MEASURED, + asWrittenTime / 1e6 / MEASURED, + MEASURED, + ratio); + assertTrue(ratio <= MAX_RATIO, String.format( + "A minified render took %.2f times as long as one not " + + "minified, more than the %.1f allowed.", ratio, MAX_RATIO)); + } + } + + private JavaScriptBuilderElement element(boolean minify) { + return new JavaScriptBuilderElementBuilder(loggerFactory) + .setEndpoint("/json") + .setHost("example.com") + .setProtocol("https") + .setMinify(minify) + .build(); + } + + private static long time(JavaScriptBuilderElement element, FlowData data) + throws Exception { + long start = System.nanoTime(); + element.process(data); + return System.nanoTime() - start; + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java new file mode 100644 index 000000000..cc21c6455 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java @@ -0,0 +1,412 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.types.JavaScript; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.Value; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * The script is minified, or not, as configured, and minification is on + * unless it is turned off. See the Process section and the SetMinify + * configuration option of the JavaScript Builder specification. + * @see Specification + */ +public class JavaScriptMinifyTests { + + /** + * A comment in the template, which a minified script does not carry. + */ + private static final String TEMPLATE_COMMENT = + "// Log any errors returned in the JSON object."; + + /** + * A value that looks like code a minifier would change, and holds + * quotes and a line break, so it shows whether minification has changed + * the data the script carries. + */ + private static final String VALUE = + "a /* not a comment */ b // nor this\n\t'c' \"d\" {{_objName}}"; + + private static final String BROWSER = + "var window = globalThis;\n" + + "var console = { log: function() {}, warn: function() {},\n" + + " error: function() {} };\n" + + "var document = { cookie: '' };\n" + + "var sessionStorage = (function() {\n" + + " var data = {};\n" + + " return {\n" + + " get length() { return Object.keys(data).length; },\n" + + " key: function(i) {\n" + + " var k = Object.keys(data)[i];\n" + + " return k === undefined ? null : k; },\n" + + " getItem: function(k) {\n" + + " return k in data ? data[k] : null; },\n" + + " setItem: function(k, v) { data[k] = String(v); },\n" + + " removeItem: function(k) { delete data[k]; }\n" + + " };\n" + + "})();\n"; + + private final TestLoggerFactory loggerFactory = new TestLoggerFactory(null); + + /** + * Renders the script for one request. + * @param minify null to leave the builder's default, otherwise the + * value given to setMinify + * @param endpoint the endpoint, or an empty string for no callback + * @param cookies the value given to setEnableCookies + */ + private String render(Boolean minify, String endpoint, boolean cookies) + throws Exception { + return render(minify, endpoint, cookies, false, false); + } + + /** + * Renders the script for one request, choosing every setting the + * minified forms are held by. + * @param minify null to leave the builder's default, otherwise the + * value given to setMinify + * @param endpoint the endpoint, or an empty string for no callback + * @param cookies the value given to setEnableCookies + * @param promises true if the request's browser supports promises + * @param delayed true if the JSON holds a delayed execution property + */ + private String render( + Boolean minify, + String endpoint, + boolean cookies, + boolean promises, + boolean delayed) throws Exception { + JavaScriptBuilderElementBuilder builder = + new JavaScriptBuilderElementBuilder(loggerFactory) + .setEndpoint(endpoint) + .setHost("example.com") + .setProtocol("https") + .setEnableCookies(cookies); + if (minify != null) { + builder.setMinify(minify); + } + JavaScriptBuilderElement jsElement = builder.build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new ValuesElement(promises, delayed)) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + .addFlowElement(jsElement) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.some-value", "x y"); + // The same session for every render, so two renders differ only + // in whether they are minified. + flowData.addEvidence("query.session-id", "abc-123"); + flowData.addEvidence("query.sequence", "1"); + flowData.process(); + return flowData.getFromElement(jsElement).getJavaScript(); + } + } + + /** + * With no setting the script is minified, so it carries none of the + * template's comments and is much shorter than the script made with + * minification turned off. + */ + @Test + public void Minify_Default_ScriptIsMinified() throws Exception { + String minified = render(null, "/json", true); + String asWritten = render(false, "/json", true); + + assertFalse(minified.contains(TEMPLATE_COMMENT), + "The default script still carries the template's comments."); + assertTrue(minified.length() < asWritten.length() * 3 / 4, + "The default script is " + minified.length() + + " characters and the script with minification off is " + + asWritten.length() + "."); + } + + /** + * With minification turned on the script is minified as it is with no + * setting. + */ + @Test + public void Minify_On_ScriptIsMinified() throws Exception { + assertEquals(render(null, "/json", true), render(true, "/json", true)); + } + + /** + * With minification turned off the script is the template as written. + */ + @Test + public void Minify_Off_ScriptIsNotMinified() throws Exception { + assertTrue(render(false, "/json", true).contains(TEMPLATE_COMMENT)); + } + + /** + * Minification changes the code and not what the script carries: the + * JSON, the callback URL, the parameters, the session id, the sequence + * and the object name are the same as in the script made with + * minification turned off. + */ + @Test + public void Minify_On_ValuesUnchanged() throws Exception { + String minified = render(true, "/json", true); + String asWritten = render(false, "/json", true); + + assertTrue(json(asWritten).similar(json(minified)), + "The JSON differs.\n" + json(asWritten).toString(2) + "\n" + + json(minified).toString(2)); + assertEquals(VALUE, json(minified).getJSONObject("values") + .getString("text")); + assertTrue(minified.contains("'https://example.com/json'"), + "The callback URL is missing."); + assertEquals( + parameters(asWritten).toString(), + parameters(minified).toString()); + assertTrue(minified.contains("\"" + sessionId(asWritten) + "\""), + "The session id is missing."); + assertTrue(Pattern.compile("var fod\\s*=\\s*new fiftyoneDegreesManager") + .matcher(minified).find(), "The object is not created."); + } + + /** + * Every choice of the four settings the minified forms are held by, + * being whether there is a callback, whether cookies are enabled, + * whether the browser supports promises and whether the JSON holds a + * delayed execution property. + */ + static Stream everyForm() { + List forms = new ArrayList<>(); + for (String endpoint : new String[] {"/json", ""}) { + for (boolean cookies : new boolean[] {true, false}) { + for (boolean promises : new boolean[] {true, false}) { + for (boolean delayed : new boolean[] {true, false}) { + forms.add(Arguments.of( + endpoint, cookies, promises, delayed)); + } + } + } + } + return forms.stream(); + } + + /** + * Each of the four settings changes the script as written, so the + * sixteen choices are sixteen different scripts and the forms tested + * below are every form the element holds. + */ + @Test + public void Minify_EveryForm_IsADifferentScript() throws Exception { + Set scripts = new HashSet<>(); + Iterable forms = everyForm()::iterator; + for (Arguments form : forms) { + Object[] a = form.get(); + scripts.add(render(false, (String) a[0], (Boolean) a[1], + (Boolean) a[2], (Boolean) a[3])); + } + assertEquals(16, scripts.size(), + "The four settings do not give sixteen different scripts."); + } + + /** + * Every form of the minified script parses. + */ + @ParameterizedTest + @MethodSource("everyForm") + public void Minify_On_EveryFormParses( + String endpoint, + boolean cookies, + boolean promises, + boolean delayed) throws Exception { + String script = render(true, endpoint, cookies, promises, delayed); + try (Context context = context()) { + try { + context.parse(Source.create("js", script)); + } catch (PolyglotException e) { + fail("The minified script does not parse: " + e.getMessage()); + } + } + } + + /** + * The minified script runs and creates the object with the values from + * the JSON. + */ + @Test + public void Minify_On_ScriptRuns() throws Exception { + String script = render(true, "", true); + try (Context context = context()) { + context.eval("js", BROWSER); + context.eval(Source.create("js", script)); + Value fod = context.eval("js", "window").getMember("fod"); + assertFalse(fod == null || fod.isNull(), "fod was not created."); + assertTrue(fod.getMember("complete").canExecute()); + assertEquals(VALUE, + fod.getMember("values").getMember("text").asString()); + } + } + + private static Context context() { + return Context.newBuilder("js") + .option("engine.WarnInterpreterOnly", "false") + .build(); + } + + /** + * The JSON the script assigns to its json variable. + */ + private static JSONObject json(String script) { + Matcher matcher = Pattern.compile("var json\\s*=\\s*").matcher(script); + assertTrue(matcher.find(), "The script holds no JSON."); + return (JSONObject) new JSONTokener(script.substring(matcher.end())) + .nextValue(); + } + + /** + * The parameters object the script's request parameters function + * returns. + */ + private static JSONObject parameters(String script) { + Matcher matcher = Pattern.compile( + "renderedParameters\\s*=\\s*function\\(\\)\\s*\\{\\s*return\\s*") + .matcher(script); + assertTrue(matcher.find(), "The script holds no parameters."); + return (JSONObject) new JSONTokener(script.substring(matcher.end())) + .nextValue(); + } + + private static String sessionId(String script) { + Matcher matcher = Pattern.compile("var sessionId\\s*=\\s*\"([^\"]*)\"") + .matcher(script); + assertTrue(matcher.find(), "The script holds no session id."); + return matcher.group(1); + } + + /** + * An element that adds a value which looks like code, in place of an + * engine. + */ + private static class ValuesElement + extends FlowElementBase { + + private final boolean promises; + + private final boolean delayed; + + /** + * @param promises true to give Promise as "Full", which the + * JavaScript Builder reads as the browser + * supporting promises + * @param delayed true to add a delayed execution property + */ + ValuesElement(boolean promises, boolean delayed) { + super(new TestLoggerFactory(null).getLogger("values"), null); + this.promises = promises; + this.delayed = delayed; + } + + @Override + protected void processInternal(FlowData data) { + ElementDataBase values = new ElementDataBase(logger, data) { }; + values.put("text", VALUE); + if (promises) { + values.put("Promise", + new AspectPropertyValueDefault<>("Full")); + } + if (delayed) { + values.put("delayed", new JavaScript("var delayed = 1;")); + } + data.getOrAdd(getElementDataKey(), d -> values); + } + + @Override + public String getElementDataKey() { + return "values"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + List properties = new ArrayList<>(); + properties.add(new ElementPropertyMetaDataDefault( + "text", this, "test", String.class, true)); + if (promises) { + properties.add(new ElementPropertyMetaDataDefault( + "Promise", this, "test", String.class, true)); + } + if (delayed) { + properties.add(new ElementPropertyMetaDataDefault( + "delayed", this, "test", JavaScript.class, true, null, + true, Collections.emptyList())); + } + return properties; + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java index 712bc6873..119ab8149 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java @@ -316,7 +316,9 @@ public void ObjectName_WindowPropertyVarCanReplace_Allowed(String name) { String script = render(element); - assertTrue(script.contains("var " + name + " = new"), + assertTrue(Pattern.compile( + "var " + Pattern.quote(name) + "\\s*=\\s*new") + .matcher(script).find(), "the script should declare '" + name + "'"); assertEquals(0, warnings().size()); } @@ -356,16 +358,20 @@ public void ObjectName_NotConfigured_DefaultUsed() { } private void assertWorkingObject(String script, String name) { + // The script is minified by default, so the patterns allow for the + // white space around '=' being taken out. assertTrue(Pattern.compile( "var " + Pattern.quote(name) + - " = new fiftyoneDegreesManager\\(\\);") + "\\s*=\\s*new fiftyoneDegreesManager\\(\\);") .matcher(script).find(), "the script should declare var " + name); if (name.equals("fod") == false) { assertFalse(Pattern.compile("var fod\\b").matcher(script).find(), "the script should not declare var fod"); } - assertTrue(script.contains("var sessionKey = \"" + name + "\";"), + assertTrue(Pattern.compile( + "var sessionKey\\s*=\\s*\"" + Pattern.quote(name) + "\";") + .matcher(script).find(), "the storage key should be " + name); assertTrue(script.contains("window[\"" + name + "Evidence\"]"), "the evidence lookup should use " + name); @@ -406,25 +412,27 @@ private List warnings() { } /** - * The script with the line that assigns its request parameters taken - * out. The script sends the page's query evidence, the requested object - * name included, back with its own request. The builder url encodes - * each value and the line holds them as JSON strings, so the requested - * name there is data and not code, and it is left out of the checks - * that the name was not written into the code. The line is found as the - * existing tests find it, by the word and a brace, because the template - * has named the declaration differently over time. + * The script with the object holding its request parameters taken out. + * The script sends the page's query evidence, the requested object name + * included, back with its own request. The builder url encodes each + * value, so no value holds a brace, and the object holds them as JSON + * strings, so the requested name there is data and not code, and it is + * left out of the checks that the name was not written into the code. + * The object is found by the word parameters and a brace, because the + * template has named the declaration differently over time, and not by + * line, because the script is minified by default. */ + private static final Pattern PARAMETERS = Pattern.compile( + "(?i)\\w*parameters\\s*=\\s*" + + "(?:function\\s*\\(\\)\\s*\\{\\s*return\\s*)?\\{[^}]*\\}"); + private static String withoutParameters(String script) { - StringBuilder rest = new StringBuilder(); - for (String line : script.split("\\r?\\n")) { - if (line.toLowerCase().contains("parameters =") && - line.contains("{")) { - continue; - } - rest.append(line).append('\n'); - } - return rest.toString(); + Matcher matcher = PARAMETERS.matcher(script); + assertTrue(matcher.find(), "the script should assign its parameters"); + String rest = matcher.replaceFirst(""); + assertFalse(PARAMETERS.matcher(rest).find(), + "the script should assign its parameters once"); + return rest; } private static int count(String text, String find) { diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index fc8cb2f61..ca316fb58 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -55,7 +55,9 @@ /** * Checks the callback URL the rendered script is given. These tests render - * the script only and need no browser. + * the script only and need no browser. The script is not minified, so it is + * read as the template writes it. {@link JavaScriptMinifyTests} checks that + * the minified script carries the same values. */ public class JavaScriptUrlTests { private final TestLoggerFactory loggerFactory; @@ -72,7 +74,8 @@ private String render(String contextRoot) throws Exception { new JavaScriptBuilderElementBuilder(loggerFactory) .setEndpoint("/json") .setHost("example.com") - .setProtocol("https"); + .setProtocol("https") + .setMinify(false); if (contextRoot != null) { builder.setContextRoot(contextRoot); } @@ -168,6 +171,7 @@ private List renderWithHosts(String... hosts) throws Exception { JavaScriptBuilderElement jsElement = new JavaScriptBuilderElementBuilder(factory) .setEndpoint("/json") + .setMinify(false) .build(); List scripts = new ArrayList<>(); try (Pipeline pipeline = new PipelineBuilder(factory) @@ -270,6 +274,7 @@ public void Url_RefusedProtocol_NoCallbackAndLoggedOnce() throws Exception { JavaScriptBuilderElement jsElement = new JavaScriptBuilderElementBuilder(factory) .setEndpoint("/json") + .setMinify(false) .build(); try (Pipeline pipeline = new PipelineBuilder(factory) .addFlowElement(new SequenceElementBuilder(factory).build()) diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java index 06bc9b6b6..e5628a3d6 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java @@ -76,11 +76,14 @@ */ public class SessionIdAndSequenceTests { + // The script is minified by default, so the patterns allow for the + // white space around '=' being taken out and for the rest of the + // script following on the same line. private static final Pattern SESSION_ID_LINE = - Pattern.compile("var sessionId = \"(.*)\";"); + Pattern.compile("var sessionId\\s*=\\s*\"([^\"]*)\";"); private static final Pattern SEQUENCE_LINE = - Pattern.compile("var sequence = (.*);"); + Pattern.compile("var sequence\\s*=\\s*([^;]*);"); private final TestLoggerFactory loggerFactory = new TestLoggerFactory(null); diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsideJsonEndpointTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsideJsonEndpointTests.java new file mode 100644 index 000000000..082ddf9ac --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsideJsonEndpointTests.java @@ -0,0 +1,212 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; +import fiftyone.pipeline.web.services.FlowDataProviderCore; +import fiftyone.pipeline.web.services.WebRequestEvidenceServiceCore; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The JSON endpoint answers with JSON in the same form as the JSON the + * JavaScript holds, as the SetEndpoint row of the JavaScript Builder's + * configuration options asks. + * @see Specification + */ +public class ClientsideJsonEndpointTests { + + private static final ILoggerFactory LOGGERS = + LoggerFactory.getILoggerFactory(); + + /** + * The text that the template puts before the JSON in the script, with + * or without the white space that minifying takes out. + */ + private static final Pattern JSON_IN_SCRIPT = + Pattern.compile("var json\\s*=\\s*"); + + /** + * The JSON served by the JSON endpoint and the JSON the script served by + * the JavaScript endpoint holds, for one request, are the same, with a + * value, a Property assigned no value and its nullreason, and the + * JavaScript Property and session values the builders add. + */ + @Test + public void JsonEndpoint_SameFormAsJsonInJavaScript() throws Exception { + try (Pipeline pipeline = new PipelineBuilder(LOGGERS) + .addFlowElement(new SequenceElementBuilder(LOGGERS).build()) + .addFlowElement(new ValuesElement()) + .addFlowElement(new JsonBuilderElementBuilder(LOGGERS).build()) + .addFlowElement(new JavaScriptBuilderElementBuilder(LOGGERS) + .setEndpoint("/51dpipeline/json") + .build()) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + HttpServletRequest request = request(); + new WebRequestEvidenceServiceCore.Default() + .addEvidenceFromRequest(data, request); + data.process(); + assertTrue("Processing recorded errors: " + data.getErrors(), + data.getErrors().isEmpty()); + + FlowDataProviderCore provider = mock(FlowDataProviderCore.class); + when(provider.getFlowData(request)).thenReturn(data); + ClientsidePropertyServiceCore service = + new ClientsidePropertyServiceCore.Default(provider, pipeline); + + StringWriter json = new StringWriter(); + service.serveJson(request, response(json)); + StringWriter javaScript = new StringWriter(); + service.serveJavascript(request, response(javaScript)); + + JSONObject served = new JSONObject(json.toString()); + JSONObject embedded = jsonInScript(javaScript.toString()); + assertTrue("The JSON endpoint answered\n" + served.toString(2) + + "\nbut the script holds\n" + embedded.toString(2), + served.similar(embedded)); + JSONObject values = served.getJSONObject("values"); + assertEquals("a value", values.getString("value")); + assertTrue(values.isNull("novalue")); + assertEquals("No value for the test.", + values.getString("novaluenullreason")); + } + } + + /** + * The JSON object the script assigns to its json variable. + */ + private static JSONObject jsonInScript(String javaScript) { + Matcher matcher = JSON_IN_SCRIPT.matcher(javaScript); + assertTrue("The script holds no JSON: " + javaScript, matcher.find()); + return (JSONObject) new JSONTokener( + javaScript.substring(matcher.end())) + .nextValue(); + } + + private static HttpServletRequest request() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn("/51dpipeline/json"); + when(request.getHeaderNames()).thenReturn( + Collections.enumeration(Arrays.asList("Host"))); + when(request.getHeader("Host")).thenReturn("example.com"); + when(request.getRemoteAddr()).thenReturn("1.2.3.4"); + when(request.getScheme()).thenReturn("https"); + return request; + } + + private static HttpServletResponse response(StringWriter body) + throws Exception { + HttpServletResponse response = mock(HttpServletResponse.class); + when(response.getWriter()).thenReturn(new PrintWriter(body, true)); + return response; + } + + /** + * An element that adds a value, a Property assigned no value and a + * JavaScript Property, in place of an engine. + */ + private static class ValuesElement + extends FlowElementBase { + + ValuesElement() { + super(LoggerFactory.getLogger(ValuesElement.class), null); + } + + @Override + protected void processInternal(FlowData data) { + ElementDataBase values = + new ElementDataBase(logger, data) { }; + values.put("value", "a value"); + AspectPropertyValueDefault noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage("No value for the test."); + values.put("novalue", noValue); + values.put("javascript", "var a = 1;"); + data.getOrAdd(getElementDataKey(), d -> values); + } + + @Override + public String getElementDataKey() { + return "values"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + return Arrays.asList( + new ElementPropertyMetaDataDefault( + "value", this, "test", String.class, true), + new ElementPropertyMetaDataDefault( + "novalue", this, "test", String.class, true), + new ElementPropertyMetaDataDefault( + "javascript", this, "test", String.class, true)); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } +} diff --git a/pipeline/pom.xml b/pipeline/pom.xml index 533f13e51..bd4a7d7e5 100644 --- a/pipeline/pom.xml +++ b/pipeline/pom.xml @@ -80,6 +80,7 @@ 3.0.2 0.9.14 + 1.0 22.0.0.2 22.0.0.2 From e2d9a0125630f4a31e0e21b69f2114fd33bbb469 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 21:13:34 +0000 Subject: [PATCH 38/42] Share every query value by default, and no cookie but 51D_ ones even when all Evidence is shared (#160) Usage sharing now shares every query string and HTTP form value by default, and shares no cookie but the `51D_` ones even when all Evidence is shared, as [specifications#155](https://github.com/51Degrees/specifications/pull/155) now says. --- .../data/EvidenceKeyFilterShareUsage.java | 40 +++--- .../EvidenceKeyFilterShareUsageTracker.java | 3 +- .../fiftyone/flowelements/Constants.java | 7 ++ .../fiftyone/flowelements/ShareUsageBase.java | 5 +- .../flowelements/ShareUsageBuilderBase.java | 15 ++- .../flowelements/ShareUsageElement.java | 7 +- .../EvidenceKeyFilterShareUsageTests.java | 118 ++++++++++++++++-- .../flowelements/ShareUsageElementTests.java | 23 ++-- .../flowelements/ShareUsageOptionsTests.java | 17 ++- 9 files changed, 188 insertions(+), 47 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 9783c4077..ac8a5b99e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -33,6 +33,7 @@ import java.util.Set; import static fiftyone.pipeline.core.Constants.*; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS; /** * This filter is used by the {@link ShareUsageElement} to decide which of the @@ -44,12 +45,13 @@ * 2) A cookie that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX} or is * the session cookie (if configured in the constructor). * 3) A session value that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX}. - * 4) A query string parameter that starts with + * 4) A query string parameter, when all query string parameters are shared, + * which is the default. Otherwise one that starts with * {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be * shared using the constructor parameter. * 5) Not a header, cookie, session value or query string parameter. - * When all Evidence is shared, everything is included. When all query string - * parameters are shared, every query string parameter is included. + * When all Evidence is shared, everything is included but cookies, which + * follow rule 2 whatever the configuration. * The prefix of a key is everything before its first '.', and the field * everything after it. An email address, the salt for hashed email * identifiers, the query string, the path and the Cookie header are never @@ -112,12 +114,14 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { }; /** - * Constructor. + * Constructor. All Evidence is not shared, and all query string + * parameters are, as are the defaults. * @param blockedHttpHeaders a list of the names of the HTTP headers that * share usage should not send to 51Degrees * @param includedQueryStringParams a list of the names of query string * parameters that share usage should send - * to 51Degrees + * to 51Degrees when not all of them are + * shared * @param includeSession if true then the session cookie will be included in * the filter * @param sessionCookieName the name of the cookie that contains the session @@ -134,7 +138,7 @@ public EvidenceKeyFilterShareUsage( includeSession, sessionCookieName, false, - false); + SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS); } /** @@ -143,13 +147,15 @@ public EvidenceKeyFilterShareUsage( * share usage should not send to 51Degrees * @param includedQueryStringParams a list of the names of query string * parameters that share usage should send - * to 51Degrees + * to 51Degrees when not all of them are + * shared * @param includeSession if true then the session cookie will be included in * the filter * @param sessionCookieName the name of the cookie that contains the session * id * @param shareAllEvidence if true then all Evidence is included, apart from - * the keys that are never shared + * the keys that are never shared and the cookies + * other than the 51Degrees ones * @param shareAllQueryStringParameters if true then all query string * parameters are included */ @@ -197,12 +203,22 @@ public boolean include(String key) { } } + int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); + + // Only the 51Degrees cookies are shared, even when all Evidence is. + // The session cookie is included only for the tracker to compare. + if (separator > 0 && + EVIDENCE_COOKIE_PREFIX.equals(lowerKey.substring(0, separator))) { + String field = lowerKey.substring(separator + 1); + return field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + (includeSession && field.equals(sessionCookieName)); + } + if (shareAllEvidence) { return true; } boolean result; - int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); if (separator > 0) { String prefix = lowerKey.substring(0, separator); String field = lowerKey.substring(separator + 1); @@ -212,12 +228,6 @@ public boolean include(String key) { // appear in the list of blocked headers. result = blockedHttpHeaders.contains(field) == false; break; - case EVIDENCE_COOKIE_PREFIX: - // Only add cookies that start with the 51Degrees cookie - // prefix. - result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || - (includeSession && field.equals(sessionCookieName)); - break; case EVIDENCE_SESSION_PREFIX: // Only add session values that start with the 51Degrees cookie // prefix. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java index 61c2c348c..e39ebbff6 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java @@ -69,7 +69,8 @@ public EvidenceKeyFilterShareUsageTracker( * @param sessionCookieName the name of the cookie that contains the session * id * @param shareAllEvidence if true then all Evidence is included, apart from - * the keys that are never shared + * the keys that are never shared and the cookies + * other than the 51Degrees ones * @param shareAllQueryStringParameters if true then all query string * parameters are included */ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index 0641be77e..d68f5f0e1 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -112,6 +112,13 @@ public class Constants { */ static final String SHARE_USAGE_DEFAULT_URL = "https://devices-v4.51degrees.com/new.ashx"; + /** + * Every query string and HTTP form parameter is shared by default, not + * only the 51Degrees ones and those named. + */ + public static final boolean + SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS = true; + /** * Constant Added to blocked HTTP Headers so that the cookie header is * blocked by default. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 48d28f657..964fc9936 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -462,7 +462,7 @@ protected ShareUsageBase( Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT, false, - false); + Constants.SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS); } /** @@ -507,7 +507,8 @@ protected ShareUsageBase( * @param recoveryMultiplier what the pause is multiplied by each time the * send tried when a pause ends fails * @param shareAllEvidence if true then all Evidence is shared, apart from - * the keys that are never shared + * the keys that are never shared and the cookies + * other than the 51Degrees ones * @param shareAllQueryStringParameters if true then all query string * parameters are shared */ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index 6481ad1f5..6865a52fd 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -65,7 +65,8 @@ public abstract class ShareUsageBuilderBase { protected double maxRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; protected double recoveryMultiplier = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; protected boolean shareAllEvidence; - protected boolean shareAllQueryStringParameters; + protected boolean shareAllQueryStringParameters = + Constants.SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS; /** * Constructor @@ -154,13 +155,14 @@ public ShareUsageBuilderBase setIncludedQueryStringParameter( * If set to true, every query string and HTTP form parameter is shared, * rather than only those that start with 51D_ or are named in the * included parameters. The whole query string and the parameters that are - * never shared are still not shared. + * never shared are still not shared. Set it to false to share only the + * 51D_ parameters and those named. *

- * Default value is false + * Default value is true * @param shareAll true to share all query string parameters * @return this builder */ - @DefaultValue(booleanValue = false) + @DefaultValue(booleanValue = true) public ShareUsageBuilderBase setShareAllQueryStringParameters( boolean shareAll) { this.shareAllQueryStringParameters = shareAll; @@ -177,10 +179,11 @@ public boolean isShareAllQueryStringParameters() { /** * If set to true, all Evidence is shared, which overrides the blocked - * HTTP headers and the rules for cookies, session values and query string + * HTTP headers and the rules for session values and query string * parameters. The keys that are never shared (an email address, the salt * for hashed email identifiers, the whole query string, the path and the - * Cookie header) are still not shared. + * Cookie header) are still not shared, and nor is any cookie whose name + * does not start with 51D_. *

* Default value is false * @param shareAll true to share all Evidence diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index bc40513d0..6db246f13 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -43,6 +43,7 @@ import java.util.zip.GZIPOutputStream; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; @@ -332,7 +333,7 @@ public class ShareUsageElement extends ShareUsageBase { SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT, false, - false); + SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS); } /** @@ -377,7 +378,9 @@ public class ShareUsageElement extends ShareUsageBase { * @param recoveryMultiplier what the pause is multiplied by each time the send * tried when a pause ends fails * @param shareAllEvidence if true then all Evidence is shared, apart from - * the keys that are never shared + * the keys that are never shared and + * the cookies other than the + * 51Degrees ones * @param shareAllQueryStringParameters if true then all query string * parameters are shared */ diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java index 8aeb6e7f8..34ca22e2a 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -56,6 +56,35 @@ private static EvidenceKeyFilterShareUsage defaults() { false); } + private static EvidenceKeyFilterShareUsage filter( + boolean includeSession, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { + return new EvidenceKeyFilterShareUsage( + Collections.emptyList(), + Collections.emptyList(), + includeSession, + DEFAULT_SESSION_COOKIE_NAME, + shareAllEvidence, + shareAllQueryStringParameters); + } + + /** + * Shares only the 51D_ query string parameters and those named, as when + * share all query string parameters is turned off. + */ + private static EvidenceKeyFilterShareUsage namedParametersOnly( + List blockedHttpHeaders, + List includedQueryStringParams) { + return new EvidenceKeyFilterShareUsage( + blockedHttpHeaders, + includedQueryStringParams, + false, + DEFAULT_SESSION_COOKIE_NAME, + false, + false); + } + /** * By default headers other than Cookie are shared, cookies, session * values and query string parameters only when their names start with @@ -73,6 +102,11 @@ public void webRequestIsNeverIncluded() { assertFalse(filter.include("Server.Http-Request")); } + /** + * By default every header but Cookie, every query string parameter but + * those never shared, and any other evidence are shared, and cookies and + * session values only when their names start with 51D_. + */ @Test public void defaultRules() { EvidenceKeyFilterShareUsage filter = defaults(); @@ -87,22 +121,93 @@ public void defaultRules() { assertFalse(filter.include("session.org.example.attribute")); assertTrue(filter.include("query.51d_gethighentropyvalues")); assertTrue(filter.include("query.51D_Test")); - assertFalse(filter.include("query.password")); + assertTrue(filter.include("query.password")); + assertTrue(filter.include("Query.Utm_Source")); assertFalse(filter.include("query.id.email")); + assertFalse(filter.include("server.request-query")); assertTrue(filter.include("server.client-ip")); assertTrue(filter.include("nodot")); } + /** + * With share all query string parameters off, only the 51D_ parameters + * and those named are shared. + */ + @Test + public void shareAllQueryStringParametersOff_OnlyNamedAnd51D() { + EvidenceKeyFilterShareUsage filter = namedParametersOnly( + Collections.emptyList(), + Collections.singletonList("campaign")); + + assertFalse(filter.include("query.password")); + assertTrue(filter.include("query.campaign")); + assertTrue(filter.include("query.51D_Test")); + assertTrue(filter.include("header.user-agent")); + } + + /** + * Even when all Evidence is shared, a cookie is shared only when its + * name starts with 51D_, whatever its case. The session cookie is kept + * only by a filter that includes it for the tracker to compare. + */ + @Test + public void shareAllEvidence_NoCookieBut51D() { + EvidenceKeyFilterShareUsage sent = filter(false, true, true); + EvidenceKeyFilterShareUsage kept = filter(true, true, true); + + for (EvidenceKeyFilterShareUsage filter : Arrays.asList(sent, kept)) { + assertTrue(filter.include("cookie.51D_Profile")); + assertTrue(filter.include("cookie.51d_screenpixelswidth")); + assertFalse(filter.include("cookie._ga")); + assertFalse(filter.include("Cookie.Other")); + assertFalse(filter.include("cookie.a.b")); + assertTrue(filter.include("session.org.example.attribute")); + assertTrue(filter.include("header.x-anything")); + assertTrue(filter.include("query.password")); + } + assertFalse(sent.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertTrue(kept.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + } + + /** + * With all Evidence shared, the tracker compares the session cookie only + * when track session is on, and no other cookie but the 51D_ ones. + */ + @Test + public void trackerWithShareAllEvidence_SessionCookieOnlyWhenTracked() { + EvidenceKeyFilterShareUsageTracker tracked = + new EvidenceKeyFilterShareUsageTracker( + Collections.emptyList(), + Collections.emptyList(), + true, + DEFAULT_SESSION_COOKIE_NAME, + true, + true); + EvidenceKeyFilterShareUsageTracker untracked = + new EvidenceKeyFilterShareUsageTracker( + Collections.emptyList(), + Collections.emptyList(), + false, + DEFAULT_SESSION_COOKIE_NAME, + true, + true); + + assertTrue(tracked.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertFalse(untracked.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertFalse(tracked.include("cookie._ga")); + assertFalse(untracked.include("cookie._ga")); + assertTrue(untracked.include("cookie.51D_Profile")); + } + /** * A name containing a dot is matched as a whole against the parameters to * share. */ @Test public void fieldIsEverythingAfterTheFirstDot() { - EvidenceKeyFilterShareUsage filter = filter( + EvidenceKeyFilterShareUsage filter = namedParametersOnly( Collections.emptyList(), - Collections.singletonList("valid.email"), - false); + Collections.singletonList("valid.email")); assertTrue(filter.include("query.valid.email")); assertFalse(filter.include("query.valid")); @@ -115,10 +220,9 @@ public void fieldIsEverythingAfterTheFirstDot() { */ @Test public void listsIgnoreCase() { - EvidenceKeyFilterShareUsage filter = filter( + EvidenceKeyFilterShareUsage filter = namedParametersOnly( Collections.singletonList("X-Secret"), - Collections.singletonList("Campaign"), - false); + Collections.singletonList("Campaign")); assertFalse(filter.include("header.x-secret")); assertTrue(filter.include("query.campaign")); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index 8107c8114..c8f6be4df 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -514,7 +514,7 @@ public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() request.put("header.X-Blocked", "blocked"); request.put("cookie.51D_Profile", "123456"); request.put("cookie.other", "not-shared"); - request.put("query.secret", "not-shared"); + request.put("query.campaign", "shared-query"); request.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); request.put("server.request-method", "GET"); request.put("server.request-path", "/a%2Fb/c"); @@ -554,9 +554,13 @@ public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() "//cookie[@Name='51D_Profile']", doc), "A 51D_ cookie from the web request was not shared."); assertEquals(0, ((NodeList) xpath.evaluate( - "//cookie[@Name='other'] | //query[@Name='secret']", doc, + "//cookie[@Name='other']", doc, XPathConstants.NODESET)).getLength(), - "A cookie or query string parameter the rules leave out was shared."); + "A cookie the rules leave out was shared."); + assertEquals("shared-query", xpath.evaluate( + "//query[@Name='campaign']", doc), + "A query string parameter from the web request was not " + + "shared by default."); assertEquals("1.2.3.4", xpath.evaluate("//ClientIP", doc), "The client IP from the web request was not shared."); assertEquals("GET", xpath.evaluate( @@ -1257,10 +1261,10 @@ public void CheckForContinueAndResumeWhenQueueFull() throws Exception { } /** - * Default settings share headers other than Cookie, the 51Degrees cookies - * and query string parameters, and other evidence, and leave out ordinary - * cookies, other parameters, session values and the evidence that is - * never shared. + * Default settings share headers other than Cookie, the 51Degrees cookies, + * every query string parameter but those never shared, and other + * evidence, and leave out ordinary cookies, session values and the + * evidence that is never shared. */ @Test public void ShareUsageElement_DefaultSharesOnlyAcceptedEvidence() throws Exception { @@ -1301,11 +1305,12 @@ public void ShareUsageElement_DefaultSharesOnlyAcceptedEvidence() throws Excepti String output = new String( connector.getBaos().toByteArray(), StandardCharsets.UTF_8); for (String shared : Arrays.asList( - "some user agent", "screen width", "fiftyone parameter")) { + "some user agent", "screen width", "fiftyone parameter", + "password parameter")) { assertTrue(output.contains(shared), shared); } for (String withheld : Arrays.asList( - "sessionid cookie", "cookie header", "password parameter", + "sessionid cookie", "cookie header", "email parameter", "email header", "salt value", "whole query string", "request path", "session attribute")) { assertFalse(output.contains(withheld), withheld); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java index 3bc53524f..fe71cf40f 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java @@ -127,6 +127,7 @@ private static Map evidence() { evidence.put("header.x-blocked", "blocked"); evidence.put("header.cookie", "a=b"); evidence.put("cookie.plain", "cookie"); + evidence.put("cookie.51D_plain", "51d-cookie"); evidence.put("session.plain", "session"); evidence.put("query.plain", "query"); evidence.put("query.id.email", "a@example.com"); @@ -178,8 +179,9 @@ public void defaults_ShareOnlyWhatTheRulesAllow() throws Exception { assertTrue(shared(sent, "header", "user-agent")); assertFalse(shared(sent, "header", "x-blocked")); assertFalse(shared(sent, "cookie", "plain")); + assertTrue(shared(sent, "cookie", "51D_plain")); assertFalse(shared(sent, "session", "plain")); - assertFalse(shared(sent, "query", "plain")); + assertTrue(shared(sent, "query", "plain")); } @Test @@ -189,7 +191,8 @@ public void shareAllEvidence_FromConfiguration_SharesEverythingButNeverShared() assertTrue(shared(sent, "header", "user-agent")); assertTrue(shared(sent, "header", "x-blocked")); - assertTrue(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "cookie", "plain")); + assertTrue(shared(sent, "cookie", "51D_plain")); assertTrue(shared(sent, "session", "plain")); assertTrue(shared(sent, "query", "plain")); assertFalse(shared(sent, "query", "id.email")); @@ -207,7 +210,7 @@ public void shareAllEvidence_False_FromConfiguration_KeepsTheRules() assertFalse(shared(sent, "header", "x-blocked")); assertFalse(shared(sent, "cookie", "plain")); - assertFalse(shared(sent, "query", "plain")); + assertTrue(shared(sent, "query", "plain")); } @Test @@ -237,13 +240,17 @@ public void shareAllQueryStringParameters_False_FromConfiguration_KeepsTheRules( assertFalse(shared(sent, "query", "plain")); } + /** + * Share all Evidence is off by default and share all query string + * parameters is on. + */ @Test - public void builder_ShareOptionsDefaultToFalse() { + public void builder_ShareOptionDefaults() { ShareUsageBuilder builder = new ShareUsageBuilder( LoggerFactory.getILoggerFactory()); assertFalse(builder.isShareAllEvidence()); - assertFalse(builder.isShareAllQueryStringParameters()); + assertTrue(builder.isShareAllQueryStringParameters()); } @Test From 933e42495b5c762587efd68c660f0e4c981f44af Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 21:14:05 +0000 Subject: [PATCH 39/42] Lock every read of IP Intelligence native results, type Latitude and Longitude as Float, and process cloud IP Intelligence with the shared engine logic (#177) The IP Intelligence packages differed from the specification in three places, and eight more rules had no test. --- .../IPIntelligenceCloudEngine.java | 212 ++++++++---------- .../flowelements/CloudRequestFailedTests.java | 11 +- .../cloud/flowelements/DataModelTests.java | 100 +++++++++ .../IPIntelligenceCloudEngineTests.java | 12 +- .../MissingPropertyHandlingTests.java | 14 +- .../flowelements/ProductMissingTests.java | 10 +- .../SharedCloudProcessingTests.java | 154 +++++++++++++ .../onpremise/data/PropertyMetaDataIPI.java | 5 + .../IPIntelligenceDataHashDefault.java | 21 +- .../data/PropertyMetaDataIPITypeTests.java | 78 +++++++ .../AcceptedEvidenceRefreshTests.java | 112 +++++++++ .../flowelements/ClientIpChoiceTests.java | 115 ++++++++++ .../flowelements/DataModelTests.java | 72 ++++++ .../flowelements/EngineNamesTests.java | 132 +++++++++++ .../flowelements/ProcessMarshallingTests.java | 133 +++++++++++ .../ProcessReadsNoValueTests.java | 97 ++++++++ .../PropertyIndexLaterDataSetTests.java | 114 ++++++++++ .../ResultsReadByOneThreadTests.java | 189 ++++++++++++++++ .../OnPremiseEngineNamesTests.java | 110 +++++++++ 19 files changed, 1543 insertions(+), 148 deletions(-) create mode 100644 ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/DataModelTests.java create mode 100644 ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/SharedCloudProcessingTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPITypeTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AcceptedEvidenceRefreshTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ClientIpChoiceTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataModelTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineNamesTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessMarshallingTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessReadsNoValueTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexLaterDataSetTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ResultsReadByOneThreadTests.java create mode 100644 ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremiseEngineNamesTests.java diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java index c1cfe7073..44c42912a 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.cloud.flowelements; import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; -import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudAspectEngineBase; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData; @@ -32,7 +31,6 @@ import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.ElementDataFactory; import fiftyone.pipeline.core.data.types.JavaScript; -import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; @@ -68,7 +66,7 @@ public class IPIntelligenceCloudEngine private String dataSourceTier; private final Object metadataLock = new Object(); // Lower-cased names of properties the cloud returns as weighted lists - // (cloud type "Weighted..."), so processEngine can read their value/weight + // (cloud type "Weighted..."), so processCloudEngine can read their value/weight // pairs rather than treating them as a plain list. private final Set weightedPropertyNames = new HashSet<>(); CloudRequestEngine cloudRequestEngine; @@ -136,128 +134,114 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { return new EvidenceKeyFilterWhitelist(new ArrayList()); } + /** + * Populate the Element Data from this engine's part of the cloud + * response. The shared Cloud Aspect Engine processing finds the Cloud + * Request Engine's data, checks the request ran, and calls this only when + * there is a response. + * @param data the Flow Data being processed + * @param aspectData the Element Data to populate + * @param json the JSON response from the {@link CloudRequestEngine} + */ @Override - protected void processEngine(FlowData data, IPIntelligenceDataCloud aspectData) { - if (cloudRequestEngine == null) { - throw new PipelineConfigurationException( - "The '" + getClass().getName() + "' requires a " + - "'CloudRequestEngine' before it in the Pipeline. This " + - "engine will be unable to produce results until this is " + - "corrected."); - } - else { - CloudRequestData requestData = - data.getFromElement(cloudRequestEngine); - setCloudRequestFailed(data, aspectData, cloudRequestEngine); - String json = requestData.getJsonResponse(); - - // A failed request can leave no response, which the Cloud - // Request Engine has reported. - if (json == null || json.isEmpty()) { - logger.warn("The " + getClass().getSimpleName() + " did " + - "not process as the JSON response from the " + - "CloudRequestEngine was null or empty. Please refer to " + - "errors generated by the CloudRequestEngine in the logs " + - "as this indicates an error occurred there."); - return; + protected void processCloudEngine( + FlowData data, + IPIntelligenceDataCloud aspectData, + String json) { + JSONObject jsonObj = new JSONObject(json); + JSONObject ipiObj = getElementJson(jsonObj); + + Map propertyMap = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + for (AspectPropertyMetaData property : getProperties()) { + // A property the response leaves out has no entry, so + // reading it throws a PropertyMissingException. + if (ipiObj.has( + property.getName().toLowerCase(Locale.ROOT)) == false) { + continue; } - - // Extract data from json to the aspectData instance. - JSONObject jsonObj = new JSONObject(json); - JSONObject ipiObj = getElementJson(jsonObj); - - Map propertyMap = - new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - - for (AspectPropertyMetaData property : getProperties()) { - // A property the response leaves out has no entry, so - // reading it throws a PropertyMissingException. - if (ipiObj.has( - property.getName().toLowerCase(Locale.ROOT)) == false) { - continue; - } - if (weightedPropertyNames.contains( - property.getName().toLowerCase())) { + if (weightedPropertyNames.contains( + property.getName().toLowerCase())) { + propertyMap.put( + property.getName(), + getWeightedListAspectPropertyValue(ipiObj, property)); + continue; + } + String type = property.getType().getSimpleName(); + switch(type) { + case ("List"): propertyMap.put( property.getName(), - getWeightedListAspectPropertyValue(ipiObj, property)); - continue; - } - String type = property.getType().getSimpleName(); - switch(type) { - case ("List"): - propertyMap.put( - property.getName(), - getListAspectPropertyValue(ipiObj, property)); - break; - case ("JavaScript"): - propertyMap.put( - property.getName(), - getJavaScriptAspectPropertyValue(ipiObj, property)); - break; - case ("String"): - propertyMap.put( - property.getName(), - getStringAspectPropertyValue(ipiObj, property)); - break; - case ("InetAddress"): - propertyMap.put( - property.getName(), - getInetAddressAspectPropertyValue(ipiObj, property)); - break; - case ("WktString"): - propertyMap.put( - property.getName(), - getWktStringAspectPropertyValue(ipiObj, property)); - break; - case ("boolean"): - propertyMap.put( - property.getName(), - getBooleanAspectPropertyValue(ipiObj, property)); - break; - case ("int"): - propertyMap.put( - property.getName(), - getIntegerAspectPropertyValue(ipiObj, property)); - break; - case ("double"): - propertyMap.put( - property.getName(), - getDoubleAspectPropertyValue(ipiObj, property)); + getListAspectPropertyValue(ipiObj, property)); + break; + case ("JavaScript"): + propertyMap.put( + property.getName(), + getJavaScriptAspectPropertyValue(ipiObj, property)); + break; + case ("String"): + propertyMap.put( + property.getName(), + getStringAspectPropertyValue(ipiObj, property)); break; - case ("Float"): - propertyMap.put( - property.getName(), - getFloatAspectPropertyValue(ipiObj, property)); + case ("InetAddress"): + propertyMap.put( + property.getName(), + getInetAddressAspectPropertyValue(ipiObj, property)); + break; + case ("WktString"): + propertyMap.put( + property.getName(), + getWktStringAspectPropertyValue(ipiObj, property)); + break; + case ("boolean"): + propertyMap.put( + property.getName(), + getBooleanAspectPropertyValue(ipiObj, property)); + break; + case ("int"): + propertyMap.put( + property.getName(), + getIntegerAspectPropertyValue(ipiObj, property)); + break; + case ("double"): + propertyMap.put( + property.getName(), + getDoubleAspectPropertyValue(ipiObj, property)); + break; + case ("Float"): + propertyMap.put( + property.getName(), + getFloatAspectPropertyValue(ipiObj, property)); + break; + default: + propertyMap.put( + property.getName(), + getStringAspectPropertyValue(ipiObj, property)); break; - default: - propertyMap.put( - property.getName(), - getStringAspectPropertyValue(ipiObj, property)); - break; - } } + } - // Add no value messages for any properties which have not yet been - // populated. - for (String key : ipiObj.keySet()) { - if (key.endsWith("nullreason")) { - String actualKey = key - .replace("nullreason", ""); - if (propertyMap.containsKey(actualKey) == false) { - AspectPropertyValue nullValue = - new AspectPropertyValueDefault(); - nullValue.setNoValueMessage( - ipiObj.optString(key, "Unknown")); - propertyMap.put( - actualKey, - nullValue); - } + // Add no value messages for any properties which have not yet been + // populated. + for (String key : ipiObj.keySet()) { + if (key.endsWith("nullreason")) { + String actualKey = key + .replace("nullreason", ""); + if (propertyMap.containsKey(actualKey) == false) { + AspectPropertyValue nullValue = + new AspectPropertyValueDefault(); + nullValue.setNoValueMessage( + ipiObj.optString(key, "Unknown")); + propertyMap.put( + actualKey, + nullValue); } } - - aspectData.populateFromMap(propertyMap); } + + aspectData.populateFromMap(propertyMap); } @Override diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java index ed50ea685..3af3407df 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java @@ -29,11 +29,14 @@ import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentMatchers; import org.slf4j.ILoggerFactory; import org.slf4j.Logger; @@ -149,9 +152,13 @@ private IPIntelligenceDataCloud process( flowData, engine, MissingPropertyServiceDefault.getInstance()); + when(flowData.getOrAdd( + ArgumentMatchers.>any(), + ArgumentMatchers.>any())) + .thenReturn(data); try { - engine.processEngine(flowData, data); - } catch (RuntimeException e) { + engine.process(flowData); + } catch (Exception e) { // A failed request's response may not be readable. The // property is read afterwards either way, as it is through a // Pipeline that suppresses process exceptions. diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/DataModelTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/DataModelTests.java new file mode 100644 index 000000000..7bfc29912 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/DataModelTests.java @@ -0,0 +1,100 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The cloud IP Intelligence engine populates Aspect Data, which is the IP + * Intelligence Element Data type. The engine processes a fixed response, + * so the test needs no resource key. + */ +public class DataModelTests { + + @Test + @SuppressWarnings("unchecked") + public void Engine_PopulatesAspectData() throws Exception { + Logger logger = mock(Logger.class); + ILoggerFactory loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + + try (IPIntelligenceCloudEngine engine = + new IPIntelligenceCloudEngineBuilder(loggerFactory).build()) { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredname", "String", "cat", null)); + Map products = new HashMap<>(); + products.put(engine.getElementDataKey(), product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + engine.addPipeline(pipeline); + + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn( + "{ \"ip\": { \"registeredname\": \"Example\" } }"); + when(cloudData.getProcessStarted()).thenReturn(true); + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + when(flowData.getOrAdd(any(TypedKey.class), any())) + .thenReturn(data); + engine.process(flowData); + + assertTrue(data instanceof AspectData); + assertTrue(data instanceof IPIntelligenceData); + } + } +} diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngineTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngineTests.java index 697af930d..faa9e6438 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngineTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngineTests.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.cloud.flowelements; import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; -import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; @@ -46,7 +45,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -205,19 +203,13 @@ private IPIntelligenceCloudEngine buildEngine(PropertyMetaData... properties) } /** - * Run {@code processEngine} against a mocked cloud JSON response and return + * Run {@code processCloudEngine} against a cloud JSON response and return * the populated data instance. */ private IPIntelligenceDataCloud process( IPIntelligenceCloudEngine engine, String json) { - CloudRequestData cloudData = mock(CloudRequestData.class); - when(cloudData.getJsonResponse()).thenReturn(json); - FlowData flowData = mock(FlowData.class); - when(flowData.getFromElement(any(CloudRequestEngine.class))) - .thenReturn(cloudData); - IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( loggerFactory.getLogger( IPIntelligenceDataCloud.class.getSimpleName()), @@ -225,7 +217,7 @@ private IPIntelligenceDataCloud process( engine, mock(MissingPropertyService.class)); - engine.processEngine(flowData, data); + engine.processCloudEngine(flowData, data, json); return data; } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java index e212f527f..c2f853c1b 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.cloud.flowelements; import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; -import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; @@ -63,7 +62,6 @@ public void PropertyInResource_NullValue() throws Exception { .build()) { FlowData flowData = mock(FlowData.class); - addResponse(flowData, nullValueJson); IPIntelligenceDataCloud ipiData = new IPIntelligenceDataCloudInternal( loggerFactory.getLogger(IPIntelligenceDataCloud.class.getSimpleName()), @@ -73,7 +71,7 @@ public void PropertyInResource_NullValue() throws Exception { engine.cloudRequestEngine = mock(CloudRequestEngine.class); engine.aspectProperties = properties; - engine.processEngine(flowData, ipiData); + engine.processCloudEngine(flowData, ipiData, nullValueJson); AspectPropertyValue latitude = ipiData.getLatitude(); assertNotNull(latitude); @@ -98,7 +96,6 @@ public void PropertyNotInResource() throws Exception { .build()) { FlowData flowData = mock(FlowData.class); - addResponse(flowData, nullValueJson); MissingPropertyService missingPropertyService = mock(MissingPropertyService.class); when(missingPropertyService.getMissingPropertyReason( @@ -116,7 +113,7 @@ public void PropertyNotInResource() throws Exception { engine.cloudRequestEngine = mock(CloudRequestEngine.class); engine.aspectProperties = properties; - engine.processEngine(flowData, ipiData); + engine.processCloudEngine(flowData, ipiData, nullValueJson); try { AspectPropertyValue longitude = ipiData.getLongitude(); @@ -155,11 +152,4 @@ public void PropertyNotInResource() throws Exception { add(new AspectPropertyMetaDataDefault("registeredcountry", null, null, String.class, null, true)); } }; - - private void addResponse(FlowData data, String json) { - CloudRequestData cloudData = mock(CloudRequestData.class); - when(cloudData.getJsonResponse()).thenReturn(json); - when(data.getFromElement(any(CloudRequestEngine.class))) - .thenReturn(cloudData); - } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java index 1c0d68e37..8f816019c 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.cloud.flowelements; import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; -import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; @@ -51,7 +50,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -248,20 +246,14 @@ private IPIntelligenceCloudEngine buildEngine( private IPIntelligenceDataCloud process( IPIntelligenceCloudEngine engine, String response) { - CloudRequestData cloudData = mock(CloudRequestData.class); - when(cloudData.getJsonResponse()).thenReturn(response); - FlowData flowData = mock(FlowData.class); - when(flowData.getFromElement(any(CloudRequestEngine.class))) - .thenReturn(cloudData); - IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( loggerFactory.getLogger( IPIntelligenceDataCloud.class.getSimpleName()), flowData, engine, MissingPropertyServiceDefault.getInstance()); - engine.processEngine(flowData, data); + engine.processCloudEngine(flowData, data, response); return data; } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/SharedCloudProcessingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/SharedCloudProcessingTests.java new file mode 100644 index 000000000..8f7ed6af5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/SharedCloudProcessingTests.java @@ -0,0 +1,154 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The cloud IP Intelligence engine leaves its processing to the shared Cloud + * Aspect Engine logic, which finds the Cloud Request Engine's data, checks + * the request ran, and passes on only a response that is there, so the + * engine itself only reads its part of the response. The engine is driven + * through its public process method with a fixed response, so the tests + * need no resource key. + */ +public class SharedCloudProcessingTests { + + private final Logger logger = mock(Logger.class); + + private final CloudRequestData cloudData = mock(CloudRequestData.class); + + private final FlowData flowData = mock(FlowData.class); + + private IPIntelligenceCloudEngine engine; + + private IPIntelligenceDataCloud data; + + @Before + @SuppressWarnings("unchecked") + public void init() throws Exception { + ILoggerFactory loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + engine = new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredname", "String", "cat", null)); + Map products = new HashMap<>(); + products.put(engine.getElementDataKey(), product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + engine.addPipeline(pipeline); + + data = new IPIntelligenceDataCloudInternal( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getOrAdd(any(TypedKey.class), any())) + .thenReturn(data); + } + + @After + public void cleanup() throws Exception { + engine.close(); + } + + @Test + public void Engine_DeclaresOnlyTheResponseReading() throws Exception { + for (Method method : + IPIntelligenceCloudEngine.class.getDeclaredMethods()) { + if (method.getName().equals("processEngine") && + method.isSynthetic() == false) { + fail("The engine should leave processEngine to the shared " + + "Cloud Aspect Engine logic."); + } + } + IPIntelligenceCloudEngine.class.getDeclaredMethod( + "processCloudEngine", + FlowData.class, + IPIntelligenceDataCloud.class, + String.class); + } + + @Test + public void Response_Read() throws Exception { + when(cloudData.getProcessStarted()).thenReturn(true); + when(cloudData.getJsonResponse()).thenReturn( + "{ \"ip\": { \"registeredname\": \"Example\" } }"); + engine.process(flowData); + assertEquals("Example", data.getRegisteredName().getValue()); + } + + @Test + public void NoResponse_WarnsAndStops() throws Exception { + when(cloudData.getProcessStarted()).thenReturn(true); + when(cloudData.getJsonResponse()).thenReturn(null); + engine.process(flowData); + verify(logger).warn(contains("did not process")); + assertTrue(data.asKeyMap().isEmpty()); + } + + @Test(expected = PipelineConfigurationException.class) + public void RequestNotRun_Throws() throws Exception { + when(cloudData.getProcessStarted()).thenReturn(false); + when(cloudData.getJsonResponse()).thenReturn( + "{ \"ip\": { \"registeredname\": \"Example\" } }"); + engine.process(flowData); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java index eabb2aecb..bcd58b4de 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java @@ -206,6 +206,11 @@ public Class getType() { return Boolean.class; case "double": return Double.class; + // A 32 bit floating point number, as Latitude and Longitude + // are, which the data model types as float. + case "single": + case "weightedsingle": + return Float.class; case "javascript": return JavaScript.class; case "string[]": diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java index 6ffe44a61..b2b6cfcf0 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java @@ -74,6 +74,12 @@ public class IPIntelligenceDataHashDefault */ private final IPIntelligenceOnPremiseEngine ipiEngine; + /** + * Held for every read from {@link #resultsList}, so that only one thread + * at a time reads from the native results. + */ + private final Object resultsLock = new Object(); + /** * Constructs a new instance. * @param logger used for logging @@ -189,6 +195,13 @@ void populateWeightedValues( public AspectPropertyValue> getValues(String propertyName, Class[] parametrizedTypes) { checkState(); + synchronized (resultsLock) { + return readValues(propertyName, parametrizedTypes); + } + } + + private AspectPropertyValue> readValues(String propertyName, + Class[] parametrizedTypes) { AspectPropertyValue> result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); @@ -401,13 +414,19 @@ protected AspectPropertyValue getValueAsDouble(String propertyName) { return result; } + /** + * Holds the results lock for the whole read, as finding whether a + * property is available reads the native results too. + */ @Override protected TryGetResult tryGetValue( String key, Class type, Class... parameterisedTypes) { checkState(); - return super.tryGetValue(key, type, parameterisedTypes); + synchronized (resultsLock) { + return super.tryGetValue(key, type, parameterisedTypes); + } } private void checkState() { diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPITypeTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPITypeTests.java new file mode 100644 index 000000000..cf897474d --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPITypeTests.java @@ -0,0 +1,78 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.data; + +import fiftyone.ipintelligence.engine.onpremise.interop.swig.PropertyMetaDataSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.pipeline.core.data.types.JavaScript; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The Java type a property's meta data gives for each type name the native + * meta data uses. The element data asks for a property's values in this + * type when the caller names no type, as get(name) does. + * + * The IP Intelligence data model gives Latitude and Longitude the type + * float (ip-intelligence-specification/data-model.md at + * 51Degrees/specifications 706de1f0), and their typed accessors return + * Float. The native meta data names them "single", a 32 bit floating point + * number, or "weightedsingle" for the weighted form. + */ +public class PropertyMetaDataIPITypeTests { + + private static Class typeOf(String nativeType) { + PropertyMetaDataSwig source = mock(PropertyMetaDataSwig.class); + when(source.getType()).thenReturn(nativeType); + when(source.getName()).thenReturn("Property"); + when(source.getDataFilesWherePresent()) + .thenReturn(mock(VectorStringSwig.class)); + return new PropertyMetaDataIPI(null, source).getType(); + } + + @Test + public void Single_IsFloat() { + assertEquals(Float.class, typeOf("single")); + } + + @Test + public void WeightedSingle_IsFloat() { + assertEquals(Float.class, typeOf("weightedsingle")); + } + + @Test + public void OtherNames_AreUnchanged() { + assertEquals(String.class, typeOf("string")); + assertEquals(Integer.class, typeOf("int")); + assertEquals(Boolean.class, typeOf("bool")); + assertEquals(Double.class, typeOf("double")); + assertEquals(JavaScript.class, typeOf("javascript")); + assertEquals(List.class, typeOf("string[]")); + assertEquals(String.class, typeOf("not a type name")); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AcceptedEvidenceRefreshTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AcceptedEvidenceRefreshTests.java new file mode 100644 index 000000000..5aad4686c --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AcceptedEvidenceRefreshTests.java @@ -0,0 +1,112 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * The engine reads the Evidence keys the data file accepts from the native + * engine when it starts and again whenever the data is refreshed. Both free + * data files accept the same keys, so the test puts a key no data file has + * in place of the keys before a refresh, and checks that the refresh + * replaces it with the keys the native engine gives. + */ +public class AcceptedEvidenceRefreshTests extends TestsBase { + + private static final String STALE_KEY = "query.stale-key"; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void StartUp_KeysAreTheNativeEngines() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + assertEquals(nativeKeys(engine), filterKeys(engine)); + } + + @Test + public void Refresh_ReadsTheKeysAgain() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + List stale = Collections.singletonList(STALE_KEY); + set(engine, "evidenceKeys", stale); + set(engine, "evidenceKeyFilter", new EvidenceKeyFilterWhitelist( + stale, String.CASE_INSENSITIVE_ORDER)); + + engine.refreshData(engine.getDataFileMetaData().getIdentifier()); + + TreeSet keys = filterKeys(engine); + assertFalse(keys.toString(), keys.contains(STALE_KEY)); + assertEquals(nativeKeys(engine), keys); + } + + private static TreeSet filterKeys( + IPIntelligenceOnPremiseEngine engine) { + Map whitelist = ((EvidenceKeyFilterWhitelist) + engine.getEvidenceKeyFilter()).getWhitelist(); + return new TreeSet<>(whitelist.keySet()); + } + + private static TreeSet nativeKeys( + IPIntelligenceOnPremiseEngine engine) throws Exception { + EngineIpiSwig nativeEngine = (EngineIpiSwig) get(engine, "engine"); + return new TreeSet<>(new ArrayList<>(nativeEngine.getKeys())); + } + + private static Object get(Object target, String name) throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } + + private static void set(Object target, String name, Object value) + throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ClientIpChoiceTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ClientIpChoiceTests.java new file mode 100644 index 000000000..b8561d5c5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ClientIpChoiceTests.java @@ -0,0 +1,115 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +/** + * When the Evidence holds more than one client IP key, one address is used, + * a query key ahead of a server key, and the Property values describe that + * address. + */ +public class ClientIpChoiceTests extends TestsBase { + + private static final String QUERY_KEY = "query.client-ip"; + + private static final String SERVER_KEY = "server.true-client-ip-51d"; + + private static final String QUERY_ADDRESS = "116.154.188.222"; + + private static final String SERVER_ADDRESS = "185.28.167.77"; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void QueryAndServerKeys_QueryAddressUsed() throws Exception { + assertAccepted(QUERY_KEY); + assertAccepted(SERVER_KEY); + Map query = values(QUERY_KEY, QUERY_ADDRESS); + Map server = values(SERVER_KEY, SERVER_ADDRESS); + assertNotEquals( + "Expected the two addresses to give different values.", + query, + server); + + assertEquals(query, values( + QUERY_KEY, QUERY_ADDRESS, SERVER_KEY, SERVER_ADDRESS)); + assertEquals(query, values( + SERVER_KEY, SERVER_ADDRESS, QUERY_KEY, QUERY_ADDRESS)); + } + + @Test + public void ServerKeyOnly_ServerAddressUsed() throws Exception { + assertAccepted(SERVER_KEY); + assertEquals( + values(QUERY_KEY, SERVER_ADDRESS), + values(SERVER_KEY, SERVER_ADDRESS)); + } + + private void assertAccepted(String key) { + assertTrue("Expected the engine to accept '" + key + "'.", + getWrapper().getEngine().getEvidenceKeyFilter().include(key)); + } + + /** + * Process Evidence given as key, value pairs and give the registered + * name, country and owner. + */ + private Map values(String... evidence) throws Exception { + Map values = new LinkedHashMap<>(); + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + for (int i = 0; i < evidence.length; i += 2) { + data.addEvidence(evidence[i], evidence[i + 1]); + } + data.process(); + IPIntelligenceData ipi = data.get(IPIntelligenceData.class); + values.put("RegisteredName", ipi.getRegisteredName().getValue()); + values.put("RegisteredCountry", + ipi.getRegisteredCountry().getValue()); + values.put("RegisteredOwner", + ipi.getRegisteredOwner().getValue()); + } + return values; + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataModelTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataModelTests.java new file mode 100644 index 000000000..de236bab5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataModelTests.java @@ -0,0 +1,72 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectData; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * The on-premise IP Intelligence engine populates Aspect Data, and the IP + * Intelligence Element Data type extends Aspect Data. + */ +public class DataModelTests extends TestsBase { + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void ElementDataType_ExtendsAspectData() { + assertTrue(AspectData.class.isAssignableFrom( + IPIntelligenceData.class)); + } + + @Test + public void Engine_PopulatesAspectData() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("query.client-ip", "116.154.188.222"); + data.process(); + ElementData element = data.get(engine.getElementDataKey()); + assertTrue(String.valueOf(element), + element instanceof AspectData); + assertTrue(String.valueOf(element), + element instanceof IPIntelligenceData); + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineNamesTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineNamesTests.java new file mode 100644 index 000000000..3c49e78b3 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineNamesTests.java @@ -0,0 +1,132 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.pipeline.engines.Constants; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static fiftyone.ipintelligence.shared.testhelpers.FileUtils.LITE_IPI_DATA_FILE_NAME; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +/** + * The names the native engine is created with, read from the configuration + * the engine passes to it, and the names the native engine then holds. + */ +public class EngineNamesTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void NoNames_NativeEngineCreatedWithNoNames() throws Exception { + try (IPIntelligenceOnPremiseEngine engine = build(null)) { + assertEquals( + "An engine configured with no names should create the " + + "native engine with no names.", + Collections.emptyList(), + namesCreatedWith(engine)); + } + } + + @Test + public void Names_NativeEngineCreatedWithOnlyThoseNames() + throws Exception { + List names = Collections.singletonList("RegisteredName"); + try (IPIntelligenceOnPremiseEngine engine = build(names)) { + assertEquals( + "An engine configured with names should create the native " + + "engine with those names and no others.", + names, + namesCreatedWith(engine)); + assertEquals( + "The native engine should hold only the names configured.", + names, + nativeNames(engine)); + } + } + + static IPIntelligenceOnPremiseEngine build(List names) + throws Exception { + File file; + try { + file = getFilePath(LITE_IPI_DATA_FILE_NAME); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + IPIntelligenceOnPremiseEngineBuilder builder = + new IPIntelligenceOnPremiseEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.MaxPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + if (names != null) { + builder.setProperties(names); + } + return builder.build(file.toString(), false); + } + + /** + * The names in the configuration the engine creates its native engine + * with, and creates it again with when the native engine is replaced. + */ + static List namesCreatedWith( + IPIntelligenceOnPremiseEngine engine) throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("propertiesConfigSwig"); + field.setAccessible(true); + RequiredPropertiesConfigSwig config = + (RequiredPropertiesConfigSwig) field.get(engine); + try (VectorStringSwig names = config.getProperties()) { + return new ArrayList<>(names); + } + } + + /** + * The names the native engine reports it was created with. + */ + static List nativeNames(IPIntelligenceOnPremiseEngine engine) + throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + field.setAccessible(true); + EngineIpiSwig nativeEngine = (EngineIpiSwig) field.get(engine); + try (VectorStringSwig names = nativeEngine.getRequiredProperties()) { + return new ArrayList<>(names); + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessMarshallingTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessMarshallingTests.java new file mode 100644 index 000000000..6b099b1a7 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessMarshallingTests.java @@ -0,0 +1,133 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EvidenceIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Processing keeps what crosses into native code small. It passes the native + * engine only the Evidence the engine accepts, and takes the accepted keys + * from the native engine once for each data set, never on each process. A + * stand-in for the native engine records what it is given and asked. + */ +public class ProcessMarshallingTests extends TestsBase { + + private static final String CLIENT_IP_KEY = "query.client-ip"; + + private static final String ADDRESS = "116.154.188.222"; + + private static final int OTHER_EVIDENCE = 20; + + private static final int PROCESSES = 5; + + private final List> passed = new ArrayList<>(); + + private EngineIpiSwig nativeEngine; + + private Field engineField; + + private Object realEngine; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + nativeEngine = mock(EngineIpiSwig.class); + when(nativeEngine.process(any(EvidenceIpiSwig.class))) + .thenAnswer(invocation -> { + passed.add(new TreeMap<>( + (EvidenceIpiSwig) invocation.getArgument(0))); + return mock(ResultsIpiSwig.class); + }); + engineField = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + engineField.setAccessible(true); + realEngine = engineField.get(getWrapper().getEngine()); + when(nativeEngine.getKeys()) + .thenReturn(((EngineIpiSwig) realEngine).getKeys()); + engineField.set(getWrapper().getEngine(), nativeEngine); + } + + @After + public void cleanup() throws Exception { + engineField.set(getWrapper().getEngine(), realEngine); + testCleanup(); + } + + @Test + public void Process_PassesOnlyAcceptedEvidence() throws Exception { + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence(CLIENT_IP_KEY, ADDRESS); + for (int i = 0; i < OTHER_EVIDENCE; i++) { + data.addEvidence("query.other-" + i, "value " + i); + data.addEvidence("cookie.other-" + i, "value " + i); + } + data.process(); + } + assertEquals(1, passed.size()); + assertEquals( + Collections.singletonMap(CLIENT_IP_KEY, ADDRESS), + passed.get(0)); + } + + @Test + public void Process_TakesNoKeysFromTheNativeEngine() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + EvidenceKeyFilter filter = engine.getEvidenceKeyFilter(); + for (int i = 0; i < PROCESSES; i++) { + try (FlowData data = + getWrapper().getPipeline().createFlowData()) { + data.addEvidence(CLIENT_IP_KEY, ADDRESS); + data.process(); + } + } + verify(nativeEngine, times(PROCESSES)) + .process(any(EvidenceIpiSwig.class)); + verify(nativeEngine, never()).getKeys(); + assertSame(filter, engine.getEvidenceKeyFilter()); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessReadsNoValueTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessReadsNoValueTests.java new file mode 100644 index 000000000..6fb1ffa95 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessReadsNoValueTests.java @@ -0,0 +1,97 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EvidenceIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.StringValueSwig; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +/** + * Processing keeps the native results and reads no value from them, and a + * value is read from them only when it is asked for. A stand-in for the + * native engine gives results that record what is asked of them. + */ +public class ProcessReadsNoValueTests extends TestsBase { + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void Process_ReadsNoValue() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + Integer index = engine.getPropertyIndex("RegisteredName"); + assertNotNull(index); + EngineIpiSwig nativeEngine = mock(EngineIpiSwig.class); + ResultsIpiSwig results = mock(ResultsIpiSwig.class); + when(nativeEngine.process(any(EvidenceIpiSwig.class))) + .thenReturn(results); + when(results.containsProperty(anyString())).thenReturn(true); + StringValueSwig value = mock(StringValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn("Example"); + when(results.getValueAsString(index.intValue())).thenReturn(value); + + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + field.setAccessible(true); + Object real = field.get(engine); + field.set(engine, nativeEngine); + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("query.client-ip", "116.154.188.222"); + data.process(); + + verify(nativeEngine).process(any(EvidenceIpiSwig.class)); + verifyNoInteractions(results); + + IPIntelligenceData ipi = data.get(IPIntelligenceData.class); + assertEquals("Example", ipi.getRegisteredName().getValue()); + verify(results).getValueAsString(index.intValue()); + verify(results, never()).getValueAsString(anyString()); + } finally { + field.set(engine, real); + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexLaterDataSetTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexLaterDataSetTests.java new file mode 100644 index 000000000..5651ca46a --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexLaterDataSetTests.java @@ -0,0 +1,114 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import org.junit.Test; + +import java.io.File; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assume.assumeTrue; + +/** + * The table of property indexes is made once, from the names the native + * engine gives when the engine is created, and is not made again from a + * later data set, even one whose native engine gives other names. The + * engine is built from the Lite file with no names, then its native engine + * is replaced by one the engine creates from the ASN data file. + */ +public class PropertyIndexLaterDataSetTests { + + private static final String ASN_IPI_DATA_FILE_NAME = + "51Degrees-IPIV4AsnIpiV41.ipi"; + + @Test + public void LaterDataSet_TableNotMadeAgain() throws Exception { + File asn; + try { + asn = getFilePath(ASN_IPI_DATA_FILE_NAME); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return; + } + try (IPIntelligenceOnPremiseEngine engine = + EngineNamesTests.build(null)) { + Map before = table(engine); + assertFalse("Expected a table from the first data set.", + before.isEmpty()); + + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + field.setAccessible(true); + EngineIpiSwig first = (EngineIpiSwig) field.get(engine); + field.set(engine, null); + try { + engine.refreshData( + engine.getDataFileMetaData().getIdentifier(), + Files.readAllBytes(asn.toPath())); + } + finally { + first.delete(); + } + + List laterNames; + EngineIpiSwig later = (EngineIpiSwig) field.get(engine); + try (VectorStringSwig names = later.getRequiredProperties()) { + laterNames = new ArrayList<>(names); + } + assertFalse( + "Expected the later data set to give names the first did " + + "not, so a table made again would differ.", + before.keySet().containsAll(laterNames)); + + assertEquals(before, table(engine)); + for (String name : laterNames) { + if (before.containsKey(name) == false) { + assertNull( + "'" + name + "' is only in the later data set.", + engine.getPropertyIndex(name)); + } + } + } + } + + @SuppressWarnings("unchecked") + private static Map table( + IPIntelligenceOnPremiseEngine engine) throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("propertyIndexes"); + field.setAccessible(true); + return new TreeMap<>((Map) field.get(engine)); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ResultsReadByOneThreadTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ResultsReadByOneThreadTests.java new file mode 100644 index 000000000..3ba77454b --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ResultsReadByOneThreadTests.java @@ -0,0 +1,189 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EvidenceIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.StringValueSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.WeightedUTF8StringListValueSwig; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.ipintelligence.shared.IPIntelligenceDataBaseOnPremise; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEFAULTS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Several threads read one finished Element Data at once, through the + * typed getters and through getValues, and only one of them at a time reads + * from the native results. A stand-in for the native engine gives results + * that count the threads inside them, and each read of them takes a moment + * so that two reads at once would overlap. + */ +public class ResultsReadByOneThreadTests extends TestsBase { + + private static final int THREADS = 4; + + private static final int READS = 10; + + private static final long READ_HOLD_MILLIS = 2; + + private static final long WAIT_SECONDS = 60; + + private static final Class[] WEIGHTED_STRINGS = + { List.class, IWeightedValue.class, String.class }; + + private final AtomicInteger inside = new AtomicInteger(); + + private final AtomicInteger most = new AtomicInteger(); + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void SeveralThreads_OneReadsTheResultsAtATime() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + EngineIpiSwig nativeEngine = mock(EngineIpiSwig.class); + ResultsIpiSwig results = countingResults(); + when(nativeEngine.process(any(EvidenceIpiSwig.class))) + .thenReturn(results); + + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + field.setAccessible(true); + Object real = field.get(engine); + field.set(engine, nativeEngine); + ExecutorService executor = Executors.newFixedThreadPool(THREADS); + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("query.client-ip", "116.154.188.222"); + data.process(); + IPIntelligenceData ipi = data.get(IPIntelligenceData.class); + + CountDownLatch start = new CountDownLatch(1); + List> readers = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + boolean getter = t % 2 == 0; + readers.add(executor.submit(read(ipi, getter, start))); + } + start.countDown(); + for (Future reader : readers) { + assertEquals(Integer.valueOf(READS), + reader.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + assertTrue("Expected the stand-in results to be read.", + most.get() > 0); + assertEquals( + "More than one thread read the native results at once.", + 1, + most.get()); + } finally { + executor.shutdownNow(); + field.set(engine, real); + } + } + + private Callable read( + IPIntelligenceData ipi, + boolean getter, + CountDownLatch start) { + return () -> { + start.await(); + int done = 0; + for (int i = 0; i < READS; i++) { + if (getter) { + assertEquals("Example", + ipi.getRegisteredName().getValue()); + } + else { + ((IPIntelligenceDataBaseOnPremise) ipi).getValues( + "RegisteredName", WEIGHTED_STRINGS); + } + done++; + } + return done; + }; + } + + /** + * Results that count the threads inside any of their methods, holding + * each call for a moment. + */ + private ResultsIpiSwig countingResults() { + StringValueSwig value = mock(StringValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn("Example"); + WeightedUTF8StringListValueSwig list = + mock(WeightedUTF8StringListValueSwig.class); + when(list.hasValue()).thenReturn(false); + when(list.getNoValueMessage()).thenReturn("No value"); + return mock(ResultsIpiSwig.class, invocation -> { + int now = inside.incrementAndGet(); + most.accumulateAndGet(now, Math::max); + try { + Thread.sleep(READ_HOLD_MILLIS); + switch (invocation.getMethod().getName()) { + case "containsProperty": + return true; + case "getValueAsString": + return value; + case "getValuesAsWeightedUTF8StringList": + return list; + default: + return RETURNS_DEFAULTS.answer(invocation); + } + } + finally { + inside.decrementAndGet(); + } + }); + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremiseEngineNamesTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremiseEngineNamesTests.java new file mode 100644 index 000000000..7c56d11be --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremiseEngineNamesTests.java @@ -0,0 +1,110 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.pipeline.core.flowelements.Pipeline; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static fiftyone.ipintelligence.shared.testhelpers.FileUtils.LITE_IPI_DATA_FILE_NAME; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +/** + * The names the on-premise pipeline builder has its engine create the native + * engine with, read from the configuration the engine passes to it. + */ +public class OnPremiseEngineNamesTests { + + @Test + public void NoNames_NativeEngineCreatedWithNoNames() throws Exception { + assertEquals( + "A pipeline configured with no names should have its " + + "engine create the native engine with no names.", + Collections.emptyList(), + namesCreatedWith(Collections.emptyList())); + } + + @Test + public void Names_NativeEngineCreatedWithOnlyThoseNames() + throws Exception { + List names = Collections.singletonList("RegisteredName"); + assertEquals( + "A pipeline configured with names should have its engine " + + "create the native engine with those names and no others.", + names, + namesCreatedWith(names)); + } + + /** + * Build an on-premise pipeline with the names given, and give the names + * its engine creates the native engine with. + */ + private static List namesCreatedWith(List names) + throws Exception { + File file; + try { + file = getFilePath(LITE_IPI_DATA_FILE_NAME); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + IPIntelligenceOnPremisePipelineBuilder builder = + new IPIntelligencePipelineBuilder( + LoggerFactory.getILoggerFactory()) + .useOnPremise(file.getAbsolutePath(), false) + .setShareUsage(false) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + for (String name : names) { + builder.setProperty(name); + } + try (Pipeline pipeline = builder.build()) { + return namesCreatedWith( + pipeline.getElement(IPIntelligenceOnPremiseEngine.class)); + } + } + + private static List namesCreatedWith( + IPIntelligenceOnPremiseEngine engine) throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("propertiesConfigSwig"); + field.setAccessible(true); + RequiredPropertiesConfigSwig config = + (RequiredPropertiesConfigSwig) field.get(engine); + try (VectorStringSwig names = config.getProperties()) { + return new ArrayList<>(names); + } + } +} From c54795a4f42f3ea980e61c812d4fd7801d70430e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 21:14:19 +0000 Subject: [PATCH 40/42] Log the data update errors table's rows as the specification gives them, and add the missing status (#186) [data-updates.md lines 478 to 497 at dbaf099c](https://github.com/51Degrees/specifications/blob/dbaf099c712e2bd3a2a80bfb4eaf284b079f05eb/pipeline-specification/features/data-updates.md?plain=1#L478-L497) says of the data update service's errors --- .../flowelements/DataRefreshTests.java | 215 ++++++++++++++++++ .../CloudAspectEngineBaseTests.java | 17 ++ .../AcceptedEvidenceDeclaredTests.java | 59 +++++ .../engines/services/DataUpdateService.java | 4 +- .../services/DataUpdateServiceDefault.java | 70 +++--- .../DataUpdateServiceLoggingTests.java | 96 +++++++- .../services/DataUpdateServiceTests.java | 6 +- 7 files changed, 414 insertions(+), 53 deletions(-) create mode 100644 device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java new file mode 100644 index 000000000..a3ec7ff01 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java @@ -0,0 +1,215 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.TestsBase; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Refreshing the engine's data, with the free Lite data file. Refreshes + * from several threads at once leave a working engine, a refresh from data + * that cannot be used fails and leaves the data the engine had, and + * processing while the data is refreshed gives the same values. + */ +public class DataRefreshTests extends TestsBase { + + private static final int THREADS = 4; + + private static final int REFRESHES = 5; + + private static final long WAIT_SECONDS = 300; + + private static final List USER_AGENTS = Arrays.asList( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 " + + "Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + + private ExecutorService executor; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.MaxPerformance); + executor = Executors.newFixedThreadPool(THREADS); + } + + @After + public void cleanup() { + executor.shutdownNow(); + testCleanup(); + } + + /** + * Several threads refresh the engine from its data file at once, and + * the engine then gives the values it gave before. + */ + @Test + public void Refresh_FromSeveralThreads_EngineStillWorks() + throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + CountDownLatch start = new CountDownLatch(1); + List> refreshers = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + refreshers.add(executor.submit(() -> { + start.await(); + for (int i = 0; i < REFRESHES; i++) { + engine.refreshData(identifier); + } + return REFRESHES; + })); + } + start.countDown(); + for (Future refresher : refreshers) { + assertEquals(Integer.valueOf(REFRESHES), + refresher.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + assertEquals(expected, isMobile()); + } + + /** + * A refresh from data that is not a data file fails with an error, and + * the engine goes on using the data it had. + */ + @Test + public void Refresh_DataNotUsable_FailsAndKeepsTheData() throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String published = String.valueOf( + engine.getDataFileMetaData().getDataPublishedDateTime()); + byte[] notData = "This is not a data file.".getBytes( + StandardCharsets.UTF_8); + + boolean failed = false; + try { + engine.refreshData( + engine.getDataFileMetaData().getIdentifier(), + notData); + } + catch (RuntimeException e) { + failed = true; + } + + assertTrue("A refresh from data that cannot be used did not fail.", + failed); + assertEquals(expected, isMobile()); + assertEquals(published, String.valueOf( + engine.getDataFileMetaData().getDataPublishedDateTime())); + } + + /** + * Threads process while another refreshes the engine from its data + * file, and every process gives the values the engine gave before. + */ + @Test + public void Process_WhileRefreshing_GivesTheSameValues() + throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + AtomicBoolean refreshing = new AtomicBoolean(true); + CountDownLatch start = new CountDownLatch(1); + List> processors = new ArrayList<>(); + for (int t = 0; t < THREADS - 1; t++) { + processors.add(executor.submit(process(expected, refreshing, start))); + } + Future refresher = executor.submit(() -> { + start.await(); + try { + for (int i = 0; i < REFRESHES; i++) { + engine.refreshData(identifier); + } + } + finally { + refreshing.set(false); + } + return REFRESHES; + }); + start.countDown(); + assertEquals(Integer.valueOf(REFRESHES), + refresher.get(WAIT_SECONDS, TimeUnit.SECONDS)); + for (Future processor : processors) { + assertTrue("A thread processed nothing during the refreshes.", + processor.get(WAIT_SECONDS, TimeUnit.SECONDS) > 0); + } + } + + private Callable process( + List expected, + AtomicBoolean refreshing, + CountDownLatch start) { + return () -> { + start.await(); + int processed = 0; + do { + List values = isMobile(); + if (expected.equals(values) == false) { + fail("Processing during a refresh gave " + values + + " where " + expected + " was given before."); + } + processed++; + } while (refreshing.get()); + return processed; + }; + } + + /** + * Whether each of the User-Agents is a mobile device. + */ + private List isMobile() throws Exception { + List values = new ArrayList<>(); + for (String userAgent : USER_AGENTS) { + try (FlowData data = + getWrapper().getPipeline().createFlowData()) { + data.addEvidence("header.user-agent", userAgent); + data.process(); + values.add( + data.get(DeviceData.class).getIsMobile().getValue()); + } + } + return values; + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index 4f073192e..46177ac95 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -131,6 +131,23 @@ public void addProfile(TestData profile) { } } + /** + * A property keyed cloud engine that declares no Evidence of its own + * takes the base class's default, which accepts no Evidence. + */ + @Test + public void propertyKeyedDefault_AcceptsNoEvidence() { + EvidenceKeyFilter filter = new TestKeyedInstance().getEvidenceKeyFilter(); + for (String key : Arrays.asList( + "header.user-agent", + "query.client-ip", + "server.client-ip", + "cookie.51D_example")) { + assertFalse(filter.include(key), + "The default Evidence filter accepted '" + key + "'."); + } + } + private class TestKeyedInstance extends PropertyKeyedCloudEngineBase { public TestKeyedInstance() { diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java new file mode 100644 index 000000000..b8f7b6aba --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java @@ -0,0 +1,59 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Every Flow Element declares the Evidence it accepts, because the base + * Flow Element leaves getEvidenceKeyFilter abstract, so the compiler makes + * each concrete element give one. + */ +public class AcceptedEvidenceDeclaredTests { + + @Test + public void FlowElement_DeclaresTheEvidenceFilter() throws Exception { + Method method = FlowElement.class.getMethod("getEvidenceKeyFilter"); + assertEquals(EvidenceKeyFilter.class, method.getReturnType()); + assertTrue("The interface gives a default, so an element need not " + + "declare the Evidence it accepts.", + method.isDefault() == false); + } + + @Test + public void FlowElementBase_LeavesTheEvidenceFilterAbstract() + throws Exception { + Method method = FlowElementBase.class.getDeclaredMethod( + "getEvidenceKeyFilter"); + assertTrue("FlowElementBase gives a default, so an element need " + + "not declare the Evidence it accepts.", + Modifier.isAbstract(method.getModifiers())); + } +} diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java index 2ee0231c0..09372390c 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java @@ -129,7 +129,9 @@ enum AutoUpdateStatus { // Refreshing the engine with the new data caused an error to occur. AUTO_UPDATE_REFRESH_FAILED, // An operation could not be completed within the expected time. - AUTO_UPDATE_TIMEOUT + AUTO_UPDATE_TIMEOUT, + // The data file checking for updates has no temporary file path. + AUTO_UPDATE_TEMP_PATH_NOT_SET } class DataUpdateCompleteArgs { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index ba4631d33..9f27f37bd 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -260,11 +260,7 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { if (newDataAvailable == false && dataFile.getConfiguration().getDataUpdateUrl() != null && dataFile.getConfiguration().getDataUpdateUrl().isEmpty() == false) { - // A failed update is tried again only when the timer is on. - result = checkForUpdateFromUrl( - dataFile, - manualUpdate == false && - dataFile.getAutomaticUpdatesEnabled()); + result = checkForUpdateFromUrl(dataFile); newDataAvailable = result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS || result == AutoUpdateStatus.AUTO_UPDATE_SUCCESS; @@ -472,9 +468,10 @@ public AutoUpdateStatus updateFromMemory(AspectEngineDataFile dataFile, byte[] d fileWrapperFactory.build(dataFile.getDataFilePath()).getWriter() .writeBytes(data); } catch (Exception ex) { - logger.error("An error occurred when writing to " + - dataFile.getDataFilePath() + - ". Error detail: " + ex.getMessage(), ex); + logger.warn("An error occurred when writing to '" + + dataFile.getDataFilePath() + "'. The engine will be " + + "updated to use the new data but the file on disk " + + "will still contain old data.", ex); } } @@ -806,13 +803,11 @@ private static List licenseKeysToMask( * * @param dataFile the data file to use * @param tempFile the temp file to write the data to - * @param automatic true if the update will be tried again if it fails * @return the {@link AutoUpdateStatus} value indicating the result */ private AutoUpdateStatus downloadFile( AspectEngineDataFile dataFile, - FileWrapper tempFile, - boolean automatic) { + FileWrapper tempFile) { String url = dataFile.getFormattedUrl(); String engineType = getEngineType(dataFile); @@ -847,7 +842,7 @@ private AutoUpdateStatus downloadFile( AutoUpdateStatus downloaded; try { downloaded = downloadOkResponse( - connection, dataFile, tempFile, automatic); + connection, dataFile, tempFile, loggedUrl); } catch (Exception e) { // A download cut short is the access error, as // a failure to connect is. An HTTP library's @@ -914,10 +909,6 @@ private long getIfModifiedSince(AspectEngineDataFile dataFile) { return publishedMillis; } - private static String retryNote(boolean automatic) { - return automatic ? " Update will be attempted again later." : ""; - } - /** * The message of the Errors table's row for a failure to connect to the * data update service, or to download all of its response. The URL is @@ -977,13 +968,15 @@ private String getEngineType(AspectEngineDataFile dataFile) { *

* A failure reading the download is thrown, for the caller to report. * A failure writing {@code tempFile} is logged here as the - * specification's write error. + * specification's write error. With MD5 verification on, a response + * with no Content-MD5 header is warned about and applied, and one whose + * hash does not match is the Errors table's integrity row. */ private AutoUpdateStatus downloadOkResponse( HttpURLConnection connection, AspectEngineDataFile dataFile, FileWrapper tempFile, - boolean automatic) throws IOException { + String loggedUrl) throws IOException { logger.debug("HTTP Status is OK"); boolean verifyMd5 = dataFile.getConfiguration().getVerifyMd5(); boolean decompress = dataFile.getConfiguration().getDecompressContent(); @@ -1017,13 +1010,21 @@ private AutoUpdateStatus downloadOkResponse( return AutoUpdateStatus.AUTO_UPDATE_ERR_READING_STREAM; } - if (verifyMd5 && !md5Matches(md, connection)) { - logger.warn("An error occurred during the integrity check of new " + - "data file for " + getEngineType(dataFile) + "." + - retryNote(automatic) + - " Error detail: the MD5 hash of the download does not " + - "match the Content-MD5 header"); - return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; + if (verifyMd5) { + String header = connection.getHeaderField("Content-MD5"); + String computed = DatatypeConverter.printHexBinary(md.digest()) + .toLowerCase(Locale.ROOT); + if (header == null) { + logger.warn("No MD5 hash included in data update response " + + "for '{}'. Unable to verify data integrity", loggedUrl); + } else if (header.equalsIgnoreCase(computed) == false) { + logCheckFailure("Integrity check failed. MD5 hash in HTTP " + + "response '" + header + "' for '" + + getEngineType(dataFile) + "'data update does not " + + "match calculated hash for the downloaded file '" + + computed + "'.", null); + return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; + } } return AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; } @@ -1108,12 +1109,6 @@ private void writeBodyTo(FileWrapper tempFile, InputStream body) throws IOExcept } } - private static boolean md5Matches(MessageDigest md, HttpURLConnection connection) { - String header = connection.getHeaderField("Content-MD5"); - String computed = DatatypeConverter.printHexBinary(md.digest()); - return header != null && header.equalsIgnoreCase(computed); - } - @Override public void close() { if (closeFutureFactory) { @@ -1126,8 +1121,7 @@ public void close() { } private AutoUpdateStatus checkForUpdateFromUrl( - AspectEngineDataFile dataFile, - boolean automatic) { + AspectEngineDataFile dataFile) { if (dataFile.getDataFilePath() == null || dataFile.getDataFilePath().isEmpty()) { // There is no data file path specified so perform the // update entirely in memory. @@ -1135,7 +1129,7 @@ private AutoUpdateStatus checkForUpdateFromUrl( FileWrapper uncompressedData = new FileWrapperMemory(); AutoUpdateStatus memoryDownloadResult = - downloadFile(dataFile, uncompressedData, automatic); + downloadFile(dataFile, uncompressedData); if (memoryDownloadResult != AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { return memoryDownloadResult; } @@ -1163,10 +1157,10 @@ private AutoUpdateStatus checkForUpdateFromUrl( if (dataFile.getTempDataDirPath() == null || dataFile.getTempDataDirPath().isEmpty()) { - logger.error("The data file '" + dataFile.getIdentifier() + + logCheckFailure("The data file '" + dataFile.getIdentifier() + "' is checking for updates but does not have a temporary " + - "file path configured."); - return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; + "file path configured.", null); + return AutoUpdateStatus.AUTO_UPDATE_TEMP_PATH_NOT_SET; } Path uncompressedTempFile = Paths.get( dataFile.getTempDataDirPath(), @@ -1175,7 +1169,7 @@ private AutoUpdateStatus checkForUpdateFromUrl( try { // Check if there is an update and download it if there is AutoUpdateStatus result = - downloadFile(dataFile, uncompressedData, automatic); + downloadFile(dataFile, uncompressedData); if (result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { WatchKey watchKey = dataFile.getConfiguration().getWatchKey(); diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java index c93f6369c..be0ce9ca1 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java @@ -43,8 +43,10 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.math.BigInteger; import java.net.HttpURLConnection; import java.net.URL; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -59,6 +61,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -80,6 +83,8 @@ public class DataUpdateServiceLoggingTests { FULL_KEY + "&Download=True&Type=HashV41"; private static final String MASKED_URL = UPDATE_URL + "?LicenseKeys=" + MASKED_KEY + "&Download=True&Type=HashV41"; + // The body of a download the tests apply. + private static final byte[] DOWNLOAD = {1, 2, 3}; private ch.qos.logback.classic.Logger logger; private ListAppender appender; @@ -418,33 +423,102 @@ public void TemporaryFileCannotBeWritten_LogsTheWriteError() } @Test - public void Md5DoesNotMatch_WarnsAboutIntegrityCheck() throws Exception { + public void Md5DoesNotMatch_LogsTheTableRow() throws Exception { when(connection.getResponseCode()) .thenReturn(HttpURLConnection.HTTP_OK); when(connection.getInputStream()) - .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + .thenReturn(new ByteArrayInputStream(DOWNLOAD)); when(connection.getHeaderField("Content-MD5")).thenReturn("0"); AspectEngineDataFileDefault file = newDataFile(true, false, false); file.getConfiguration().setVerifyMd5(true); - dataUpdate.checkForUpdate(engine); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred during the integrity " + - "check of new data file for " + engineType + ". Error detail: " + - "the MD5 hash of the download does not match the Content-MD5 " + - "header"); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED, + status); + assertCheckFailed("Integrity check failed. MD5 hash in HTTP " + + "response '0' for '" + engineType + "'data update does not " + + "match calculated hash for the downloaded file '" + + md5Hex(DOWNLOAD) + "'.", null); + verify(engine, never()).refreshData(anyString(), any(byte[].class)); } + /** + * A response with no Content-MD5 header is warned about, and the check + * goes on to apply the download. + */ @Test - public void NoTemporaryFilePath_LogsError() { + public void Md5Missing_WarnsAndAppliesTheUpdate() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(DOWNLOAD)); + AspectEngineDataFileDefault file = newDataFile(true, false, false); + file.getConfiguration().setVerifyMd5(true); + + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "No MD5 hash included in data update " + + "response for '" + UPDATE_URL + "'. Unable to verify data " + + "integrity"); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS, + status); + verify(engine).refreshData(anyString(), any(byte[].class)); + } + + @Test + public void NoTemporaryFilePath_LogsTheTableRow() { AspectEngineDataFileDefault file = newDataFile(false, false, false); file.setTempDataDirPath(null); when(engine.getTempDataDirPath()).thenReturn(null); - dataUpdate.checkForUpdate(engine); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); - assertLogged(Level.ERROR, "The data file 'default' is checking for " + - "updates but does not have a temporary file path configured."); + assertEquals("AUTO_UPDATE_TEMP_PATH_NOT_SET", status.name()); + assertCheckFailed("The data file 'default' is checking for " + + "updates but does not have a temporary file path configured.", + null); + } + + /** + * Data passed in from memory that cannot be written to the data file is + * warned about, and the engine is still refreshed with it. + */ + @Test + public void UpdateFromMemory_FileCannotBeWritten_WarnsAndRefreshes() + throws Exception { + AspectEngineDataFileDefault file = newDataFile(false, false, false); + String unwritable = new File(new File( + System.getProperty("java.io.tmpdir"), + "no-such-directory-" + System.nanoTime()), "data.hash") + .getAbsolutePath(); + file.getConfiguration().setDataFilePath(unwritable); + + DataUpdateService.AutoUpdateStatus status = + dataUpdate.updateFromMemory(file, DOWNLOAD); + + assertLogged(Level.WARN, "An error occurred when writing to '" + + unwritable + "'. The engine will be updated to use the new data " + + "but the file on disk will still contain old data."); + assertTrue("An error was logged: " + logged(Level.ERROR), + logged(Level.ERROR).isEmpty()); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS, + status); + verify(engine).refreshData(anyString(), any(byte[].class)); + } + + /** + * The MD5 hash of the bytes as lower case hexadecimal. + */ + private static String md5Hex(byte[] bytes) throws Exception { + byte[] digest = MessageDigest.getInstance("MD5").digest(bytes); + return String.format("%032x", new BigInteger(1, digest)); } /** diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 6f139fca2..50ec8f88f 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -762,9 +762,9 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { } }); - // The invalid MD5 will cause a warning to be logged so we want - // to ignore this - ignoreWranings = 1; + // The invalid MD5 is the Errors table integrity row, logged once + // at Error, so it is ignored in cleanup. + ignoreErrors = 1; OnPremiseAspectEngine engine = mock(OnPremiseAspectEngine.class); DataFileConfigurationDefault config = new DataFileConfigurationDefault(); From fd2bda3d050641a1d022cceea16b651651317967 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 21:46:56 +0000 Subject: [PATCH 41/42] Split a property list given as one string at every comma (#166) An engine builder's `setProperties(String)`, the setter for a property list given as one string, now splits the list at every comma and trims the spaces round each name. Until now it split only where a comma had one white space character on each side, so `IsMobile,BrowserName`, the form the specification gives, was passed on as one property of that name. --- .../flowelements/AspectEngineBuilderBase.java | 11 ++- ...AspectEngineBuilderBasePropertiesTest.java | 89 +++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBasePropertiesTest.java diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java index aaba5f317..cd13fe07f 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java @@ -136,14 +136,19 @@ public TBuilder setProperties(List properties) { * Configure the properties that the engine will populate in the response. *

* By default, all properties will be populated. - * @param properties The properties that we want the engine to populate + * @param properties The properties that we want the engine to populate, + * separated by commas as a configuration file gives a + * list, with any spaces round each name ignored * @return this builder */ @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperties(String properties) { - for (String property : properties.split("\\s,\\s")) { - tryAddProperty(property); + for (String property : properties.split(",")) { + String name = property.trim(); + if (name.isEmpty() == false) { + tryAddProperty(name); + } } return (TBuilder) this; } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBasePropertiesTest.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBasePropertiesTest.java new file mode 100644 index 000000000..6bdef8e56 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBasePropertiesTest.java @@ -0,0 +1,89 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +/** + * A list of properties given as one string, as a configuration file gives + * it, is a comma-separated list. + */ +public class AspectEngineBuilderBasePropertiesTest { + + private static EmptyEngineBuilder builder() { + return new EmptyEngineBuilder(LoggerFactory.getILoggerFactory()); + } + + @Test + public void setProperties_CommasWithoutSpaces_EachProperty() { + EmptyEngineBuilder builder = builder() + .setProperties("IsMobile,BrowserName,PlatformName"); + assertEquals( + Arrays.asList("IsMobile", "BrowserName", "PlatformName"), + builder.getProperties()); + } + + @Test + public void setProperties_SpacesAroundNames_Trimmed() { + EmptyEngineBuilder builder = builder() + .setProperties(" IsMobile, BrowserName ,PlatformName , DeviceType"); + assertEquals( + Arrays.asList("IsMobile", "BrowserName", "PlatformName", + "DeviceType"), + builder.getProperties()); + } + + @Test + public void setProperties_EmptyEntries_LeftOut() { + EmptyEngineBuilder builder = builder() + .setProperties("IsMobile,,BrowserName,"); + assertEquals( + Arrays.asList("IsMobile", "BrowserName"), + builder.getProperties()); + } + + @Test + public void setProperties_OneName_OneProperty() { + EmptyEngineBuilder builder = builder().setProperties("IsMobile"); + assertEquals( + Collections.singletonList("IsMobile"), + builder.getProperties()); + } + + @Test + public void setProperties_Repeated_AddedOnce() { + EmptyEngineBuilder builder = builder() + .setProperties("IsMobile,IsMobile") + .setProperties("BrowserName, IsMobile"); + assertEquals( + Arrays.asList("IsMobile", "BrowserName"), + builder.getProperties()); + } +} From 46115d7c63187ebf38e10aa465a9f4437362d462 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 21:47:19 +0000 Subject: [PATCH 42/42] Find Element Data by its key whatever the case (#167) Element Data keys are now compared ignoring case, so `flowData.get("Device")` finds the Element Data stored under `device`, and adding Element Data under a key that differs only in case finds the Element Data already there. Until now the lookup compared keys exactly, so `flowData.get("Device")` threw `NoSuchElementException`. --- .../core/typed/TypedKeyMapBuilder.java | 4 +- .../core/flowelements/FlowDataTests.java | 25 ++++++ .../core/typed/TypedKeyMapKeyCaseTests.java | 77 +++++++++++++++++++ 3 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/typed/TypedKeyMapKeyCaseTests.java diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java index 72e87bd3d..86c2f22c2 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java @@ -106,11 +106,11 @@ private static class TypedKeyMapInternal implements TypedKeyMap, Map) flowData -> new TestElementData(mock(Logger.class), flowData)); + + assertEquals(data, flowData.get("Device")); + assertEquals(data, flowData.get("DEVICE")); + } + + @Test + public void FlowData_GetOrAddOtherCase_SameData() { + flowData.process(); + ElementData data = flowData.getOrAdd( + "device", + (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData)); + + ElementData again = flowData.getOrAdd( + "Device", + (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData)); + + assertSame(data, again); + } + @Test public void FlowData_AddDataString_GetDataTypedKey() { flowData.process(); diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/typed/TypedKeyMapKeyCaseTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/typed/TypedKeyMapKeyCaseTests.java new file mode 100644 index 000000000..d11442221 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/typed/TypedKeyMapKeyCaseTests.java @@ -0,0 +1,77 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.typed; + +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Keys ignore case, as the specification's access to results asks of + * string keys, in both kinds of map the builder makes. + */ +public class TypedKeyMapKeyCaseTests { + + private static final boolean[] CONCURRENT = {false, true}; + + @Test + public void get_OtherCase_FindsTheValue() { + for (boolean concurrent : CONCURRENT) { + TypedKeyMap map = new TypedKeyMapBuilder(concurrent).build(); + map.put(new TypedKeyDefault<>("device", String.class), "value"); + + assertEquals("value", + map.get(new TypedKeyDefault<>("Device", String.class))); + assertEquals("value", + map.get(new TypedKeyDefault<>("DEVICE", String.class))); + assertTrue(map.containsKey("Device")); + } + } + + @Test + public void put_OtherCase_ReplacesTheValue() { + for (boolean concurrent : CONCURRENT) { + TypedKeyMap map = new TypedKeyMapBuilder(concurrent).build(); + map.put(new TypedKeyDefault<>("device", String.class), "first"); + map.put(new TypedKeyDefault<>("Device", String.class), "second"); + + assertEquals(1, map.getKeys().size()); + assertEquals("second", + map.get(new TypedKeyDefault<>("device", String.class))); + } + } + + @Test + public void getKeys_KeepsTheSpellingFirstGiven() { + for (boolean concurrent : CONCURRENT) { + TypedKeyMap map = new TypedKeyMapBuilder(concurrent).build(); + map.put(new TypedKeyDefault<>("device", String.class), "first"); + map.put(new TypedKeyDefault<>("DEVICE", String.class), "second"); + + assertEquals(Collections.singletonList("device"), map.getKeys()); + } + } +}