diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java index c9c8d475c..96c33418c 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java @@ -54,22 +54,22 @@ import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.OutputStream; import java.io.PrintWriter; import java.util.List; +import java.util.Locale; import java.util.Map; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asFloatProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIPAddressProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIntegerProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWktStringProperty; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWeightedString; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.weightedValues; import static fiftyone.pipeline.util.FileFinder.getFilePath; /** @@ -199,24 +199,51 @@ private static void analyzeEvidence(Map evidence, /* Display the results of the lookup, which are called IP Intelligence properties. See the property dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedcloud.java&utm_term=property-dictionary for details of all available properties. */ - writer.println("\tRegisteredName: " + asStringProperty(tryGet(ipData::getRegisteredName))); - writer.println("\tRegisteredOwner: " + asStringProperty(tryGet(ipData::getRegisteredOwner))); - writer.println("\tRegisteredCountry: " + asStringProperty(tryGet(ipData::getRegisteredCountry))); - writer.println("\tIpRangeStart: " + asIPAddressProperty(tryGet(ipData::getIpRangeStart))); - writer.println("\tIpRangeEnd: " + asIPAddressProperty(tryGet(ipData::getIpRangeEnd))); - writer.println("\tCountry: " + asStringProperty(tryGet(ipData::getCountry))); - writer.println("\tCountryCode: " + asStringProperty(tryGet(ipData::getCountryCode))); - writer.println("\tCountryCode3: " + asStringProperty(tryGet(ipData::getCountryCode3))); - writer.println("\tRegion: " + asStringProperty(tryGet(ipData::getRegion))); - writer.println("\tState: " + asStringProperty(tryGet(ipData::getState))); - writer.println("\tTown: " + asStringProperty(tryGet(ipData::getTown))); - writer.println("\tLatitude: " + asFloatProperty(tryGet(ipData::getLatitude))); - writer.println("\tLongitude: " + asFloatProperty(tryGet(ipData::getLongitude))); - writer.println("\tAreas: " + asWktStringProperty(tryGet(ipData::getAreas))); - writer.println("\tAccuracyRadiusMin: " + asIntegerProperty(tryGet(ipData::getAccuracyRadiusMin))); - writer.println("\tTimeZoneOffset: " + asIntegerProperty(tryGet(ipData::getTimeZoneOffset))); + writeValue(writer, "RegisteredName", tryGet(ipData::getRegisteredName)); + writeValue(writer, "RegisteredOwner", tryGet(ipData::getRegisteredOwner)); + writeValue(writer, "RegisteredCountry", tryGet(ipData::getRegisteredCountry)); + writeValue(writer, "IpRangeStart", tryGet(ipData::getIpRangeStart)); + writeValue(writer, "IpRangeEnd", tryGet(ipData::getIpRangeEnd)); + writeValue(writer, "Country", tryGet(ipData::getCountry)); + writeValue(writer, "CountryCode", tryGet(ipData::getCountryCode)); + writeValue(writer, "CountryCode3", tryGet(ipData::getCountryCode3)); + writeValue(writer, "Region", tryGet(ipData::getRegion)); + writeValue(writer, "State", tryGet(ipData::getState)); + writeValue(writer, "Town", tryGet(ipData::getTown)); + writeValue(writer, "Latitude", tryGet(ipData::getLatitude)); + writeValue(writer, "Longitude", tryGet(ipData::getLongitude)); + writeValue(writer, "Areas", tryGet(ipData::getAreas)); + writeValue(writer, "AccuracyRadiusMin", tryGet(ipData::getAccuracyRadiusMin)); + writeValue(writer, "TimeZoneOffset", tryGet(ipData::getTimeZoneOffset)); } writer.println(); writer.flush(); } + + /** + * Writes each value of a property with its weighting, or why it has no + * value. + */ + private static void writeValue(PrintWriter writer, + String property, + AspectPropertyValue result) { + if (result.hasValue() == false) { + writer.println("\t" + property + ": " + asWeightedString(result)); + return; + } + /* IP Intelligence properties return weighted values. Each value comes + with a weighting between 0 and 1, which says how much of the IP range + the value applies to, and a property with a single answer returns it + with a weighting of 1. */ + StringBuilder line = new StringBuilder(); + for (IWeightedValue weighted : weightedValues(result)) { + if (line.length() > 0) { + line.append(", "); + } + line.append(weighted.getValue()) + .append(String.format(Locale.ROOT, " (weighting %.2f)", + weighted.getWeighting())); + } + writer.println("\t" + property + ": " + line); + } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index ca39cc6fe..64c060ed8 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -64,11 +64,13 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.weightedValues; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; @@ -261,7 +263,25 @@ private static boolean writeValue(PrintWriter writer, String property, Supplier> value) { if (DataFileHelper.hasProperty(engine, property)) { - writer.println("\t" + property + ": " + asString(value.get())); + AspectPropertyValue result = value.get(); + if (result.hasValue() == false) { + writer.println("\t" + property + ": " + asString(result)); + return false; + } + /* IP Intelligence properties return weighted values. Each value + comes with a weighting between 0 and 1, which says how much of the + IP range the value applies to, and a property with a single answer + returns it with a weighting of 1. */ + StringBuilder line = new StringBuilder(); + for (IWeightedValue weighted : weightedValues(result)) { + if (line.length() > 0) { + line.append(", "); + } + line.append(weighted.getValue()) + .append(String.format(Locale.ROOT, " (weighting %.2f)", + weighted.getWeighting())); + } + writer.println("\t" + property + ": " + line); return false; } writer.println("\t" + property + ": " + DataFileHelper.NEEDS_ENTERPRISE); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/WeightingShownTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/WeightingShownTest.java new file mode 100644 index 000000000..1ade8711a --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/WeightingShownTest.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.assertTrue; + +/** + * "Examples MUST demonstrate accessing both the value and its associated + * weighting" (ip-intelligence-specification/required-examples.md line 107). + * The getting started examples are where a developer first reads how to get + * a result, so their own code reads each value with its weighting, rather + * than leaving it to a helper the developer does not see. + */ +public class WeightingShownTest { + + private static String source(String name) throws IOException { + Path path = Paths.get("src", "main", "java", "fiftyone", + "ipintelligence", "examples", "console", name + ".java"); + return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } + + private static void assertShowsWeighting(String name) throws IOException { + String source = source(name); + assertTrue(name + " does not read a weighting", + source.contains(".getWeighting()")); + assertTrue(name + " does not read the weighted value", + source.contains("weightedValues(") && + source.contains(".getValue()")); + } + + @Test + public void gettingStartedOnPrem_ShowsValueAndWeighting() + throws IOException { + assertShowsWeighting("GettingStartedOnPrem"); + } + + @Test + public void gettingStartedCloud_ShowsValueAndWeighting() + throws IOException { + assertShowsWeighting("GettingStartedCloud"); + } +} diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/PropertyHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/PropertyHelper.java index 68b45b83b..66f79cef9 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/PropertyHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/PropertyHelper.java @@ -23,6 +23,7 @@ package fiftyone.ipintelligence.examples.shared; import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WeightedValue; import fiftyone.pipeline.core.data.WktString; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; @@ -30,8 +31,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -193,6 +196,59 @@ public static String asString(AspectPropertyValue value) { return "Unknown. " + value.getNoValueMessage(); } + /** + * Get each value of an IP Intelligence property with its weighting. IP + * Intelligence properties return weighted values, and a property with a + * single answer returns it with a weighting of 1. Due to Java generics + * type erasure, the on-premise engine returns + * List<IWeightedValue<T>> even when the declared type is T, so + * a value that is not such a list is given here with a weighting of 1. + * @param property the property value, may be null + * @return the weighted values, or an empty list when there is no value + */ + public static List> weightedValues( + AspectPropertyValue property) { + List> result = new ArrayList<>(); + if (property == null || property.hasValue() == false) { + return result; + } + Object value = property.getValue(); + if (value instanceof List) { + for (Object item : (List) value) { + if (item instanceof IWeightedValue) { + result.add((IWeightedValue) item); + } else { + result.add(new WeightedValue<>(FULL_WEIGHTING, item)); + } + } + } else { + result.add(new WeightedValue<>(FULL_WEIGHTING, value)); + } + return result; + } + + /** + * The raw weighting of a value that is the only answer, which is a + * weighting of 1. + */ + private static final int FULL_WEIGHTING = 65535; + + /** + * Write each value of an IP Intelligence property with its weighting, + * for example "gb (weighting 1.00)", or why there is no value. + * @param property the property value, may be null + * @return the values with their weightings, joined with ", " + */ + public static String asWeightedString(AspectPropertyValue property) { + if (property == null || property.hasValue() == false) { + return reportNoValue(property); + } + return weightedValues(property).stream() + .map(weighted -> String.format(Locale.ROOT, "%s (weighting %.2f)", + weighted.getValue(), weighted.getWeighting())) + .collect(Collectors.joining(", ")); + } + /** * Helper to get the first raw value of an IP Intelligence property. * Due to Java generics type erasure, the on-premise engine may return diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperWeightingTests.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperWeightingTests.java new file mode 100644 index 000000000..b8f93d5d6 --- /dev/null +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperWeightingTests.java @@ -0,0 +1,111 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * "Examples MUST demonstrate accessing both the value and its associated + * weighting" (ip-intelligence-specification/required-examples.md line 107). + * The examples read each value of a property with its weighting through + * {@link PropertyHelper#weightedValues}. + */ +public class PropertyHelperWeightingTests { + + private static IWeightedValue weighted( + final int rawWeighting, + final Object value) { + return new IWeightedValue() { + @Override + public int getRawWeighting() { + return rawWeighting; + } + + @Override + public Object getValue() { + return value; + } + }; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Test + public void weightedValues_ListOfWeightedValues_KeepsEachWeighting() { + List> list = new ArrayList<>(); + list.add(weighted(49151, "gb")); + list.add(weighted(16384, "ie")); + AspectPropertyValue property = + new AspectPropertyValueDefault(list); + + List> values = + PropertyHelper.weightedValues(property); + + assertEquals(2, values.size()); + assertEquals("gb", values.get(0).getValue()); + assertEquals(49151 / 65535f, values.get(0).getWeighting(), 0.0001); + assertEquals("ie", values.get(1).getValue()); + assertEquals(16384 / 65535f, values.get(1).getWeighting(), 0.0001); + } + + /** + * "a property with a single answer returns it with a weighting of 1" + * (required-examples.md line 107) + */ + @Test + public void weightedValues_SingleValue_HasWeightingOfOne() { + AspectPropertyValue property = + new AspectPropertyValueDefault<>("gb"); + + List> values = + PropertyHelper.weightedValues(property); + + assertEquals(1, values.size()); + assertEquals("gb", values.get(0).getValue()); + assertEquals(1f, values.get(0).getWeighting(), 0.0001); + } + + @Test + public void weightedValues_NoValue_IsEmpty() { + assertTrue(PropertyHelper.weightedValues( + new AspectPropertyValueDefault()).isEmpty()); + assertTrue(PropertyHelper.weightedValues(null).isEmpty()); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Test + public void asWeightedString_ShowsValueAndWeighting() { + List> list = new ArrayList<>(); + list.add(weighted(65535, "gb")); + assertEquals("gb (weighting 1.00)", PropertyHelper.asWeightedString( + new AspectPropertyValueDefault(list))); + } +} diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelist.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelist.java index 53cd41464..dd27dbad2 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelist.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelist.java @@ -36,13 +36,12 @@ public class EvidenceKeyFilterWhitelist implements EvidenceKeyFilter { /** * Construct a new instance using the list of evidence keys provided. - * By default, all keys will have the same order of precedence. + * By default, all keys will have the same order of precedence. Keys are + * matched without regard to case, as Evidence keys are case-insensitive. * @param whitelist evidence keys to add to the whitelist */ public EvidenceKeyFilterWhitelist(List whitelist) { - this.whitelist = new TreeMap<>(); - addValues(whitelist); - this.comparator = null; + this(whitelist, String.CASE_INSENSITIVE_ORDER); } /** @@ -65,13 +64,12 @@ public EvidenceKeyFilterWhitelist( * Construct a new instance using the map of evidence keys and order of * precedence provided. * The order of precedence of each key is given by the value of the - * key/value pair. + * key/value pair. Keys are matched without regard to case, as Evidence + * keys are case-insensitive. * @param whitelist evidence keys to add to the whitelist */ public EvidenceKeyFilterWhitelist(Map whitelist) { - this.whitelist = new TreeMap<>(); - addValues(whitelist); - this.comparator = null; + this(whitelist, String.CASE_INSENSITIVE_ORDER); } /** diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java index 72e87bd3d..8787cc622 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java @@ -103,14 +103,19 @@ private static class TypedKeyMapInternal implements TypedKeyMap, MapnaturalOrder()); EvidenceKeyFilterAggregator filter = new EvidenceKeyFilterAggregator(); filter.addFilter(filter1); diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelistCaseTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelistCaseTests.java new file mode 100644 index 000000000..7fb1fa785 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelistCaseTests.java @@ -0,0 +1,76 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.data; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * "Keys MUST be case-insensitive and will usually be split into + * <prefix>.<field>." (pipeline-specification/features/evidence.md + * line 10) + */ +public class EvidenceKeyFilterWhitelistCaseTests { + + @Test + public void listConstructor_KeysAreCaseInsensitive() { + EvidenceKeyFilterWhitelist filter = new EvidenceKeyFilterWhitelist( + Arrays.asList("header.User-Agent", "query.51D_Pixels")); + + assertTrue(filter.include("header.user-agent")); + assertTrue(filter.include("HEADER.USER-AGENT")); + assertTrue(filter.include("Query.51d_pixels")); + assertEquals(Integer.valueOf(0), filter.order("HEADER.user-agent")); + assertFalse(filter.include("header.accept")); + } + + @Test + public void mapConstructor_KeysAreCaseInsensitive() { + Map keys = new HashMap<>(); + keys.put("header.User-Agent", 1); + keys.put("cookie.51D_ScreenPixelsWidth", 2); + EvidenceKeyFilterWhitelist filter = new EvidenceKeyFilterWhitelist(keys); + + assertTrue(filter.include("header.user-agent")); + assertTrue(filter.include("COOKIE.51d_screenpixelswidth")); + assertEquals(Integer.valueOf(2), + filter.order("Cookie.51D_SCREENPIXELSWIDTH")); + } + + @Test + public void aggregator_DefaultWhitelistKeysAreCaseInsensitive() { + EvidenceKeyFilterAggregator filter = new EvidenceKeyFilterAggregator(); + filter.addFilter(new EvidenceKeyFilterWhitelist( + Arrays.asList("key2"))); + + assertTrue(filter.include("key2")); + assertTrue(filter.include("KEY2")); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java index 613033fa2..d7e2f247f 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java @@ -134,6 +134,53 @@ public void FlowData_AddDataString_GetDataString() { assertEquals(data, result); } + /** + * "String comparisons MUST be case insensitive." + * (pipeline-specification/features/access-to-results.md line 11) + * Element Data stored under "device" is found as "Device" and "DEVICE". + */ + @Test + public void FlowData_GetDataString_IgnoresCase() { + flowData.process(); + ElementData data = flowData.getOrAdd( + "device", + (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData)); + + assertEquals(data, flowData.get("Device")); + assertEquals(data, flowData.get("DEVICE")); + assertEquals(data, flowData.get( + new TypedKeyDefault("Device"))); + assertTrue(flowData.tryGetValue( + new TypedKeyDefault("dEvIcE")).hasValue()); + // Adding under another case finds the data that is there already. + assertEquals(data, flowData.getOrAdd( + "DEVICE", + (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData))); + assertEquals(1, flowData.elementDataAsMap().size()); + } + + /** + * "String comparisons MUST be case insensitive." + * (pipeline-specification/features/access-to-results.md line 11) + */ + @Test + public void TypedKeyMap_KeysIgnoreCase() { + for (boolean concurrent : new boolean[]{false, true}) { + fiftyone.pipeline.core.typed.TypedKeyMap map = + new fiftyone.pipeline.core.typed.TypedKeyMapBuilder(concurrent) + .build(); + map.put(new TypedKeyDefault<>("device", String.class), "value"); + + assertEquals("value", + map.get(new TypedKeyDefault<>("Device", String.class))); + assertTrue(map.containsKey("DEVICE")); + assertEquals("value", map.asStringKeyMap().get("dEvIcE")); + assertEquals("value", map.removeIfExists( + new TypedKeyDefault<>("DeViCe", String.class))); + assertFalse(map.containsKey("device")); + } + } + @Test public void FlowData_AddDataString_GetDataTypedKey() { flowData.process(); diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 9783c4077..d72fa0efc 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -48,8 +48,10 @@ * {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be * shared using the constructor parameter. * 5) Not a header, cookie, session value or query string parameter. - * When all Evidence is shared, everything is included. When all query string - * parameters are shared, every query string parameter is included. + * When all Evidence is shared, everything is included apart from cookies that + * do not start with {@link Constants#FIFTYONE_COOKIE_PREFIX}, which are never + * shared. When all query string parameters are shared, every query string + * parameter is included. * The prefix of a key is everything before its first '.', and the field * everything after it. An email address, the salt for hashed email * identifiers, the query string, the path and the Cookie header are never @@ -197,12 +199,24 @@ public boolean include(String key) { } } + int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); + + // Only cookies that start with the 51Degrees cookie prefix are + // shared, whatever the configuration and even when all Evidence is + // shared. The session cookie is included only for the tracker, which + // compares it and never sends it. + if (separator > 0 && + EVIDENCE_COOKIE_PREFIX.equals(lowerKey.substring(0, separator))) { + String field = lowerKey.substring(separator + 1); + return field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + (includeSession && field.equals(sessionCookieName)); + } + if (shareAllEvidence) { return true; } boolean result; - int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); if (separator > 0) { String prefix = lowerKey.substring(0, separator); String field = lowerKey.substring(separator + 1); @@ -212,12 +226,6 @@ public boolean include(String key) { // appear in the list of blocked headers. result = blockedHttpHeaders.contains(field) == false; break; - case EVIDENCE_COOKIE_PREFIX: - // Only add cookies that start with the 51Degrees cookie - // prefix. - result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || - (includeSession && field.equals(sessionCookieName)); - break; case EVIDENCE_SESSION_PREFIX: // Only add session values that start with the 51Degrees cookie // prefix. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index 1ab56e3f9..00d74103e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -65,7 +65,8 @@ public abstract class ShareUsageBuilderBase { protected double maxRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; protected double recoveryMultiplier = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; protected boolean shareAllEvidence; - protected boolean shareAllQueryStringParameters; + // On by default, as the specification's configuration options give it. + protected boolean shareAllQueryStringParameters = true; /** * Constructor @@ -90,8 +91,9 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { } /** - * By default, only query string parameters that start with 51D_ are - * shared. + * By default every query string parameter is shared. When share all + * query string parameters is turned off, only those that start with 51D_ + * and those named here are shared. *

* This setter replaces the current list, if any *

@@ -103,7 +105,7 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { * @return this builder */ - @DefaultValue("No sharing unless the parameter starts with 51D_") + @DefaultValue("Every parameter is shared, unless share all query string parameters is off") public ShareUsageBuilderBase setIncludedQueryStringParameters( List queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -112,8 +114,9 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, only query string parameters that start with 51D_ are - * shared. + * By default every query string parameter is shared. When share all + * query string parameters is turned off, only those that start with 51D_ + * and those named here are shared. *

* This setter replaces the current list, if any *

@@ -124,7 +127,7 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( * query string parameters to include * @return this builder */ - @DefaultValue("No sharing unless the parameter starts with 51D_") + @DefaultValue("Every parameter is shared, unless share all query string parameters is off") public ShareUsageBuilderBase setIncludedQueryStringParameters( String queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -134,8 +137,9 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, only query string parameters that start with 51D_ are - * shared. + * By default every query string parameter is shared. When share all + * query string parameters is turned off, only those that start with 51D_ + * and those named here are shared. *

* This setting adds the name of a parameter to share with 51Degrees * if needed. @@ -143,7 +147,7 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( * query string parameter to include * @return this builder */ - @DefaultValue("No sharing unless the parameter starts with 51D_") + @DefaultValue("Every parameter is shared, unless share all query string parameters is off") public ShareUsageBuilderBase setIncludedQueryStringParameter( String queryStringParameterName) { includedQueryStringParameters.add(queryStringParameterName.trim()); @@ -156,11 +160,12 @@ public ShareUsageBuilderBase setIncludedQueryStringParameter( * included parameters. The whole query string and the parameters that are * never shared are still not shared. *

- * Default value is false + * Default value is true. Set it to false to share only the parameters + * that start with 51D_ or are named in the included parameters. * @param shareAll true to share all query string parameters * @return this builder */ - @DefaultValue(booleanValue = false) + @DefaultValue(booleanValue = true) public ShareUsageBuilderBase setShareAllQueryStringParameters( boolean shareAll) { this.shareAllQueryStringParameters = shareAll; @@ -177,10 +182,11 @@ public boolean isShareAllQueryStringParameters() { /** * If set to true, all Evidence is shared, which overrides the blocked - * HTTP headers and the rules for cookies, session values and query string + * HTTP headers and the rules for session values and query string * parameters. The keys that are never shared (an email address, the salt * for hashed email identifiers, the whole query string, the path and the - * Cookie header) are still not shared. + * Cookie header) are still not shared, and nor is any cookie whose name + * does not start with 51D_, the session cookie among them. *

* Default value is false * @param shareAll true to share all Evidence diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java index 8aeb6e7f8..293e0f75a 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -144,6 +144,58 @@ public void sessionCookie() { assertFalse(sent.include("cookie._ga")); } + /** + * "It MUST also return false, whatever the configuration and even when + * all Evidence is shared, for any key whose prefix is `cookie.` and whose + * field does not start with `51D_`." (usage-sharing-element.md line 65) + */ + @Test + public void shareAllEvidence_CookieNotStarting51D_IsNotShared() { + EvidenceKeyFilterShareUsage shareAll = new EvidenceKeyFilterShareUsage( + Collections.emptyList(), + Collections.emptyList(), + false, + DEFAULT_SESSION_COOKIE_NAME, + true, + true); + + assertFalse(shareAll.include("cookie.plain")); + assertFalse(shareAll.include("Cookie._ga")); + assertFalse(shareAll.include("cookie.a.b")); + assertTrue(shareAll.include("cookie.51D_ScreenPixelsWidth")); + assertTrue(shareAll.include("cookie.51d_screenpixelswidth")); + // Everything else is still shared. + assertTrue(shareAll.include("header.x-anything")); + assertTrue(shareAll.include("session.plain")); + } + + /** + * "The session cookie MUST NOT be shared unless its name starts with + * `51D_`, as no other cookie is." (usage-sharing-element.md line 99) + */ + @Test + public void shareAllEvidence_SessionCookieIsNotShared() { + EvidenceKeyFilterShareUsage shareAll = new EvidenceKeyFilterShareUsage( + Collections.emptyList(), + Collections.emptyList(), + false, + DEFAULT_SESSION_COOKIE_NAME, + true, + true); + + assertFalse(shareAll.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertFalse(shareAll.include("cookie.jsessionid")); + + EvidenceKeyFilterShareUsage named51D = new EvidenceKeyFilterShareUsage( + Collections.emptyList(), + Collections.emptyList(), + false, + "51D_Session", + true, + true); + assertTrue(named51D.include("cookie.51D_Session")); + } + /** * The email address, the salt for hashed email identifiers, the query * string and the path are never shared, whether bare or after a prefix, diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java index 3bc53524f..e387d2cb4 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java @@ -179,7 +179,31 @@ public void defaults_ShareOnlyWhatTheRulesAllow() throws Exception { assertFalse(shared(sent, "header", "x-blocked")); assertFalse(shared(sent, "cookie", "plain")); assertFalse(shared(sent, "session", "plain")); - assertFalse(shared(sent, "query", "plain")); + // Share all query string parameters is on by default. + assertTrue(shared(sent, "query", "plain")); + } + + /** + * "Share all query string parameters is on by default, so every query + * string and HTTP form parameter is shared unless a user turns it off." + * and "Anything else MUST return true." (usage-sharing-element.md lines + * 80 to 83), so a query string parameter that neither starts 51D_ nor is + * named is shared when nothing is configured. + */ + @Test + public void defaults_ShareEveryQueryStringParameter() throws Exception { + Map evidence = evidence(); + evidence.put("query.campaign", "spring"); + + Document sent = send(options(), evidence); + + assertTrue(shared(sent, "query", "campaign")); + assertTrue(shared(sent, "query", "plain")); + // The keys that are never shared stay out. + assertFalse(shared(sent, "query", "id.email")); + assertFalse(shared(sent, "server", "request-query")); + assertTrue(new ShareUsageBuilder(LoggerFactory.getILoggerFactory()) + .isShareAllQueryStringParameters()); } @Test @@ -189,7 +213,8 @@ public void shareAllEvidence_FromConfiguration_SharesEverythingButNeverShared() assertTrue(shared(sent, "header", "user-agent")); assertTrue(shared(sent, "header", "x-blocked")); - assertTrue(shared(sent, "cookie", "plain")); + // A cookie whose name does not start 51D_ is never shared. + assertFalse(shared(sent, "cookie", "plain")); assertTrue(shared(sent, "session", "plain")); assertTrue(shared(sent, "query", "plain")); assertFalse(shared(sent, "query", "id.email")); @@ -203,7 +228,11 @@ public void shareAllEvidence_FromConfiguration_SharesEverythingButNeverShared() @Test public void shareAllEvidence_False_FromConfiguration_KeepsTheRules() throws Exception { - Document sent = send(options("ShareAllEvidence", "false"), evidence()); + Document sent = send( + options( + "ShareAllEvidence", "false", + "ShareAllQueryStringParameters", "false"), + evidence()); assertFalse(shared(sent, "header", "x-blocked")); assertFalse(shared(sent, "cookie", "plain")); @@ -238,12 +267,12 @@ public void shareAllQueryStringParameters_False_FromConfiguration_KeepsTheRules( } @Test - public void builder_ShareOptionsDefaultToFalse() { + public void builder_ShareAllEvidenceOffAndShareAllQueryStringParametersOn() { ShareUsageBuilder builder = new ShareUsageBuilder( LoggerFactory.getILoggerFactory()); assertFalse(builder.isShareAllEvidence()); - assertFalse(builder.isShareAllQueryStringParameters()); + assertTrue(builder.isShareAllQueryStringParameters()); } @Test diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java index aaba5f317..105194aa8 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java @@ -136,14 +136,20 @@ public TBuilder setProperties(List properties) { * Configure the properties that the engine will populate in the response. *

* By default, all properties will be populated. - * @param properties The properties that we want the engine to populate + * @param properties The properties that we want the engine to populate, + * as a comma-separated list. White space around each + * name is ignored, so "IsMobile,BrowserName" and + * "IsMobile, BrowserName" are the same list * @return this builder */ @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperties(String properties) { - for (String property : properties.split("\\s,\\s")) { - tryAddProperty(property); + for (String property : properties.split(",")) { + String name = property.trim(); + if (name.isEmpty() == false) { + tryAddProperty(name); + } } return (TBuilder) this; } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java index 2ee0231c0..148f954db 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java @@ -129,7 +129,11 @@ enum AutoUpdateStatus { // Refreshing the engine with the new data caused an error to occur. AUTO_UPDATE_REFRESH_FAILED, // An operation could not be completed within the expected time. - AUTO_UPDATE_TIMEOUT + AUTO_UPDATE_TIMEOUT, + // An error the service did not expect ended an automatic check. + AUTO_UPDATE_UNKNOWN_ERROR, + // The data file checks for updates but has no temporary file path. + AUTO_UPDATE_TEMP_PATH_NOT_SET } class DataUpdateCompleteArgs { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index ba4631d33..fdca29562 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -201,13 +201,120 @@ public void onUpdateComplete(OnUpdateComplete onUpdateComplete) { @Override public AutoUpdateStatus checkForUpdate(OnPremiseAspectEngine engine) { - return checkForUpdate(engine.getDataFileMetaData(), true); + try { + return checkForUpdate(engine.getDataFileMetaData(), true); + } catch (CallbackFailure failure) { + // A programmatic check passes the error to its caller. + throw failure.getCause(); + } } @Override public AutoUpdateStatus checkForUpdate(OnPremiseAspectEngine engine, String identifier) { - return checkForUpdate(engine.getDataFileMetaData(identifier), true); + try { + return checkForUpdate(engine.getDataFileMetaData(identifier), true); + } catch (CallbackFailure failure) { + throw failure.getCause(); + } + } + + /** + * An error raised by an onUpdateComplete callback, carried out of a + * check so that it is told apart from an error in the check itself. + */ + private static final class CallbackFailure extends RuntimeException { + private static final long serialVersionUID = 1L; + + CallbackFailure(RuntimeException cause) { + super(cause); + } + + @Override + public synchronized RuntimeException getCause() { + return (RuntimeException) super.getCause(); + } + } + + /** + * Report a finished check to each onUpdateComplete callback in turn. A + * callback that throws is carried out as a {@link CallbackFailure}, and + * the callbacks after it are not called. + */ + private void reportComplete(Object sender, DataUpdateCompleteArgs args) { + for (OnUpdateComplete onUpdateComplete : onUpdateCompleteList) { + try { + onUpdateComplete.call(sender, args); + } catch (RuntimeException e) { + throw new CallbackFailure(e); + } + } + } + + /** + * Log an error that ended a check started by update polling or by the + * file watcher, with the message the specification gives for an error + * of any other kind and for an error raised by a callback. + */ + private void logUnhandled(AspectEngineDataFile dataFile, Throwable error) { + logger.error("An unhandled error occurred while checking for " + + "automatic updates for engine '" + getEngineType(dataFile) + + "'", error); + } + + /** + * Handle an error that ended a check started by update polling or by the + * file watcher. The error is logged rather than left in the task's + * future, where nothing reads it. An error in the check itself is then + * reported to the callbacks as {@link AutoUpdateStatus#AUTO_UPDATE_UNKNOWN_ERROR}. + * An error raised by a callback has been reported already. + */ + private void handleAutomaticFailure( + AspectEngineDataFile dataFile, + RuntimeException error) { + if (error instanceof CallbackFailure) { + logUnhandled(dataFile, error.getCause()); + return; + } + logUnhandled(dataFile, error); + try { + reportComplete(dataFile, new DataUpdateCompleteArgs( + AutoUpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR, + dataFile)); + } catch (CallbackFailure failure) { + logUnhandled(dataFile, failure.getCause()); + } + } + + /** + * Run a check started by update polling. A failure is caught and logged, + * and the next check is still scheduled, so a later good data file is + * still applied. + */ + private void runPollingCheck(AspectEngineDataFileDefault dataFile) { + try { + checkForUpdate(dataFile, false); + } catch (CallbackFailure failure) { + // The check finished and set the next one before reporting. + handleAutomaticFailure(dataFile, failure); + } catch (RuntimeException e) { + // The check may have ended before it set the next one. + try { + if (dataFile.getIsRegistered()) { + synchronized (futureLock) { + if (dataFile.getFuture() != null) { + dataFile.getFuture().cancel(false); + dataFile.setFuture(null); + } + } + registerDataFile(dataFile); + } + } catch (RuntimeException scheduling) { + logger.error("Exception when scheduling update check", + scheduling); + } + handleAutomaticFailure(dataFile, e); + } } /** @@ -299,7 +406,7 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { new Runnable() { @Override public void run() { - checkForUpdate(dataFile, false); + runPollingCheck(dataFile); } }, getIntervalMillis(dataFile.getConfiguration()))); @@ -320,11 +427,7 @@ public void run() { } } - for (OnUpdateComplete onUpdateComplete : onUpdateCompleteList) { - onUpdateComplete.call(state, new DataUpdateCompleteArgs( - result, - dataFile)); - } + reportComplete(state, new DataUpdateCompleteArgs(result, dataFile)); return result; } @@ -457,9 +560,7 @@ private void dataFileUpdated(Object sender) { } } } - for (OnUpdateComplete onUpdateComplete : onUpdateCompleteList) { - onUpdateComplete.call(sender, new DataUpdateCompleteArgs(status, dataFile)); - } + reportComplete(sender, new DataUpdateCompleteArgs(status, dataFile)); } @Override @@ -472,9 +573,11 @@ public AutoUpdateStatus updateFromMemory(AspectEngineDataFile dataFile, byte[] d fileWrapperFactory.build(dataFile.getDataFilePath()).getWriter() .writeBytes(data); } catch (Exception ex) { - logger.error("An error occurred when writing to " + - dataFile.getDataFilePath() + - ". Error detail: " + ex.getMessage(), ex); + // The engine is still refreshed from the data in memory. + logger.warn("An error occurred when writing to '" + + dataFile.getDataFilePath() + "'. The engine " + + "will be updated to use the new data but the " + + "file on disk will still contain old data.", ex); } } @@ -508,7 +611,13 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { if (dataFile.getConfiguration().getUpdateOnStartup() && isFalse(alreadyRegistered)) { logger.info("Updating on start-up"); logger.debug("Checking for Update on Startup {}", dataFile.getDataFilePath()); - AutoUpdateStatus status = checkForUpdate(dataFile, false); + AutoUpdateStatus status; + try { + status = checkForUpdate(dataFile, false); + } catch (CallbackFailure failure) { + // The update on start-up passes the error to its caller. + throw failure.getCause(); + } if (ObjectUtils.notEqual(status, AUTO_UPDATE_SUCCESS) && Check.notFileExists(dataFile.getDataFilePath())){ throw new IllegalStateException("Update on start up failed and there is no " + @@ -548,7 +657,7 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { public void run() { logger.debug("running checkForUpdate {}", aspectDataFile.getDataFilePath()); - checkForUpdate(aspectDataFile, false); + runPollingCheck(aspectDataFile); logger.debug("Returning from checkForUpdate"); } }, @@ -616,15 +725,23 @@ public void run() { if (watchKey == null) { return; } - for (WatchEvent event : watchKey.pollEvents()) { - if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { - if (((Path) event.context()).endsWith(aspectDataFilePath.getFileName())) { - dataFileUpdated(aspectDataFile); + try { + for (WatchEvent event : watchKey.pollEvents()) { + if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { + if (((Path) event.context()).endsWith(aspectDataFilePath.getFileName())) { + dataFileUpdated(aspectDataFile); + } } } + } catch (RuntimeException e) { + // A failure is logged and must not + // end this repeating task, so later + // changes are still applied. + handleAutomaticFailure(aspectDataFile, e); + } finally { + // make sure to reset the watch key or we are not called again + watchKey.reset(); } - // make sure to reset the watch key or we are not called again - watchKey.reset(); } }, pollingInterval)); @@ -847,7 +964,7 @@ private AutoUpdateStatus downloadFile( AutoUpdateStatus downloaded; try { downloaded = downloadOkResponse( - connection, dataFile, tempFile, automatic); + connection, dataFile, tempFile, loggedUrl); } catch (Exception e) { // A download cut short is the access error, as // a failure to connect is. An HTTP library's @@ -914,10 +1031,6 @@ private long getIfModifiedSince(AspectEngineDataFile dataFile) { return publishedMillis; } - private static String retryNote(boolean automatic) { - return automatic ? " Update will be attempted again later." : ""; - } - /** * The message of the Errors table's row for a failure to connect to the * data update service, or to download all of its response. The URL is @@ -983,7 +1096,7 @@ private AutoUpdateStatus downloadOkResponse( HttpURLConnection connection, AspectEngineDataFile dataFile, FileWrapper tempFile, - boolean automatic) throws IOException { + String loggedUrl) throws IOException { logger.debug("HTTP Status is OK"); boolean verifyMd5 = dataFile.getConfiguration().getVerifyMd5(); boolean decompress = dataFile.getConfiguration().getDecompressContent(); @@ -1017,13 +1130,22 @@ private AutoUpdateStatus downloadOkResponse( return AutoUpdateStatus.AUTO_UPDATE_ERR_READING_STREAM; } - if (verifyMd5 && !md5Matches(md, connection)) { - logger.warn("An error occurred during the integrity check of new " + - "data file for " + getEngineType(dataFile) + "." + - retryNote(automatic) + - " Error detail: the MD5 hash of the download does not " + - "match the Content-MD5 header"); - return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; + if (verifyMd5) { + String header = connection.getHeaderField("Content-MD5"); + String computed = DatatypeConverter.printHexBinary(md.digest()); + if (header == null || header.isEmpty()) { + // Nothing to check against, so the check goes on. + logger.warn("No MD5 hash included in data update response " + + "for '" + loggedUrl + "'. Unable to verify data " + + "integrity"); + } else if (header.equalsIgnoreCase(computed) == false) { + logCheckFailure("Integrity check failed. MD5 hash in HTTP " + + "response '" + header + "' for '" + + getEngineType(dataFile) + "'data update does not " + + "match calculated hash for the downloaded file '" + + computed + "'.", null); + return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; + } } return AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; } @@ -1108,12 +1230,6 @@ private void writeBodyTo(FileWrapper tempFile, InputStream body) throws IOExcept } } - private static boolean md5Matches(MessageDigest md, HttpURLConnection connection) { - String header = connection.getHeaderField("Content-MD5"); - String computed = DatatypeConverter.printHexBinary(md.digest()); - return header != null && header.equalsIgnoreCase(computed); - } - @Override public void close() { if (closeFutureFactory) { @@ -1166,7 +1282,7 @@ private AutoUpdateStatus checkForUpdateFromUrl( logger.error("The data file '" + dataFile.getIdentifier() + "' is checking for updates but does not have a temporary " + "file path configured."); - return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; + return AutoUpdateStatus.AUTO_UPDATE_TEMP_PATH_NOT_SET; } Path uncompressedTempFile = Paths.get( dataFile.getTempDataDirPath(), @@ -1224,8 +1340,8 @@ private AutoUpdateStatus checkForUpdateFromUrl( } return result; } catch (Throwable e) { - logger.error("Exception thrown", e); - return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; + logUnhandled(dataFile, e); + return AutoUpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR; } finally { // Make sure the temp files are cleaned up try { diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderPropertiesTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderPropertiesTests.java new file mode 100644 index 000000000..948e9dfbf --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderPropertiesTests.java @@ -0,0 +1,78 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * "Configuration files MUST be human-readable and consistent between + * languages." (pipeline-specification/features/pipeline-configuration.md + * line 50) A list of strings is a comma-separated string in a configuration + * file (line 60), so the list of properties is read the same whatever white + * space there is around each comma. + */ +public class AspectEngineBuilderPropertiesTests { + + private static List read(String properties) { + return new EmptyEngineBuilder(LoggerFactory.getILoggerFactory()) + .setProperties(properties) + .getProperties(); + } + + private static final List EXPECTED = + Arrays.asList("IsMobile", "BrowserName"); + + @Test + public void setProperties_CommaWithNoSpaces() { + assertEquals(EXPECTED, read("IsMobile,BrowserName")); + } + + @Test + public void setProperties_CommaThenSpace() { + assertEquals(EXPECTED, read("IsMobile, BrowserName")); + } + + @Test + public void setProperties_SpaceOnEachSide() { + assertEquals(EXPECTED, read("IsMobile , BrowserName")); + } + + @Test + public void setProperties_SurroundingWhiteSpace() { + assertEquals(EXPECTED, read(" IsMobile\t, BrowserName ")); + } + + @Test + public void setProperties_OneProperty() { + assertEquals(Collections.singletonList("IsMobile"), + read("IsMobile")); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceCheckFailureTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceCheckFailureTests.java new file mode 100644 index 000000000..7e869e707 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceCheckFailureTests.java @@ -0,0 +1,315 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import fiftyone.common.wrappers.io.FileWrapperFactoryDefault; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledFuture; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * A failure in a check started by update polling or by the file watcher, as + * the Errors section of the data update specification describes. The tasks + * the service schedules are kept and run here directly, so nothing waits on + * a timer. + * @see Specification + */ +@SuppressWarnings("unchecked") +public class DataUpdateServiceCheckFailureTests { + + private ch.qos.logback.classic.Logger logger; + private ListAppender appender; + private FutureFactory futureFactory; + private final List scheduled = new CopyOnWriteArrayList<>(); + private final List repeating = new CopyOnWriteArrayList<>(); + private DataUpdateServiceDefault dataUpdate; + private OnPremiseAspectEngine engine; + private String engineType; + private File dataFile; + private File tempFile; + + @Before + public void init() throws IOException { + logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger( + "DataUpdateServiceCheckFailureTests"); + logger.setLevel(Level.INFO); + appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> { + scheduled.add(invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> { + repeating.add(invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + + dataUpdate = new DataUpdateServiceDefault( + logger, null, new FileWrapperFactoryDefault(), futureFactory); + + engine = mock(OnPremiseAspectEngine.class); + engineType = engine.getClass().getSimpleName(); + dataFile = File.createTempFile("check-failure", ".dat"); + tempFile = File.createTempFile("check-failure", ".tmp"); + } + + @After + public void cleanup() { + logger.detachAppender(appender); + dataFile.delete(); + tempFile.delete(); + } + + private AspectEngineDataFileDefault newDataFile(boolean watcher) { + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(watcher == false); + config.setFileSystemWatcherEnabled(watcher); + config.setUpdateOnStartup(false); + config.setPollingIntervalSeconds(86400); + config.setMaxRandomisationSeconds(0); + config.setDataFilePath(dataFile.getAbsolutePath()); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setIdentifier("default"); + file.setEngine(engine); + file.setConfiguration(config); + file.setTempDataFilePath(tempFile.getAbsolutePath()); + when(engine.getDataFileMetaData()).thenReturn(file); + return file; + } + + private String unhandled() { + return "An unhandled error occurred while checking for automatic " + + "updates for engine '" + engineType + "'"; + } + + private void assertLoggedAtError(String message, Throwable attached) { + List errors = new ArrayList<>(); + for (ILoggingEvent event : appender.list) { + if (event.getLevel() == Level.ERROR) { + errors.add(event.getFormattedMessage()); + if (event.getFormattedMessage().equals(message) && + event.getThrowableProxy() != null && + event.getThrowableProxy().getMessage() + .equals(attached.getMessage())) { + return; + } + } + } + fail("Expected '" + message + "' at ERROR with '" + + attached.getMessage() + "' attached, but got " + errors); + } + + /** + * Run a scheduled task as the executor would. A task that throws has its + * error kept in its future, where nothing reads it, and a repeating task + * that throws never runs again. + */ + private static void runAsExecutor(Runnable task) { + try { + task.run(); + } catch (Throwable t) { + fail("The scheduled task threw, so its error is lost and a " + + "repeating task stops: " + t); + } + } + + /** + * "A failure in a check started by update polling or by the file watcher + * MUST be caught and logged, rather than being lost or ending the + * process, and is then reported to onUpdateCallbacks." + * (data-updates.md line 474) A callback that throws on update polling is + * logged at Error with the last row's message. + */ + @Test + public void polling_CallbackThrows_IsCaughtAndLogged() { + AspectEngineDataFileDefault file = newDataFile(false); + assertTrue(dataFile.setLastModified(tempFile.lastModified() + 60000)); + final RuntimeException thrown = + new IllegalStateException("callback failed"); + final List reported = + new CopyOnWriteArrayList<>(); + dataUpdate.onUpdateComplete((sender, args) -> { + reported.add(args.getStatus()); + throw thrown; + }); + dataUpdate.registerDataFile(file); + assertEquals(1, scheduled.size()); + + runAsExecutor(scheduled.get(0)); + + verify(engine, times(1)).refreshData("default"); + assertEquals(Collections.singletonList( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS), reported); + assertLoggedAtError(unhandled(), thrown); + } + + /** + * "A failure in a check started by update polling or by the file watcher + * MUST be caught and logged, rather than being lost or ending the + * process, and is then reported to onUpdateCallbacks." + * (data-updates.md line 474) and "A failure MUST NOT stop later checks + * for updates" (line 476). An unexpected error in a polling check is + * logged, reported with AUTO_UPDATE_UNKNOWN_ERROR, and the next check is + * still scheduled. + */ + @Test + public void polling_CheckThrows_IsLoggedReportedAndRescheduled() { + final RuntimeException thrown = + new IllegalStateException("check broke"); + // An unchecked error that escapes the check itself. + FailingFile failing = new FailingFile(newDataFile(false), thrown); + final List reported = new CopyOnWriteArrayList<>(); + dataUpdate.onUpdateComplete( + (sender, args) -> reported.add(args.getStatus().name())); + dataUpdate.registerDataFile(failing); + assertEquals(1, scheduled.size()); + Runnable check = scheduled.get(0); + int before = scheduled.size(); + + runAsExecutor(check); + + assertLoggedAtError(unhandled(), thrown); + assertEquals(Collections.singletonList("AUTO_UPDATE_UNKNOWN_ERROR"), + reported); + assertTrue("No later check was scheduled", + scheduled.size() > before); + } + + /** + * "A failure MUST NOT stop later checks for updates, so a later good + * data file is still applied." (data-updates.md line 476) A callback + * that throws during a watcher check must not end the repeating watcher + * task, and the watch key is reset so the next change is seen. + */ + @Test + public void watcher_CallbackThrows_LaterFileStillApplied() { + AspectEngineDataFileDefault file = newDataFile(true); + final RuntimeException thrown = + new IllegalStateException("callback failed"); + final boolean[] fail = {true}; + dataUpdate.onUpdateComplete((sender, args) -> { + if (fail[0]) { + throw thrown; + } + }); + dataUpdate.registerDataFile(file); + assertEquals(1, repeating.size()); + assertNotNull(file.getConfiguration().getWatchKey()); + file.getConfiguration().getWatchKey().cancel(); + + WatchEvent event = mock(WatchEvent.class); + doReturn(StandardWatchEventKinds.ENTRY_MODIFY).when(event).kind(); + when(event.context()).thenReturn( + Paths.get(dataFile.getName())); + WatchKey key = mock(WatchKey.class); + when(key.pollEvents()).thenReturn( + Collections.>singletonList(event)); + file.getConfiguration().setWatchKey(key); + + assertTrue(dataFile.setLastModified( + System.currentTimeMillis() + 60000)); + runAsExecutor(repeating.get(0)); + verify(key, atLeastOnce()).reset(); + assertLoggedAtError(unhandled(), thrown); + + // A later good file is applied by the next run. + fail[0] = false; + assertTrue(dataFile.setLastModified( + System.currentTimeMillis() + 120000)); + runAsExecutor(repeating.get(0)); + verify(engine, times(2)).refreshData(anyString()); + } + + /** + * A data file whose check throws an unchecked error that the service + * does not expect, to stand for any such error in a polling check. + */ + private static class FailingFile extends AspectEngineDataFileDefault { + private final RuntimeException error; + private volatile boolean armed = false; + + FailingFile(AspectEngineDataFileDefault from, RuntimeException error) { + this.error = error; + setIdentifier(from.getIdentifier()); + setEngine(from.getEngine()); + setConfiguration(from.getConfiguration()); + setTempDataFilePath(from.getTempDataFilePath()); + } + + @Override + public void setFuture(ScheduledFuture future) { + super.setFuture(future); + armed = true; + } + + @Override + public String getTempDataFilePath() { + if (armed) { + throw error; + } + return super.getTempDataFilePath(); + } + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java index c93f6369c..13d7bface 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java @@ -417,36 +417,136 @@ public void TemporaryFileCannotBeWritten_LogsTheWriteError() assertTrue(logged(Level.WARN).isEmpty()); } + /** + * "These messages MUST also be consistent across languages to make life + * easier for 51Degrees support." (data-updates.md line 481) The + * integrity check row is at Error, with the table's message and + * AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED. + */ @Test - public void Md5DoesNotMatch_WarnsAboutIntegrityCheck() throws Exception { + public void Md5DoesNotMatch_LogsTheTableRow() throws Exception { + byte[] body = new byte[]{1, 2, 3}; when(connection.getResponseCode()) .thenReturn(HttpURLConnection.HTTP_OK); when(connection.getInputStream()) - .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + .thenReturn(new ByteArrayInputStream(body)); when(connection.getHeaderField("Content-MD5")).thenReturn("0"); AspectEngineDataFileDefault file = newDataFile(true, false, false); file.getConfiguration().setVerifyMd5(true); + String computed = jakarta.xml.bind.DatatypeConverter.printHexBinary( + java.security.MessageDigest.getInstance("MD5").digest(body)); - dataUpdate.checkForUpdate(engine); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED, + dataUpdate.checkForUpdate(engine)); + + assertCheckFailed("Integrity check failed. MD5 hash in HTTP " + + "response '0' for '" + engineType + "'data update does not " + + "match calculated hash for the downloaded file '" + computed + + "'.", null); + } + + /** + * "These messages MUST also be consistent across languages to make life + * easier for 51Degrees support." (data-updates.md line 481) A response + * with no Content-MD5 header is a Warning, and the check goes on. + */ + @Test + public void NoMd5Header_WarnsAndGoesOn() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + when(connection.getHeaderField("Content-MD5")).thenReturn(null); + AspectEngineDataFileDefault file = newDataFile(true, false, false); + file.getConfiguration().setVerifyMd5(true); - assertLogged(Level.WARN, "An error occurred during the integrity " + - "check of new data file for " + engineType + ". Error detail: " + - "the MD5 hash of the download does not match the Content-MD5 " + - "header"); + assertEquals(DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS, + dataUpdate.checkForUpdate(engine)); + + assertLogged(Level.WARN, "No MD5 hash included in data update " + + "response for '" + UPDATE_URL + "'. Unable to verify data " + + "integrity"); + verify(engine).refreshData(anyString(), any(byte[].class)); } + /** + * "These messages MUST also be consistent across languages to make life + * easier for 51Degrees support." (data-updates.md line 481) No temporary + * path gives AUTO_UPDATE_TEMP_PATH_NOT_SET. + */ @Test public void NoTemporaryFilePath_LogsError() { AspectEngineDataFileDefault file = newDataFile(false, false, false); file.setTempDataDirPath(null); when(engine.getTempDataDirPath()).thenReturn(null); - dataUpdate.checkForUpdate(engine); + assertEquals("AUTO_UPDATE_TEMP_PATH_NOT_SET", + dataUpdate.checkForUpdate(engine).name()); assertLogged(Level.ERROR, "The data file 'default' is checking for " + "updates but does not have a temporary file path configured."); } + /** + * "These messages MUST also be consistent across languages to make life + * easier for 51Degrees support." (data-updates.md line 481) An error of + * any other kind is logged at Error with the last row's message and the + * error attached, and gives AUTO_UPDATE_UNKNOWN_ERROR. + */ + @Test + public void UnexpectedError_LogsTheLastRow() { + AspectEngineDataFileDefault file = newDataFile(false, false, false); + // The engine's copy is the data file itself, so the file on disk is + // not newer and the URL is checked. + file.setTempDataFilePath(dataFile.getAbsolutePath()); + final IllegalStateException broke = + new IllegalStateException("formatter broke"); + file.getConfiguration().setUrlFormatter(d -> { + throw broke; + }); + + assertEquals("AUTO_UPDATE_UNKNOWN_ERROR", + dataUpdate.checkForUpdate(engine).name()); + + for (ILoggingEvent event : appender.list) { + if (event.getLevel() == Level.ERROR && + event.getFormattedMessage().equals("An unhandled error " + + "occurred while checking for automatic updates for " + + "engine '" + engineType + "'") && + event.getThrowableProxy() != null && + "formatter broke".equals( + event.getThrowableProxy().getMessage())) { + return; + } + } + fail("No unhandled error entry in " + logged(Level.ERROR)); + } + + /** + * "These messages MUST also be consistent across languages to make life + * easier for 51Degrees support." (data-updates.md line 481) A failed + * write of data passed in from memory is a Warning, and the Engine is + * still refreshed. + */ + @Test + public void UpdateFromMemory_WriteFails_WarnsAndRefreshes() { + AspectEngineDataFileDefault file = newDataFile(false, false, false); + // A folder cannot be written as a file. + String path = System.getProperty("java.io.tmpdir"); + file.getConfiguration().setDataFilePath(path); + + assertEquals(DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS, + dataUpdate.updateFromMemory(file, new byte[]{1, 2, 3})); + + assertLogged(Level.WARN, "An error occurred when writing to '" + + path + "'. The engine will be updated to use the new data but " + + "the file on disk will still contain old data."); + assertTrue("Logged at Error: " + logged(Level.ERROR), + logged(Level.ERROR).isEmpty()); + verify(engine).refreshData(anyString(), any(byte[].class)); + } + /** * A data file whose formatted URL carries a license key, as the * Distributor's URL does, with every level logged. diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 6f139fca2..134caec09 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -762,9 +762,9 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { } }); - // The invalid MD5 will cause a warning to be logged so we want - // to ignore this - ignoreWranings = 1; + // The invalid MD5 is logged at Error, as the specification's Errors + // table gives, so we want to ignore this + ignoreErrors = 1; OnPremiseAspectEngine engine = mock(OnPremiseAspectEngine.class); DataFileConfigurationDefault config = new DataFileConfigurationDefault(); diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index 30e7ffbf2..08052c255 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -69,6 +69,10 @@ public class JavaScriptBuilderElement protected String contextRoot; protected final String objName; protected final boolean enableCookies; + /** + * True if the script is minified before it is returned. + */ + protected final boolean minify; private final Mustache mustache; /** @@ -205,8 +209,49 @@ public JavaScriptBuilderElement( String host, String protocol, String contextRoot) { + this(logger, elementDataFactory, endpoint, objName, enableCookies, + host, protocol, contextRoot, + JavaScriptBuilderElementBuilder.MINIFY); + } + //! [constructor] + + //! [constructor] + /** + * Constructor. + * @param logger The logger. + * @param elementDataFactory The element data factory. + * @param endpoint Set the endpoint which will be queried on the host. + * e.g /api/v4/json + * @param objName The default name of the object instantiated by the client + * JavaScript. + * @param enableCookies Set whether the client JavaScript stored results of + * client side processing in cookies. + * @param host The host that the client JavaScript should query for updates. + * If null or blank then the host from the request will be used + * @param protocol The protocol (HTTP or HTTPS) that the client JavaScript + * will use when querying for updates. If null or empty + * then the protocol from the request will be used + * @param contextRoot The <context-root> setting from the web.xml. + * This is needed when creating the callback URL. + * @param minify True if the script is minified before it is returned + * @throws PipelineConfigurationException if protocol is not null or empty + * and is neither http nor https in any case of its ASCII letters, white + * space included, or if objName is not null and is + * not a valid JavaScript identifier, see {@link #isValidObjectName}. + */ + public JavaScriptBuilderElement( + Logger logger, + ElementDataFactory elementDataFactory, + String endpoint, + String objName, + boolean enableCookies, + String host, + String protocol, + String contextRoot, + boolean minify) { super(logger, elementDataFactory); - + this.minify = minify; + MustacheFactory mf = new DefaultMustacheFactory(); InputStream in = getClass().getResourceAsStream(Constants.TEMPLATE); BufferedReader reader = new BufferedReader(new InputStreamReader(in)); @@ -408,8 +453,10 @@ private Map getParameters(FlowData data) throws UnsupportedEncod .asKeyMap(); for(Map.Entry entry : queryEvidence.entrySet()){ - if(entry.getKey().startsWith(EVIDENCE_QUERY_PREFIX) && - EXCLUDED_PARAMETERS.contains(entry.getKey()) == false){ + // Evidence keys are case-insensitive. + String lowerKey = entry.getKey().toLowerCase(Locale.ROOT); + if(lowerKey.startsWith(EVIDENCE_QUERY_PREFIX + EVIDENCE_SEPERATOR) && + EXCLUDED_PARAMETERS.contains(lowerKey) == false){ String key = entry.getKey().substring(entry.getKey().indexOf(EVIDENCE_SEPERATOR) + 1); key = URLEncoder.encode(key, "UTF-8"); String value = URLEncoder.encode(entry.getValue().toString(), "UTF-8"); @@ -624,6 +671,9 @@ private void buildJavaScript( mustache.execute(stringWriter, javaScriptObj.asMap()); String content = stringWriter.toString(); + if (minify) { + content = JavaScriptMinifier.minify(content); + } elementData.setJavaScript(content); } diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java index 47a449373..b0f7815a8 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java @@ -40,6 +40,7 @@ @ElementBuilder public class JavaScriptBuilderElementBuilder { public static final boolean ENABLE_COOKIES = true; + public static final boolean MINIFY = true; private final ILoggerFactory loggerFactory; private final Logger logger; @@ -49,6 +50,7 @@ public class JavaScriptBuilderElementBuilder { protected String contextRoot = ""; protected String objName = Constants.DEFAULT_OBJECT_NAME; private boolean enableCookies = ENABLE_COOKIES; + private boolean minify = MINIFY; /** * Construct a new instance. @@ -171,6 +173,21 @@ public JavaScriptBuilderElementBuilder setEnableCookies( return this; } + /** + * Set whether the JavaScript that is produced is minified. Minification + * removes comments, indentation and empty lines, and keeps line breaks + * between statements. + *

+ * Default is true + * @param minify true to minify the JavaScript + * @return this builder + */ + @DefaultValue(booleanValue = MINIFY) + public JavaScriptBuilderElementBuilder setMinify(boolean minify) { + this.minify = minify; + return this; + } + /** * Build the {@link JavaScriptBuilderElement} * @return new {@link JavaScriptBuilderElement} instance @@ -194,7 +211,8 @@ public JavaScriptBuilderData create( enableCookies, host, protocol, - contextRoot); - } + contextRoot, + minify); + } } //! [class] diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptMinifier.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptMinifier.java new file mode 100644 index 000000000..fef30f8f1 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptMinifier.java @@ -0,0 +1,249 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder.flowelements; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * A cut-down minifier for the script the {@link JavaScriptBuilderElement} + * renders, as the specification allows where a full minifier would cost too + * much on every request. It makes one pass over the script and: + *

    + *
  • Removes comments.
  • + *
  • Removes indentation, trailing white space and empty lines.
  • + *
  • Collapses other runs of white space to one space, and removes that + * space next to punctuation where nothing can join.
  • + *
+ * Line breaks between statements are kept, so automatic semicolon insertion + * works as it did. String, template and regular expression literals are + * copied as they are. + */ +public final class JavaScriptMinifier { + + /** + * Punctuation that a space can be removed next to. A slash is left out, + * as are plus and minus, so that {@code a + +b} keeps its meaning, and + * the angle brackets, so that no HTML comment closer can be formed. + */ + private static final String TIGHT = "{}()[];,:=?*%&|!"; + + /** + * A slash after one of these characters starts a regular expression + * rather than a division. + */ + private static final String BEFORE_REGEX = "(,=:[!&|?{};+-*%<>~^"; + + /** + * A slash after one of these words starts a regular expression. + */ + private static final Set KEYWORDS_BEFORE_REGEX = + Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + "return", "typeof", "case", "do", "else", "in", "of", "new", + "delete", "void", "throw", "instanceof", "yield", "await"))); + + private JavaScriptMinifier() { + } + + /** + * Minify the script given. + * @param script the JavaScript to minify + * @return the minified JavaScript, or null if the script is null + */ + public static String minify(String script) { + if (script == null) { + return null; + } + int length = script.length(); + StringBuilder out = new StringBuilder(length); + boolean pendingSpace = false; + boolean pendingNewline = false; + int i = 0; + while (i < length) { + char c = script.charAt(i); + char next = i + 1 < length ? script.charAt(i + 1) : '\0'; + if (c == '\n' || c == '\r') { + pendingNewline = true; + i++; + continue; + } + if (c == ' ' || c == '\t' || c == '\f' || c == '\u000B') { + pendingSpace = true; + i++; + continue; + } + if (c == '/' && next == '/') { + // A line comment runs to the end of the line, which is kept. + i += 2; + while (i < length && + script.charAt(i) != '\n' && script.charAt(i) != '\r') { + i++; + } + continue; + } + if (c == '/' && next == '*') { + int end = script.indexOf("*/", i + 2); + end = end < 0 ? length : end + 2; + // A block comment with a line break in it counts as a line + // break, and one without counts as a space. + boolean newline = false; + for (int j = i; j < end && newline == false; j++) { + char d = script.charAt(j); + newline = d == '\n' || d == '\r'; + } + if (newline) { + pendingNewline = true; + } else { + pendingSpace = true; + } + i = end; + continue; + } + + if (out.length() > 0) { + if (pendingNewline) { + out.append('\n'); + } else if (pendingSpace && + needsSpace(out.charAt(out.length() - 1), c)) { + out.append(' '); + } + } + pendingSpace = false; + pendingNewline = false; + + if (c == '\'' || c == '"' || c == '`') { + i = copyString(script, i, out); + } else if (c == '/' && regexAllowed(out)) { + i = copyRegex(script, i, out); + } else { + out.append(c); + i++; + } + } + return out.toString(); + } + + /** + * Whether the space between two characters is kept. It is kept between + * two characters that are not punctuation, and wherever removing it + * would form a comment or an HTML comment opener. + */ + private static boolean needsSpace(char before, char after) { + if ((before == '/' && (after == '/' || after == '*')) || + (before == '*' && after == '/') || + (before == '<' && after == '!')) { + return true; + } + return TIGHT.indexOf(before) < 0 && TIGHT.indexOf(after) < 0; + } + + /** + * Copy the string or template literal that starts at the index given, + * and return the index after it. A quote string that reaches the end of + * a line unclosed is not valid, and is copied to there. + */ + private static int copyString(String script, int start, StringBuilder out) { + char quote = script.charAt(start); + int length = script.length(); + out.append(quote); + int i = start + 1; + while (i < length) { + char c = script.charAt(i); + if (c == '\\' && i + 1 < length) { + out.append(c).append(script.charAt(i + 1)); + i += 2; + continue; + } + if (quote != '`' && (c == '\n' || c == '\r')) { + return i; + } + out.append(c); + i++; + if (c == quote) { + return i; + } + } + return i; + } + + /** + * Copy the regular expression literal that starts at the index given, + * and return the index after its closing slash. Its flags follow as + * ordinary characters. + */ + private static int copyRegex(String script, int start, StringBuilder out) { + int length = script.length(); + out.append('/'); + int i = start + 1; + boolean inClass = false; + while (i < length) { + char c = script.charAt(i); + if (c == '\n' || c == '\r') { + return i; + } + if (c == '\\' && i + 1 < length) { + out.append(c).append(script.charAt(i + 1)); + i += 2; + continue; + } + out.append(c); + i++; + if (c == '[') { + inClass = true; + } else if (c == ']') { + inClass = false; + } else if (c == '/' && inClass == false) { + return i; + } + } + return i; + } + + /** + * Whether a slash written next is the start of a regular expression, + * from what was written before it. + */ + private static boolean regexAllowed(StringBuilder out) { + int i = out.length() - 1; + while (i >= 0 && (out.charAt(i) == ' ' || out.charAt(i) == '\n')) { + i--; + } + if (i < 0) { + return true; + } + char before = out.charAt(i); + if (BEFORE_REGEX.indexOf(before) >= 0) { + return true; + } + if (Character.isJavaIdentifierPart(before)) { + int end = i + 1; + while (i >= 0 && Character.isJavaIdentifierPart(out.charAt(i))) { + i--; + } + return KEYWORDS_BEFORE_REGEX.contains(out.substring(i + 1, end)); + } + return false; + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java index 7ec0c95e1..9c87b0f97 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java @@ -256,8 +256,10 @@ public void JavaScriptBuilderElement_DelayExecution() throws Exception { public void JavaScriptBuilderElement_VerifySession() throws Exception { configureMocks(); + // The rendered text is read, so it is not minified. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); @@ -326,8 +328,10 @@ public void JavaScriptBuilderElement_ParametersExcludeSessionAndSequence() throws Exception { configureMocks(); + // The rendered text is read, so it is not minified. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); String javaScript = result.getJavaScript(); @@ -378,9 +382,11 @@ public void JavaScriptBuilderElement_NoParameters() throws Exception { evidence.remove("query.latitude"); evidence.remove("query.longitude"); + // The rendered text is read, so it is not minified. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) .setEndpoint("/json") + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java new file mode 100644 index 000000000..491b2e8c7 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java @@ -0,0 +1,138 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptMinifier; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The minification of the script, which the SetMinify option turns on and + * off and which is on by default, as the configuration options in the + * JavaScript Builder specification give. The performance of minification is + * tested in {@link fiftyone.pipeline.javascriptbuilder.performance.MinifyOverheadTests}. + */ +public class JavaScriptMinifyTests { + private final TestLoggerFactory loggerFactory; + + public JavaScriptMinifyTests() { + ILoggerFactory internalLogger = mock(ILoggerFactory.class); + when(internalLogger.getLogger(anyString())) + .thenReturn(mock(Logger.class)); + loggerFactory = new TestLoggerFactory(internalLogger); + } + + /** + * Render the script with the builder given. + */ + static String render( + ILoggerFactory loggerFactory, + JavaScriptBuilderElementBuilder builder) throws Exception { + JavaScriptBuilderElement jsElement = builder + .setEndpoint("/json") + .setHost("example.com") + .setProtocol("https") + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + .addFlowElement(jsElement) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + return flowData.getFromElement(jsElement).getJavaScript(); + } + } + + @Test + public void Default_IsMinified() throws Exception { + String full = render(loggerFactory, + new JavaScriptBuilderElementBuilder(loggerFactory) + .setMinify(false)); + String byDefault = render(loggerFactory, + new JavaScriptBuilderElementBuilder(loggerFactory)); + + assertTrue(full.contains("// Set to true when the `catchError` is called."), + "The full script has lost its comments."); + assertFalse(byDefault.contains("// Set to true when the `catchError` is called."), + "The default script was not minified."); + assertTrue(byDefault.length() < full.length() * 3 / 4, + "Minified " + byDefault.length() + " characters from " + + full.length() + "."); + } + + @Test + public void Minify_RemovesCommentsAndIndentationKeepsLineBreaks() { + assertEquals("var a=1\nvar b=a + 1;", + JavaScriptMinifier.minify( + " // first\n var a = 1 /* one */\n\n\n" + + " var b = a + 1; // second\n")); + } + + @Test + public void Minify_KeepsStringsTemplatesAndRegularExpressions() { + assertEquals("var u='http://x/*y*/'", + JavaScriptMinifier.minify("var u = 'http://x/*y*/'")); + assertEquals("var s=\"a b // c\"", + JavaScriptMinifier.minify("var s = \"a b // c\"")); + assertEquals("var t=`one\n two // three`", + JavaScriptMinifier.minify("var t = `one\n two // three`")); + assertEquals("var r=/[/]*\\/ x/g.test(s)", + JavaScriptMinifier.minify("var r = /[/]*\\/ x/g.test(s)")); + assertEquals("return /a b/.test(s)", + JavaScriptMinifier.minify("return /a b/.test(s)")); + assertEquals("var e=new RegExp('a\\\\.b\\'c','g')", + JavaScriptMinifier.minify("var e = new RegExp('a\\\\.b\\'c', 'g')")); + } + + @Test + public void Minify_KeepsTheMeaningOfOperators() { + assertEquals("x=a / b / c", + JavaScriptMinifier.minify("x = a / b / c")); + assertEquals("x=a + +b - -c", + JavaScriptMinifier.minify("x = a + +b - -c")); + assertEquals("if(a < !b){return c}", + JavaScriptMinifier.minify("if (a < !b) { return c }")); + assertEquals("x=a-- > b", + JavaScriptMinifier.minify("x = a-- > b")); + assertEquals("typeof a", + JavaScriptMinifier.minify("typeof a")); + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java index 712bc6873..b8ccefa49 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java @@ -115,7 +115,7 @@ public void init() { @Test public void ObjectName_FromBuilder_UsedThroughout() { JavaScriptBuilderElement element = - new JavaScriptBuilderElementBuilder(loggerFactory) + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) .setObjectName("myFod") .build(); @@ -128,7 +128,7 @@ public void ObjectName_FromBuilder_UsedThroughout() { @Test public void ObjectName_FromEvidence_UsedThroughout() { JavaScriptBuilderElement element = - new JavaScriptBuilderElementBuilder(loggerFactory).build(); + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false).build(); evidence.put(EVIDENCE_OBJECT_NAME, "myFod"); String script = render(element); @@ -160,7 +160,7 @@ public void ObjectName_InvalidFromEvidence_DefaultUsed( String name, String[] absent) { JavaScriptBuilderElement element = - new JavaScriptBuilderElementBuilder(loggerFactory).build(); + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false).build(); evidence.put(EVIDENCE_OBJECT_NAME, name); String script = render(element); @@ -181,7 +181,7 @@ public void ObjectName_InvalidFromEvidence_DefaultUsed( @Test public void ObjectName_InvalidFromEvidence_ConfiguredNameUsed() { JavaScriptBuilderElement element = - new JavaScriptBuilderElementBuilder(loggerFactory) + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) .setObjectName("myFod") .build(); evidence.put(EVIDENCE_OBJECT_NAME, "9bad"); @@ -198,7 +198,7 @@ public void ObjectName_InvalidFromEvidence_ConfiguredNameUsed() { public void ObjectName_InvalidFromBuilder_Refused(String name) { assertThrows( PipelineConfigurationException.class, - () -> new JavaScriptBuilderElementBuilder(loggerFactory) + () -> new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) .setObjectName(name) .build()); } @@ -259,7 +259,7 @@ static Stream allowedWindowNames() { @MethodSource("refusedNames") public void ObjectName_RefusedNameFromEvidence_DefaultUsed(String name) { JavaScriptBuilderElement element = - new JavaScriptBuilderElementBuilder(loggerFactory).build(); + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false).build(); evidence.put(EVIDENCE_OBJECT_NAME, name); String script = render(element); @@ -279,7 +279,7 @@ public void ObjectName_RefusedNameFromEvidence_DefaultUsed(String name) { public void ObjectName_RefusedNameFromBuilder_Refused(String name) { assertThrows( PipelineConfigurationException.class, - () -> new JavaScriptBuilderElementBuilder(loggerFactory) + () -> new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) .setObjectName(name) .build()); } @@ -310,7 +310,7 @@ public void ObjectName_IsValidObjectName_RefusedName(String name) { public void ObjectName_WindowPropertyVarCanReplace_Allowed(String name) { assertTrue(JavaScriptBuilderElement.isValidObjectName(name)); JavaScriptBuilderElement element = - new JavaScriptBuilderElementBuilder(loggerFactory) + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) .setObjectName(name) .build(); @@ -334,11 +334,11 @@ public void ObjectName_NearMissOfRefusedName_Allowed(String name) { @Test public void ObjectName_NotConfigured_DefaultUsed() { String fromBuilder = render( - new JavaScriptBuilderElementBuilder(loggerFactory).build()); + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false).build()); assertWorkingObject(fromBuilder, Constants.DEFAULT_OBJECT_NAME); String fromNull = render( - new JavaScriptBuilderElementBuilder(loggerFactory) + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) .setObjectName(null) .build()); assertWorkingObject(fromNull, Constants.DEFAULT_OBJECT_NAME); diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index fc8cb2f61..6f94dda3e 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -69,7 +69,7 @@ public JavaScriptUrlTests() { private String render(String contextRoot) throws Exception { JavaScriptBuilderElementBuilder builder = - new JavaScriptBuilderElementBuilder(loggerFactory) + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) .setEndpoint("/json") .setHost("example.com") .setProtocol("https"); @@ -139,6 +139,48 @@ public void Url_CarriesNoQueryEvidence() throws Exception { "The parameters object lost the query evidence: " + parameters); } + /** + * "Keys MUST be case-insensitive and will usually be split into + * <prefix>.<field>." (pipeline-specification/features/evidence.md + * line 10) Query evidence whose prefix is not in lower case still + * reaches the parameters object, and the session id and sequence are + * still left out whatever their case. + */ + @Test + public void Parameters_QueryPrefixIsCaseInsensitive() throws Exception { + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) + .setEndpoint("/json") + .setHost("example.com") + .setProtocol("https") + .build(); + String javaScript; + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + .addFlowElement(jsElement) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("Query.Upper", "kept"); + flowData.addEvidence("QUERY.SESSION-ID", "left-out"); + flowData.process(); + javaScript = flowData.getFromElement(jsElement).getJavaScript(); + } + String parameters = null; + for (String line : javaScript.split("\\r?\\n")) { + if (line.toLowerCase().contains("parameters =") + && line.contains("{")) { + parameters = line; + break; + } + } + assertNotNull(parameters, "No parameters object in the script."); + assertTrue(parameters.contains("\"Upper\":\"kept\""), + "Query evidence with prefix 'Query.' was dropped: " + parameters); + assertFalse(parameters.contains("left-out"), + "The session id was sent as a parameter: " + parameters); + } + /** * A context root still sits between the host and the endpoint, and the * URL still carries no query string. @@ -166,7 +208,7 @@ public void Url_KeepsContextRoot() throws Exception { private List renderWithHosts(String... hosts) throws Exception { TestLoggerFactory factory = hostLoggerFactory; JavaScriptBuilderElement jsElement = - new JavaScriptBuilderElementBuilder(factory) + new JavaScriptBuilderElementBuilder(factory).setMinify(false) .setEndpoint("/json") .build(); List scripts = new ArrayList<>(); @@ -268,7 +310,7 @@ public void Url_RefusedHost_LoggedWithoutUserInformation( public void Url_RefusedProtocol_NoCallbackAndLoggedOnce() throws Exception { TestLoggerFactory factory = hostLoggerFactory; JavaScriptBuilderElement jsElement = - new JavaScriptBuilderElementBuilder(factory) + new JavaScriptBuilderElementBuilder(factory).setMinify(false) .setEndpoint("/json") .build(); try (Pipeline pipeline = new PipelineBuilder(factory) @@ -307,7 +349,7 @@ public void Url_RefusedProtocol_NoCallbackAndLoggedOnce() throws Exception { public void Protocol_ConfiguredOtherThanHttpOrHttps_IsRefused( String protocol) { JavaScriptBuilderElementBuilder builder = - new JavaScriptBuilderElementBuilder(loggerFactory); + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false); assertThrows(PipelineConfigurationException.class, () -> builder.setProtocol(protocol)); } @@ -315,7 +357,7 @@ public void Protocol_ConfiguredOtherThanHttpOrHttps_IsRefused( @ParameterizedTest @ValueSource(strings = {"http", "https", "HTTPS"}) public void Protocol_ConfiguredHttpOrHttps_IsAccepted(String protocol) { - assertNotNull(new JavaScriptBuilderElementBuilder(loggerFactory) + assertNotNull(new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false) .setProtocol(protocol)); } diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java index 06bc9b6b6..387ac4124 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java @@ -333,7 +333,7 @@ public void Sequence_InvalidWithSequenceElement_RenderedOne(int value) private String renderWithSequenceData(Object sessionId, Object sequence) throws Exception { JavaScriptBuilderElement jsElement = - new JavaScriptBuilderElementBuilder(loggerFactory).build(); + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false).build(); PipelineBuilder builder = new PipelineBuilder(loggerFactory); builder.addFlowElement(new SequenceStandIn( loggerFactory.getLogger("sequence"), sessionId, sequence)); @@ -361,7 +361,7 @@ private String render( String sessionId, Object sequence) throws Exception { JavaScriptBuilderElement jsElement = - new JavaScriptBuilderElementBuilder(loggerFactory).build(); + new JavaScriptBuilderElementBuilder(loggerFactory).setMinify(false).build(); PipelineBuilder builder = new PipelineBuilder(loggerFactory); if (sequenceElement) { builder.addFlowElement( diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/performance/MinifyOverheadTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/performance/MinifyOverheadTests.java new file mode 100644 index 000000000..9ec5d87b4 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/performance/MinifyOverheadTests.java @@ -0,0 +1,244 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder.performance; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptMinifier; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * "Performance tests MUST be in place to verify the performance impact of + * this feature." (pipeline-specification/pipeline-elements/javascript-builder.md + * line 109) The feature is the minification of the script. These tests time + * the minification of a rendered script, and Process calls through the + * JavaScript Builder with minification on and off. + */ +public class MinifyOverheadTests { + + private static final Map overheadMillis = + new LinkedHashMap<>(); + + /** + * Writes the measured times to the results file, if one was asked for. + */ + @AfterAll + public static void writeResults() throws IOException { + String path = System.getProperty("fiftyone.performance.json"); + if (path == null) { + return; + } + JSONObject results = new JSONObject() + .put("LowerIsBetter", new JSONObject(overheadMillis)); + Files.write(Paths.get(path), + results.toString(2).getBytes(StandardCharsets.UTF_8)); + } + + /** + * Rounds measured, and the number discarded first so that the JIT has + * compiled the code. The median of the rounds is checked, so one round + * slowed by a busy runner or a garbage collection does not fail a test, + * whilst a cost on every call does. + */ + private static final int MeasuredRounds = 7; + private static final int WarmUpRounds = 2; + + /** + * The most the median minification of one script may take. Two runs on + * a busy Windows 11 x64 machine with JDK 21 measured medians of 0.20 ms + * and 0.34 ms, so this is about ten times the slower and a runner + * several times slower than that machine still passes. + */ + private static final double MinifyLimitMillis = 3; + + /** + * The most the median Process call through the JavaScript Builder with + * minification on may take more than one with it off. The same two runs + * measured 0.31 ms and 0.53 ms, with Process calls of 0.11 ms and 0.22 ms + * with minification off, so this is about ten times the slower. + */ + private static final double ProcessOverheadLimitMillis = 5; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private Pipeline buildPipeline(JavaScriptBuilderElement jsElement) + throws Exception { + return new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + .addFlowElement(jsElement) + .build(); + } + + private JavaScriptBuilderElement buildElement(boolean minify) { + return new JavaScriptBuilderElementBuilder(loggerFactory) + .setEndpoint("/json") + .setHost("example.com") + .setProtocol("https") + .setMinify(minify) + .build(); + } + + /** + * The full script as the element renders it with minification off. + */ + private String renderFull() throws Exception { + JavaScriptBuilderElement jsElement = buildElement(false); + try (Pipeline pipeline = buildPipeline(jsElement); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + return flowData.getFromElement(jsElement).getJavaScript(); + } + } + + private static double median(double[] rounds) { + double[] sorted = rounds.clone(); + Arrays.sort(sorted); + return sorted[sorted.length / 2]; + } + + /** + * Fails when the median round is not below the limit. The message lists + * every round, because the build log shows only the message, and the + * rounds tell a runner slow for the whole test from a slow round or two. + */ + private static void assertBelowLimit( + String what, + double[] rounds, + double limitMillis) { + double median = median(rounds); + Assertions.assertTrue(median < limitMillis, + what + " was " + median + "ms. Maximum permitted is " + + limitMillis + ". Rounds in ms " + Arrays.toString(rounds)); + } + + private static double minifyRound(String script, int iterations) { + long start = System.nanoTime(); + int length = 0; + for (int i = 0; i < iterations; i++) { + length += JavaScriptMinifier.minify(script).length(); + } + long end = System.nanoTime(); + Assertions.assertTrue(length > 0); + return (end - start) / 1_000_000.0 / iterations; + } + + @Test + public void MinifyOverhead_PerScript() throws Exception { + String script = renderFull(); + int iterations = 500; + for (int i = 0; i < WarmUpRounds; i++) { + minifyRound(script, iterations); + } + double[] rounds = new double[MeasuredRounds]; + for (int i = 0; i < rounds.length; i++) { + rounds[i] = minifyRound(script, iterations); + } + overheadMillis.put("MinifyOverhead_PerScript_ms", median(rounds)); + assertBelowLimit("Minifying one script", rounds, MinifyLimitMillis); + } + + /** + * Returns the milliseconds per Process call, for one round. The FlowData + * are all made before the timing starts and closed after it ends. + */ + @SuppressWarnings("PMD.FlowDataInTryWithResources") + private static double processRound(Pipeline pipeline, int iterations) + throws Exception { + List data = new ArrayList<>(); + long start; + long end; + try { + for (int i = 0; i < iterations; i++) { + FlowData flowData = pipeline.createFlowData(); + data.add(flowData); + flowData.addEvidence("query.user-agent", "iPhone"); + } + start = System.nanoTime(); + for (FlowData flowData : data) { + flowData.process(); + } + end = System.nanoTime(); + } finally { + for (FlowData flowData : data) { + flowData.close(); + } + } + return (end - start) / 1_000_000.0 / iterations; + } + + /** + * Times Process calls with minification on and off in turn, round by + * round, so that a runner that slows down part way through slows both + * alike. The difference of each pair of rounds is the overhead. + */ + @Test + public void MinifyOverhead_PerProcessCall() throws Exception { + int iterations = 200; + try (Pipeline minified = buildPipeline(buildElement(true)); + Pipeline full = buildPipeline(buildElement(false))) { + for (int i = 0; i < WarmUpRounds; i++) { + processRound(minified, iterations); + processRound(full, iterations); + } + double[] on = new double[MeasuredRounds]; + double[] off = new double[MeasuredRounds]; + double[] overhead = new double[MeasuredRounds]; + for (int i = 0; i < MeasuredRounds; i++) { + on[i] = processRound(minified, iterations); + off[i] = processRound(full, iterations); + overhead[i] = on[i] - off[i]; + } + overheadMillis.put("Process_MinifyOn_ms", median(on)); + overheadMillis.put("Process_MinifyOff_ms", median(off)); + overheadMillis.put("MinifyOverhead_PerProcessCall_ms", + median(overhead)); + assertBelowLimit("Minification overhead per Process call " + + "(on " + Arrays.toString(on) + ", off " + + Arrays.toString(off) + ")", + overhead, ProcessOverheadLimitMillis); + } + } +}