diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java new file mode 100644 index 000000000..a3ec7ff01 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java @@ -0,0 +1,215 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.TestsBase; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Refreshing the engine's data, with the free Lite data file. Refreshes + * from several threads at once leave a working engine, a refresh from data + * that cannot be used fails and leaves the data the engine had, and + * processing while the data is refreshed gives the same values. + */ +public class DataRefreshTests extends TestsBase { + + private static final int THREADS = 4; + + private static final int REFRESHES = 5; + + private static final long WAIT_SECONDS = 300; + + private static final List USER_AGENTS = Arrays.asList( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 " + + "Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + + private ExecutorService executor; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.MaxPerformance); + executor = Executors.newFixedThreadPool(THREADS); + } + + @After + public void cleanup() { + executor.shutdownNow(); + testCleanup(); + } + + /** + * Several threads refresh the engine from its data file at once, and + * the engine then gives the values it gave before. + */ + @Test + public void Refresh_FromSeveralThreads_EngineStillWorks() + throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + CountDownLatch start = new CountDownLatch(1); + List> refreshers = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + refreshers.add(executor.submit(() -> { + start.await(); + for (int i = 0; i < REFRESHES; i++) { + engine.refreshData(identifier); + } + return REFRESHES; + })); + } + start.countDown(); + for (Future refresher : refreshers) { + assertEquals(Integer.valueOf(REFRESHES), + refresher.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + assertEquals(expected, isMobile()); + } + + /** + * A refresh from data that is not a data file fails with an error, and + * the engine goes on using the data it had. + */ + @Test + public void Refresh_DataNotUsable_FailsAndKeepsTheData() throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String published = String.valueOf( + engine.getDataFileMetaData().getDataPublishedDateTime()); + byte[] notData = "This is not a data file.".getBytes( + StandardCharsets.UTF_8); + + boolean failed = false; + try { + engine.refreshData( + engine.getDataFileMetaData().getIdentifier(), + notData); + } + catch (RuntimeException e) { + failed = true; + } + + assertTrue("A refresh from data that cannot be used did not fail.", + failed); + assertEquals(expected, isMobile()); + assertEquals(published, String.valueOf( + engine.getDataFileMetaData().getDataPublishedDateTime())); + } + + /** + * Threads process while another refreshes the engine from its data + * file, and every process gives the values the engine gave before. + */ + @Test + public void Process_WhileRefreshing_GivesTheSameValues() + throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + AtomicBoolean refreshing = new AtomicBoolean(true); + CountDownLatch start = new CountDownLatch(1); + List> processors = new ArrayList<>(); + for (int t = 0; t < THREADS - 1; t++) { + processors.add(executor.submit(process(expected, refreshing, start))); + } + Future refresher = executor.submit(() -> { + start.await(); + try { + for (int i = 0; i < REFRESHES; i++) { + engine.refreshData(identifier); + } + } + finally { + refreshing.set(false); + } + return REFRESHES; + }); + start.countDown(); + assertEquals(Integer.valueOf(REFRESHES), + refresher.get(WAIT_SECONDS, TimeUnit.SECONDS)); + for (Future processor : processors) { + assertTrue("A thread processed nothing during the refreshes.", + processor.get(WAIT_SECONDS, TimeUnit.SECONDS) > 0); + } + } + + private Callable process( + List expected, + AtomicBoolean refreshing, + CountDownLatch start) { + return () -> { + start.await(); + int processed = 0; + do { + List values = isMobile(); + if (expected.equals(values) == false) { + fail("Processing during a refresh gave " + values + + " where " + expected + " was given before."); + } + processed++; + } while (refreshing.get()); + return processed; + }; + } + + /** + * Whether each of the User-Agents is a mobile device. + */ + private List isMobile() throws Exception { + List values = new ArrayList<>(); + for (String userAgent : USER_AGENTS) { + try (FlowData data = + getWrapper().getPipeline().createFlowData()) { + data.addEvidence("header.user-agent", userAgent); + data.process(); + values.add( + data.get(DeviceData.class).getIsMobile().getValue()); + } + } + return values; + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index 4f073192e..46177ac95 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -131,6 +131,23 @@ public void addProfile(TestData profile) { } } + /** + * A property keyed cloud engine that declares no Evidence of its own + * takes the base class's default, which accepts no Evidence. + */ + @Test + public void propertyKeyedDefault_AcceptsNoEvidence() { + EvidenceKeyFilter filter = new TestKeyedInstance().getEvidenceKeyFilter(); + for (String key : Arrays.asList( + "header.user-agent", + "query.client-ip", + "server.client-ip", + "cookie.51D_example")) { + assertFalse(filter.include(key), + "The default Evidence filter accepted '" + key + "'."); + } + } + private class TestKeyedInstance extends PropertyKeyedCloudEngineBase { public TestKeyedInstance() { diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java new file mode 100644 index 000000000..b8f7b6aba --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java @@ -0,0 +1,59 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Every Flow Element declares the Evidence it accepts, because the base + * Flow Element leaves getEvidenceKeyFilter abstract, so the compiler makes + * each concrete element give one. + */ +public class AcceptedEvidenceDeclaredTests { + + @Test + public void FlowElement_DeclaresTheEvidenceFilter() throws Exception { + Method method = FlowElement.class.getMethod("getEvidenceKeyFilter"); + assertEquals(EvidenceKeyFilter.class, method.getReturnType()); + assertTrue("The interface gives a default, so an element need not " + + "declare the Evidence it accepts.", + method.isDefault() == false); + } + + @Test + public void FlowElementBase_LeavesTheEvidenceFilterAbstract() + throws Exception { + Method method = FlowElementBase.class.getDeclaredMethod( + "getEvidenceKeyFilter"); + assertTrue("FlowElementBase gives a default, so an element need " + + "not declare the Evidence it accepts.", + Modifier.isAbstract(method.getModifiers())); + } +} diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java index 2ee0231c0..09372390c 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java @@ -129,7 +129,9 @@ enum AutoUpdateStatus { // Refreshing the engine with the new data caused an error to occur. AUTO_UPDATE_REFRESH_FAILED, // An operation could not be completed within the expected time. - AUTO_UPDATE_TIMEOUT + AUTO_UPDATE_TIMEOUT, + // The data file checking for updates has no temporary file path. + AUTO_UPDATE_TEMP_PATH_NOT_SET } class DataUpdateCompleteArgs { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index ba4631d33..9f27f37bd 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -260,11 +260,7 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { if (newDataAvailable == false && dataFile.getConfiguration().getDataUpdateUrl() != null && dataFile.getConfiguration().getDataUpdateUrl().isEmpty() == false) { - // A failed update is tried again only when the timer is on. - result = checkForUpdateFromUrl( - dataFile, - manualUpdate == false && - dataFile.getAutomaticUpdatesEnabled()); + result = checkForUpdateFromUrl(dataFile); newDataAvailable = result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS || result == AutoUpdateStatus.AUTO_UPDATE_SUCCESS; @@ -472,9 +468,10 @@ public AutoUpdateStatus updateFromMemory(AspectEngineDataFile dataFile, byte[] d fileWrapperFactory.build(dataFile.getDataFilePath()).getWriter() .writeBytes(data); } catch (Exception ex) { - logger.error("An error occurred when writing to " + - dataFile.getDataFilePath() + - ". Error detail: " + ex.getMessage(), ex); + logger.warn("An error occurred when writing to '" + + dataFile.getDataFilePath() + "'. The engine will be " + + "updated to use the new data but the file on disk " + + "will still contain old data.", ex); } } @@ -806,13 +803,11 @@ private static List licenseKeysToMask( * * @param dataFile the data file to use * @param tempFile the temp file to write the data to - * @param automatic true if the update will be tried again if it fails * @return the {@link AutoUpdateStatus} value indicating the result */ private AutoUpdateStatus downloadFile( AspectEngineDataFile dataFile, - FileWrapper tempFile, - boolean automatic) { + FileWrapper tempFile) { String url = dataFile.getFormattedUrl(); String engineType = getEngineType(dataFile); @@ -847,7 +842,7 @@ private AutoUpdateStatus downloadFile( AutoUpdateStatus downloaded; try { downloaded = downloadOkResponse( - connection, dataFile, tempFile, automatic); + connection, dataFile, tempFile, loggedUrl); } catch (Exception e) { // A download cut short is the access error, as // a failure to connect is. An HTTP library's @@ -914,10 +909,6 @@ private long getIfModifiedSince(AspectEngineDataFile dataFile) { return publishedMillis; } - private static String retryNote(boolean automatic) { - return automatic ? " Update will be attempted again later." : ""; - } - /** * The message of the Errors table's row for a failure to connect to the * data update service, or to download all of its response. The URL is @@ -977,13 +968,15 @@ private String getEngineType(AspectEngineDataFile dataFile) { *

* A failure reading the download is thrown, for the caller to report. * A failure writing {@code tempFile} is logged here as the - * specification's write error. + * specification's write error. With MD5 verification on, a response + * with no Content-MD5 header is warned about and applied, and one whose + * hash does not match is the Errors table's integrity row. */ private AutoUpdateStatus downloadOkResponse( HttpURLConnection connection, AspectEngineDataFile dataFile, FileWrapper tempFile, - boolean automatic) throws IOException { + String loggedUrl) throws IOException { logger.debug("HTTP Status is OK"); boolean verifyMd5 = dataFile.getConfiguration().getVerifyMd5(); boolean decompress = dataFile.getConfiguration().getDecompressContent(); @@ -1017,13 +1010,21 @@ private AutoUpdateStatus downloadOkResponse( return AutoUpdateStatus.AUTO_UPDATE_ERR_READING_STREAM; } - if (verifyMd5 && !md5Matches(md, connection)) { - logger.warn("An error occurred during the integrity check of new " + - "data file for " + getEngineType(dataFile) + "." + - retryNote(automatic) + - " Error detail: the MD5 hash of the download does not " + - "match the Content-MD5 header"); - return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; + if (verifyMd5) { + String header = connection.getHeaderField("Content-MD5"); + String computed = DatatypeConverter.printHexBinary(md.digest()) + .toLowerCase(Locale.ROOT); + if (header == null) { + logger.warn("No MD5 hash included in data update response " + + "for '{}'. Unable to verify data integrity", loggedUrl); + } else if (header.equalsIgnoreCase(computed) == false) { + logCheckFailure("Integrity check failed. MD5 hash in HTTP " + + "response '" + header + "' for '" + + getEngineType(dataFile) + "'data update does not " + + "match calculated hash for the downloaded file '" + + computed + "'.", null); + return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; + } } return AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; } @@ -1108,12 +1109,6 @@ private void writeBodyTo(FileWrapper tempFile, InputStream body) throws IOExcept } } - private static boolean md5Matches(MessageDigest md, HttpURLConnection connection) { - String header = connection.getHeaderField("Content-MD5"); - String computed = DatatypeConverter.printHexBinary(md.digest()); - return header != null && header.equalsIgnoreCase(computed); - } - @Override public void close() { if (closeFutureFactory) { @@ -1126,8 +1121,7 @@ public void close() { } private AutoUpdateStatus checkForUpdateFromUrl( - AspectEngineDataFile dataFile, - boolean automatic) { + AspectEngineDataFile dataFile) { if (dataFile.getDataFilePath() == null || dataFile.getDataFilePath().isEmpty()) { // There is no data file path specified so perform the // update entirely in memory. @@ -1135,7 +1129,7 @@ private AutoUpdateStatus checkForUpdateFromUrl( FileWrapper uncompressedData = new FileWrapperMemory(); AutoUpdateStatus memoryDownloadResult = - downloadFile(dataFile, uncompressedData, automatic); + downloadFile(dataFile, uncompressedData); if (memoryDownloadResult != AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { return memoryDownloadResult; } @@ -1163,10 +1157,10 @@ private AutoUpdateStatus checkForUpdateFromUrl( if (dataFile.getTempDataDirPath() == null || dataFile.getTempDataDirPath().isEmpty()) { - logger.error("The data file '" + dataFile.getIdentifier() + + logCheckFailure("The data file '" + dataFile.getIdentifier() + "' is checking for updates but does not have a temporary " + - "file path configured."); - return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; + "file path configured.", null); + return AutoUpdateStatus.AUTO_UPDATE_TEMP_PATH_NOT_SET; } Path uncompressedTempFile = Paths.get( dataFile.getTempDataDirPath(), @@ -1175,7 +1169,7 @@ private AutoUpdateStatus checkForUpdateFromUrl( try { // Check if there is an update and download it if there is AutoUpdateStatus result = - downloadFile(dataFile, uncompressedData, automatic); + downloadFile(dataFile, uncompressedData); if (result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { WatchKey watchKey = dataFile.getConfiguration().getWatchKey(); diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java index c93f6369c..be0ce9ca1 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java @@ -43,8 +43,10 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.math.BigInteger; import java.net.HttpURLConnection; import java.net.URL; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -59,6 +61,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -80,6 +83,8 @@ public class DataUpdateServiceLoggingTests { FULL_KEY + "&Download=True&Type=HashV41"; private static final String MASKED_URL = UPDATE_URL + "?LicenseKeys=" + MASKED_KEY + "&Download=True&Type=HashV41"; + // The body of a download the tests apply. + private static final byte[] DOWNLOAD = {1, 2, 3}; private ch.qos.logback.classic.Logger logger; private ListAppender appender; @@ -418,33 +423,102 @@ public void TemporaryFileCannotBeWritten_LogsTheWriteError() } @Test - public void Md5DoesNotMatch_WarnsAboutIntegrityCheck() throws Exception { + public void Md5DoesNotMatch_LogsTheTableRow() throws Exception { when(connection.getResponseCode()) .thenReturn(HttpURLConnection.HTTP_OK); when(connection.getInputStream()) - .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + .thenReturn(new ByteArrayInputStream(DOWNLOAD)); when(connection.getHeaderField("Content-MD5")).thenReturn("0"); AspectEngineDataFileDefault file = newDataFile(true, false, false); file.getConfiguration().setVerifyMd5(true); - dataUpdate.checkForUpdate(engine); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred during the integrity " + - "check of new data file for " + engineType + ". Error detail: " + - "the MD5 hash of the download does not match the Content-MD5 " + - "header"); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED, + status); + assertCheckFailed("Integrity check failed. MD5 hash in HTTP " + + "response '0' for '" + engineType + "'data update does not " + + "match calculated hash for the downloaded file '" + + md5Hex(DOWNLOAD) + "'.", null); + verify(engine, never()).refreshData(anyString(), any(byte[].class)); } + /** + * A response with no Content-MD5 header is warned about, and the check + * goes on to apply the download. + */ @Test - public void NoTemporaryFilePath_LogsError() { + public void Md5Missing_WarnsAndAppliesTheUpdate() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(DOWNLOAD)); + AspectEngineDataFileDefault file = newDataFile(true, false, false); + file.getConfiguration().setVerifyMd5(true); + + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "No MD5 hash included in data update " + + "response for '" + UPDATE_URL + "'. Unable to verify data " + + "integrity"); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS, + status); + verify(engine).refreshData(anyString(), any(byte[].class)); + } + + @Test + public void NoTemporaryFilePath_LogsTheTableRow() { AspectEngineDataFileDefault file = newDataFile(false, false, false); file.setTempDataDirPath(null); when(engine.getTempDataDirPath()).thenReturn(null); - dataUpdate.checkForUpdate(engine); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); - assertLogged(Level.ERROR, "The data file 'default' is checking for " + - "updates but does not have a temporary file path configured."); + assertEquals("AUTO_UPDATE_TEMP_PATH_NOT_SET", status.name()); + assertCheckFailed("The data file 'default' is checking for " + + "updates but does not have a temporary file path configured.", + null); + } + + /** + * Data passed in from memory that cannot be written to the data file is + * warned about, and the engine is still refreshed with it. + */ + @Test + public void UpdateFromMemory_FileCannotBeWritten_WarnsAndRefreshes() + throws Exception { + AspectEngineDataFileDefault file = newDataFile(false, false, false); + String unwritable = new File(new File( + System.getProperty("java.io.tmpdir"), + "no-such-directory-" + System.nanoTime()), "data.hash") + .getAbsolutePath(); + file.getConfiguration().setDataFilePath(unwritable); + + DataUpdateService.AutoUpdateStatus status = + dataUpdate.updateFromMemory(file, DOWNLOAD); + + assertLogged(Level.WARN, "An error occurred when writing to '" + + unwritable + "'. The engine will be updated to use the new data " + + "but the file on disk will still contain old data."); + assertTrue("An error was logged: " + logged(Level.ERROR), + logged(Level.ERROR).isEmpty()); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS, + status); + verify(engine).refreshData(anyString(), any(byte[].class)); + } + + /** + * The MD5 hash of the bytes as lower case hexadecimal. + */ + private static String md5Hex(byte[] bytes) throws Exception { + byte[] digest = MessageDigest.getInstance("MD5").digest(bytes); + return String.format("%032x", new BigInteger(1, digest)); } /** diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 6f139fca2..50ec8f88f 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -762,9 +762,9 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { } }); - // The invalid MD5 will cause a warning to be logged so we want - // to ignore this - ignoreWranings = 1; + // The invalid MD5 is the Errors table integrity row, logged once + // at Error, so it is ignored in cleanup. + ignoreErrors = 1; OnPremiseAspectEngine engine = mock(OnPremiseAspectEngine.class); DataFileConfigurationDefault config = new DataFileConfigurationDefault();