feat: never-evicting per-entry store behind the API cache

pom.xml

@@ -296,6 +296,11 @@
         <version>3.5.5</version>
         <configuration>
           <argLine>@{argLine} -javaagent:${org.mockito:mockito-core:jar} -Djava.awt.headless=true</argLine>
+          <environmentVariables>
+            <!-- Tests that start a WicketApplication must not read or write the developer's
+                 real API cache persistence (snapshot file and entry store) in ~/.nanopub. -->
+            <NANODASH_API_CACHE_FILE>none</NANODASH_API_CACHE_FILE>
+          </environmentVariables>
         </configuration>
       </plugin>
       <plugin>

src/main/java/com/knowledgepixels/nanodash/ApiCache.java

@@ -98,6 +98,44 @@ private static void cleanupMetadata(String cacheId) {
         forcedRefresh.remove(cacheId);
     }
 
+    /**
+     * Fills a memory miss from the per-entry store (see
+     * {@link ApiCachePersistence#loadEntry}): the stored response goes back into the
+     * in-memory cache with its <em>original</em> refresh timestamp, so the normal staleness
+     * logic takes over from there — the restored content is served while anything older than
+     * {@link #REFRESH_AGE_THRESHOLD_MS} re-fetches in the background. This is what makes
+     * memory eviction invisible to callers: the persistent tier never evicts, so content
+     * that once arrived stays available (however outdated) until a re-fetch replaces it.
+     * A timestamp from the future (a clock jump) is not adopted, leaving the entry to count
+     * as stale rather than as fresh indefinitely.
+     *
+     * @param cacheId the cache id (the query's URL string)
+     * @return the restored response, or null if the store has none
+     */
+    private static ApiResponse loadResponseFromStore(String cacheId) {
+        ApiCachePersistence.PersistedEntry entry = ApiCachePersistence.loadEntry(cacheId);
+        if (entry == null || !(entry.value instanceof ApiResponse response)) return null;
+        cachedResponses.put(cacheId, response);
+        if (entry.lastRefresh <= System.currentTimeMillis()) {
+            lastRefresh.putIfAbsent(cacheId, entry.lastRefresh);
+        }
+        return response;
+    }
+
+    /**
+     * The map counterpart of {@link #loadResponseFromStore(String)}.
+     */
+    @SuppressWarnings("unchecked")
+    private static Map<String, String> loadMapFromStore(String cacheId) {
+        ApiCachePersistence.PersistedEntry entry = ApiCachePersistence.loadEntry(cacheId);
+        if (entry == null || !(entry.value instanceof Map<?, ?> map)) return null;
+        cachedMaps.put(cacheId, (Map<String, String>) map);
+        if (entry.lastRefresh <= System.currentTimeMillis()) {
+            lastRefresh.putIfAbsent(cacheId, entry.lastRefresh);
+        }
+        return (Map<String, String>) map;
+    }
+
     /**
      * Checks if a cache refresh is currently running for the given cache ID.
      *
@@ -168,14 +206,19 @@ private static void updateResponse(QueryRef queryRef, boolean forced) throws Fai
         }
         String cacheId = queryRef.getAsUrlString();
         logger.info("Updating cached API response for {}", cacheId);
+        long timeNow = System.currentTimeMillis();
         cachedResponses.put(cacheId, response);
-        lastRefresh.put(cacheId, System.currentTimeMillis());
+        lastRefresh.put(cacheId, timeNow);
+        ApiCachePersistence.storeEntry(cacheId, response, timeNow);
     }
 
     public static ApiResponse retrieveResponseSync(QueryRef queryRef, boolean forced) {
         long timeNow = System.currentTimeMillis();
         String cacheId = queryRef.getAsUrlString();
         logger.debug("Retrieving cached API response synchronously for {}", cacheId);
+        if (cachedResponses.getIfPresent(cacheId) == null) {
+            loadResponseFromStore(cacheId);
+        }
         boolean needsRefresh = true;
         if (cachedResponses.getIfPresent(cacheId) != null) {
             // lastRefresh can be missing for a cached entry (racing invalidation or
@@ -297,6 +340,9 @@ public static ApiResponse retrieveResponseAsync(QueryRef queryRef) {
             forcedRefresh.add(cacheId);
         }
         boolean forced = forcedRefresh.contains(cacheId);
+        if (cachedResponses.getIfPresent(cacheId) == null) {
+            loadResponseFromStore(cacheId);
+        }
         boolean isCached = false;
         boolean needsRefresh = true;
         if (cachedResponses.getIfPresent(cacheId) != null) {
@@ -366,8 +412,10 @@ private static void updateMap(QueryRef queryRef) throws FailedApiCallException,
             map.put(resultEntry.get("key"), resultEntry.get("value"));
         }
         String cacheId = queryRef.getAsUrlString();
+        long timeNow = System.currentTimeMillis();
         cachedMaps.put(cacheId, map);
-        lastRefresh.put(cacheId, System.currentTimeMillis());
+        lastRefresh.put(cacheId, timeNow);
+        ApiCachePersistence.storeEntry(cacheId, (Serializable) map, timeNow);
     }
 
     /**
@@ -384,6 +432,9 @@ public static Map<String, String> retrieveMap(QueryRef queryRef) {
             cachedMaps.invalidate(cacheId);
             lastRefresh.remove(cacheId);
         }
+        if (cachedMaps.getIfPresent(cacheId) == null) {
+            loadMapFromStore(cacheId);
+        }
         boolean isCached = false;
         boolean needsRefresh = true;
         if (cachedMaps.getIfPresent(cacheId) != null) {
@@ -519,7 +570,9 @@ public static Model retrieveRdfModelAsync(QueryRef queryRef) {
 
     /**
      * Returns whatever response is cached for a query reference, however outdated, without
-     * triggering a refresh or any other side effect. Meant for showing the previous content
+     * triggering a refresh. A memory miss falls through to the per-entry store, which never
+     * evicts, so this finds any response that ever arrived for the query — restored quickly
+     * from a local file, never the network. Meant for showing the previous content
      * while a refresh is in flight (issue #599) — never as a substitute for the current data,
      * which is what {@link #retrieveResponseAsync(QueryRef)} and
      * {@link #retrieveResponseSync(QueryRef, boolean)} return.
@@ -528,7 +581,10 @@ public static Model retrieveRdfModelAsync(QueryRef queryRef) {
      * @return The cached response of any age, or null if nothing is cached.
      */
     public static ApiResponse retrieveStaleResponse(QueryRef queryRef) {
-        return cachedResponses.getIfPresent(queryRef.getAsUrlString());
+        String cacheId = queryRef.getAsUrlString();
+        ApiResponse response = cachedResponses.getIfPresent(cacheId);
+        if (response != null) return response;
+        return loadResponseFromStore(cacheId);
     }
 
     /**
@@ -623,6 +679,28 @@ static int importSnapshot(Snapshot snapshot, long maxAgeMs) {
         return count;
     }
 
+    /**
+     * Copies a restored snapshot's entries into the per-entry store, so content saved by a
+     * version from before the store existed is not lost to memory eviction again. Entries
+     * the store already has are left alone (its version is at least as new), and no age
+     * limit applies — unlike the in-memory import, the store keeps everything. Meant to run
+     * once at startup, right after the snapshot file is read.
+     *
+     * @param snapshot the restored snapshot
+     */
+    static void backfillEntryStore(Snapshot snapshot) {
+        for (Map.Entry<String, ApiResponse> e : snapshot.responses.entrySet()) {
+            Long t = snapshot.refreshTimes.get(e.getKey());
+            if (t == null || e.getValue() == null) continue;
+            ApiCachePersistence.storeEntryIfAbsent(e.getKey(), e.getValue(), t);
+        }
+        for (Map.Entry<String, Map<String, String>> e : snapshot.maps.entrySet()) {
+            Long t = snapshot.refreshTimes.get(e.getKey());
+            if (t == null || e.getValue() == null) continue;
+            ApiCachePersistence.storeEntryIfAbsent(e.getKey(), (Serializable) e.getValue(), t);
+        }
+    }
+
     /**
      * Marks the cached response for a specific query reference as outdated and sets a delay
      * before the next refresh can occur. The previous response is kept and remains available

src/main/java/com/knowledgepixels/nanodash/ApiCachePersistence.java

@@ -12,9 +12,12 @@
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -28,7 +31,17 @@
  * restored ones are served — instead of starting cold and re-fetching everything from the
  * query API and the registry at once.
  *
- * <p>The snapshot is a cache, not a store of record: a missing, corrupt, or (after a library
+ * <p>Next to the snapshot file sits a per-entry store (a directory of one small file per
+ * query response or map), from which the persistent tier never evicts: every successful
+ * fetch overwrites its entry, and nothing is ever aged out. The in-memory caches keep their
+ * bounded size and idle expiry; when a request misses there, {@link ApiCache} reads the
+ * entry back from this store with its original timestamp, so the stored content is shown
+ * right away while the usual staleness logic re-queries in the background. Without this
+ * tier, memory eviction used to propagate into the snapshot (which only captured what was
+ * still in memory), so rarely visited pages came back from a restart with a blank loading
+ * state instead of their previous content.</p>
+ *
+ * <p>Both tiers are caches, not stores of record: a missing, corrupt, or (after a library
  * upgrade) unreadable file only means starting cold, never failing startup. The default file
  * location is inside {@code ~/.nanopub}, which the standard Docker setup already mounts as a
  * volume, so deployments get persistence without any configuration.</p>
@@ -54,6 +67,10 @@ private ApiCachePersistence() {
     private static ScheduledExecutorService scheduler;
     private static File snapshotFile;
 
+    // Directory of the per-entry store; null while persistence is not (yet) initialized,
+    // which makes all entry-store operations no-ops.
+    private static volatile File entryStoreDir;
+
     /**
      * The root object written to the snapshot file: the query cache content together with
      * the cached nanopubs. The nanopubs matter as much as the query responses for a warm
@@ -94,6 +111,7 @@ public static synchronized void init() {
             return;
         }
         snapshotFile = new File(path);
+        entryStoreDir = new File(path + ".d");
         load(snapshotFile);
         scheduler = Executors.newSingleThreadScheduledExecutor((r) -> {
             Thread t = new Thread(r, "nanodash-cache-persistence");
@@ -134,10 +152,12 @@ static void load(File file) {
                 int viewCount = state.views == null ? 0 : View.importViews(state.views);
                 logger.info("Restored {} cached query results, {} cached nanopubs, {} views and {} view resolutions from {}",
                         queryCount, npCount, viewCount, resolvedCount, file);
+                ApiCache.backfillEntryStore(state.queryCache);
             } else if (obj instanceof ApiCache.Snapshot snapshot) {
                 // A file from before the snapshot also carried the nanopub cache.
                 int count = ApiCache.importSnapshot(snapshot, MAX_SNAPSHOT_AGE_MS);
                 logger.info("Restored {} cached query results from {}", count, file);
+                ApiCache.backfillEntryStore(snapshot);
             } else {
                 logger.warn("Ignoring cache snapshot file {} with unexpected content", file);
             }
@@ -178,4 +198,125 @@ static void save(File file) {
         }
     }
 
+    /**
+     * One entry of the per-entry store: a query response or map together with the cache id
+     * it belongs to and when it was last refreshed. The cache id is stored inside the file
+     * (whose name is only a hash of it) so a read can verify it got the entry it asked for.
+     */
+    static class PersistedEntry implements Serializable {
+
+        private static final long serialVersionUID = 1L;
+
+        final String cacheId;
+        final long lastRefresh;
+        final Serializable value;
+
+        private PersistedEntry(String cacheId, Serializable value, long lastRefresh) {
+            this.cacheId = cacheId;
+            this.value = value;
+            this.lastRefresh = lastRefresh;
+        }
+
+    }
+
+    /**
+     * Points the per-entry store at the given directory (null disables it). Normally set by
+     * {@link #init()}; exposed for tests.
+     *
+     * @param dir the store directory, or null to disable the store
+     */
+    static void initEntryStore(File dir) {
+        entryStoreDir = dir;
+    }
+
+    /**
+     * Writes one cache entry to the per-entry store, replacing any previous version of it.
+     * Called for every successfully fetched query response or map, so the store always holds
+     * the latest content that ever arrived for each query — this tier never evicts. Written
+     * atomically (unique temporary file, then move), so concurrent writers and a crash
+     * mid-write can at worst leave the previous version in place. A no-op while persistence
+     * is disabled; any failure is logged and otherwise ignored.
+     *
+     * @param cacheId         the cache id (the query's URL string)
+     * @param value           the response or map to store
+     * @param lastRefreshTime when the content was fetched
+     */
+    static void storeEntry(String cacheId, Serializable value, long lastRefreshTime) {
+        File dir = entryStoreDir;
+        if (dir == null) return;
+        try {
+            dir.mkdirs();
+            File file = new File(dir, entryFileName(cacheId));
+            File tmpFile = new File(dir, file.getName() + ".tmp." + Thread.currentThread().threadId() + "." + System.nanoTime());
+            try (ObjectOutputStream out = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(tmpFile)))) {
+                out.writeObject(new PersistedEntry(cacheId, value, lastRefreshTime));
+            }
+            try {
+                Files.move(tmpFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+            } catch (AtomicMoveNotSupportedException ex) {
+                Files.move(tmpFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING);
+            }
+        } catch (Exception ex) {
+            logger.warn("Could not write cache entry for {}: {}", cacheId, ex.toString());
+        }
+    }
+
+    /**
+     * Writes one cache entry to the per-entry store only if the store has none for that id
+     * yet. Used to backfill the store from a snapshot file at startup without overwriting
+     * entries the store may hold in a newer version.
+     *
+     * @param cacheId         the cache id (the query's URL string)
+     * @param value           the response or map to store
+     * @param lastRefreshTime when the content was fetched
+     */
+    static void storeEntryIfAbsent(String cacheId, Serializable value, long lastRefreshTime) {
+        File dir = entryStoreDir;
+        if (dir == null) return;
+        if (new File(dir, entryFileName(cacheId)).isFile()) return;
+        storeEntry(cacheId, value, lastRefreshTime);
+    }
+
+    /**
+     * Reads one cache entry back from the per-entry store, however old it may be — age is
+     * the caller's concern, this tier never evicts. An unreadable file (corrupt, written by
+     * an incompatible earlier version, or a hash collision) is deleted, since it can never
+     * be read again and its slot is rewritten on the next successful fetch anyway.
+     *
+     * @param cacheId the cache id (the query's URL string)
+     * @return the stored entry, or null if the store has none (or persistence is disabled)
+     */
+    static PersistedEntry loadEntry(String cacheId) {
+        File dir = entryStoreDir;
+        if (dir == null) return null;
+        File file = new File(dir, entryFileName(cacheId));
+        if (!file.isFile()) return null;
+        try (ObjectInputStream in = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)))) {
+            if (in.readObject() instanceof PersistedEntry entry && cacheId.equals(entry.cacheId)) {
+                return entry;
+            }
+        } catch (Exception ex) {
+            logger.warn("Could not read cache entry file {}: {}", file, ex.toString());
+        }
+        file.delete();
+        return null;
+    }
+
+    /**
+     * The store file name for a cache id: a hash, since cache ids are URL strings of
+     * arbitrary length and content.
+     */
+    private static String entryFileName(String cacheId) {
+        try {
+            byte[] hash = MessageDigest.getInstance("SHA-256").digest(cacheId.getBytes(StandardCharsets.UTF_8));
+            StringBuilder sb = new StringBuilder(hash.length * 2);
+            for (byte b : hash) {
+                sb.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16));
+            }
+            return sb.append(".ser").toString();
+        } catch (NoSuchAlgorithmException ex) {
+            throw new RuntimeException(ex); // SHA-256 is guaranteed to exist
+        }
+    }
+
 }