Issue #153 treat upstream fetch failure as drift; Issue #154 refresh sync tooling

Squash of PR #155. Closes #153 and #154. Tracker blindspot fix (UPSTREAM_ERROR now counts as drift, fingerprint + Missing Upstream section) and sync tooling retargeted to the jdk.incubator.json module. 1679 tests green; ci.yml exp_tests=1679. No release, no version bump.

Author: simbo1905

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
                       for k in totals: totals[k]+=int(r.get(k,'0'))
                   except Exception:
                       pass
-          exp_tests=1676
+          exp_tests=1679
           exp_skipped=0
           if totals['tests']!=exp_tests or totals['skipped']!=exp_skipped:
               print(f"Unexpected test totals: {totals} != expected tests={exp_tests}, skipped={exp_skipped}")

json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTracker.java

@@ -947,7 +947,7 @@ static String fetchUpstreamSource(String className) {
     /// @param report the full comparison report
     /// @return 7-character fingerprint or "0000000" if no differences
     static String generateFingerprint(JsonObject report) {
-        if (getDifferentApiCount(report) == 0) {
+        if (!hasDifferences(report)) {
             return "0000000";
         }
 
@@ -959,6 +959,13 @@ static String generateFingerprint(JsonObject report) {
             final var diffObj = (JsonObject) diff;
             final var status = ((JsonString) diffObj.asMap().get("status")).asString();
 
+            if ("UPSTREAM_ERROR".equals(status)) {
+                // Fetch failures are drift too: include them so the issue
+                // dedup fingerprint is distinct per affected class set.
+                stableLines.add(((JsonString) diffObj.asMap().get("className")).asString() + ":UPSTREAM_ERROR");
+                continue;
+            }
+            
             if (!"DIFFERENT".equals(status)) continue;
 
             final var className = ((JsonString) diffObj.asMap().get("className")).asString();
@@ -1009,6 +1016,21 @@ private static long getDifferentApiCount(JsonObject report) {
         return 0;
     }
 
+    /// Extracts the missingUpstream count from a report summary
+    /// @param report the comparison report
+    /// @return the count of classes whose upstream fetch or parse failed
+    private static long getMissingUpstreamCount(JsonObject report) {
+        final var summary = (JsonObject) report.asMap().get("summary");
+        if (summary == null) {
+            return 0;
+        }
+        final var missingUpstreamValue = summary.asMap().get("missingUpstream");
+        if (missingUpstreamValue instanceof JsonNumber num) {
+            return num.asLong();
+        }
+        return 0;
+    }
+
     /// Generates a terse human-readable summary of the API differences
     /// Suitable for GitHub issue body
     /// @param report the full comparison report
@@ -1068,6 +1090,23 @@ static String generateSummary(JsonObject report) {
             }
         }
 
+        if (missingUpstream > 0) {
+            sb.append("## Missing Upstream\n\n");
+            sb.append("These classes could not be fetched or parsed upstream. A fetch failure is drift: the tracker cannot confirm we are current.\n\n");
+            for (final var diff : differences.asList()) {
+                final var diffObj = (JsonObject) diff;
+                final var status = ((JsonString) diffObj.asMap().get("status")).asString();
+                
+                if (!"UPSTREAM_ERROR".equals(status)) continue;
+                
+                final var className = ((JsonString) diffObj.asMap().get("className")).asString();
+                final var errorValue = diffObj.asMap().get("error");
+                final var error = errorValue instanceof JsonString js ? js.asString() : "unknown error";
+                sb.append("- ⚠️ **").append(className).append("**: `").append(error).append("`\n");
+            }
+            sb.append("\n");
+        }
+        
         sb.append("---\n");
         final var timestamp = ((JsonString) report.asMap().get("timestamp")).asString();
         sb.append("*Generated by API Tracker on ").append(timestamp.split("T")[0]).append("*\n");
@@ -1077,8 +1116,8 @@ static String generateSummary(JsonObject report) {
 
     /// Checks if there are any API differences in the report
     /// @param report the comparison report
-    /// @return true if differentApi > 0
+    /// @return true if differentApi > 0 or missingUpstream > 0
     static boolean hasDifferences(JsonObject report) {
-        return getDifferentApiCount(report) > 0;
+        return getDifferentApiCount(report) > 0 || getMissingUpstreamCount(report) > 0;
     }
 }

json-java21-api-tracker/src/test/java/io/github/simbo1905/tracker/ApiTrackerTest.java

@@ -9,8 +9,10 @@
 
 import jdk.incubator.java.util.json.JsonBoolean;
 import jdk.incubator.java.util.json.JsonArray;
+import jdk.incubator.java.util.json.JsonNumber;
 import jdk.incubator.java.util.json.JsonObject;
 import jdk.incubator.java.util.json.JsonString;
+import jdk.incubator.java.util.json.Json;
 
 import java.util.Set;
 import java.util.Map;
@@ -234,4 +236,108 @@ void testNormalizeTypeName() {
                 .isEqualTo("String");
         }
     }
+
+    @Nested
+    @DisplayName("Drift Detection Gates")
+    class DriftDetectionGateTests {
+
+        private static JsonObject upstreamErrorDiff(String className, String error) {
+            return JsonObject.of(Map.of(
+                "className", JsonString.of(className),
+                "status", JsonString.of("UPSTREAM_ERROR"),
+                "error", JsonString.of(error)
+            ));
+        }
+
+        private static JsonObject report(long differentApi, long missingUpstream, JsonArray differences) {
+            return JsonObject.of(Map.of(
+                "timestamp", JsonString.of("2026-08-30T00:00:00Z"),
+                "summary", JsonObject.of(Map.of(
+                    "totalClasses", JsonNumber.of(differentApi + missingUpstream),
+                    "matchingClasses", JsonNumber.of(0),
+                    "differentApi", JsonNumber.of(differentApi),
+                    "missingUpstream", JsonNumber.of(missingUpstream)
+                )),
+                "differences", differences
+            ));
+        }
+
+        @Test
+        @DisplayName("Upstream fetch failures must count as drift, not all-clear")
+        void testHasDifferencesTreatsUpstreamErrorAsDrift() {
+            LoggingControl.setupCleanLogging();
+            java.util.logging.Logger.getLogger(getClass().getName())
+                .info(() -> "TEST: testHasDifferencesTreatsUpstreamErrorAsDrift");
+            final var allErrors = report(0, 2, JsonArray.of(java.util.List.of(
+                upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found"),
+                upstreamErrorDiff("JsonObject", "HTTP_ERROR: Status 500")
+            )));
+
+            assertThat(ApiTracker.hasDifferences(allErrors))
+                .as("a fully-blind detector run (every class UPSTREAM_ERROR) must report drift")
+                .isTrue();
+
+            final var mixed = report(1, 1, JsonArray.of(java.util.List.of(
+                upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found"),
+                JsonObject.of(Map.of(
+                    "className", JsonString.of("JsonValue"),
+                    "status", JsonString.of("DIFFERENT"),
+                    "differences", JsonArray.of(java.util.List.of())
+                ))
+            )));
+            assertThat(ApiTracker.hasDifferences(mixed)).isTrue();
+        }
+
+        @Test
+        @DisplayName("Fingerprint covers UPSTREAM_ERROR classes and is stable and distinct")
+        void testFingerprintCoversUpstreamErrors() {
+            LoggingControl.setupCleanLogging();
+            java.util.logging.Logger.getLogger(getClass().getName())
+                .info(() -> "TEST: testFingerprintCoversUpstreamErrors");
+            final var none = report(0, 0, JsonArray.of(java.util.List.of()));
+            assertThat(ApiTracker.generateFingerprint(none)).isEqualTo("0000000");
+
+            final var errorsA = report(0, 2, JsonArray.of(java.util.List.of(
+                upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found"),
+                upstreamErrorDiff("JsonObject", "NOT_FOUND: Upstream file not found")
+            )));
+            final var errorsAgain = report(0, 2, JsonArray.of(java.util.List.of(
+                upstreamErrorDiff("JsonObject", "NOT_FOUND: Upstream file not found"),
+                upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found")
+            )));
+            final var errorsB = report(0, 1, JsonArray.of(java.util.List.of(
+                upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found")
+            )));
+
+            final var fpA = ApiTracker.generateFingerprint(errorsA);
+            assertThat(fpA)
+                .as("fetch-failure drift must not hash to the no-differences sentinel")
+                .isNotEqualTo("0000000");
+            assertThat(ApiTracker.generateFingerprint(errorsAgain))
+                .as("same error set in different order must fingerprint identically")
+                .isEqualTo(fpA);
+            assertThat(ApiTracker.generateFingerprint(errorsB))
+                .as("different error sets must fingerprint differently")
+                .isNotEqualTo(fpA);
+        }
+
+        @Test
+        @DisplayName("Summary renders a Missing Upstream section for fetch failures")
+        void testSummaryRendersMissingUpstreamSection() {
+            LoggingControl.setupCleanLogging();
+            java.util.logging.Logger.getLogger(getClass().getName())
+                .info(() -> "TEST: testSummaryRendersMissingUpstreamSection");
+            final var allErrors = report(0, 2, JsonArray.of(java.util.List.of(
+                upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found (possibly deleted or renamed)"),
+                upstreamErrorDiff("JsonObject", "HTTP_ERROR: Status 500")
+            )));
+
+            final var summary = ApiTracker.generateSummary(allErrors);
+
+            assertThat(summary).contains("Missing Upstream");
+            assertThat(summary).contains("JsonNumber");
+            assertThat(summary).contains("JsonObject");
+            assertThat(summary).contains("NOT_FOUND: Upstream file not found (possibly deleted or renamed)");
+        }
+    }
 }

updates/2025-09-04/RefreshFromUpstream.java

@@ -1,5 +1,6 @@
 // Compact single-file Java 25 script to refresh and compare impl sources
 // Run: java RefreshFromUpstream.java
+// Refreshed 2026-08-30 for the upstream jdk.incubator.json module layout (issue #154).
 
 import java.io.*;
 import java.net.URI;
@@ -37,19 +38,20 @@ void main() throws Exception {
 
     // Work dirs (not checked in)
     Path updatesDir = repoRoot.resolve("updates/2025-09-04");
-    Path upstreamDir = updatesDir.resolve("upstream/jdk.internal.util.json");
+    Path upstreamDir = updatesDir.resolve("upstream/jdk.incubator.json.impl");
     Path reportDir = updatesDir.resolve("reports");
     Files.createDirectories(upstreamDir);
     Files.createDirectories(reportDir);
 
-    // Upstream raw base for impl package
-    String upstreamBase = "https://raw.githubusercontent.com/openjdk/jdk-sandbox/refs/heads/json/src/java.base/share/classes/jdk/internal/util/json/";
+    // Upstream raw base for the incubator module impl package
+    String upstreamBase = "https://raw.githubusercontent.com/openjdk/jdk-sandbox/refs/heads/json/src/jdk.incubator.json/share/classes/jdk/incubator/json/impl/";
 
-    // Discover local impl files
+    // Discover local impl files (LazyConstant.java is a local-only polyfill with no upstream counterpart)
     List<Path> localFiles;
     try (var stream = Files.list(localImplDir)) {
         localFiles = stream
             .filter(p -> p.getFileName().toString().endsWith(".java"))
+            .filter(p -> !p.getFileName().toString().equals("LazyConstant.java"))
             .sorted()
             .toList();
     }

updates/2025-09-04/transform_upstream.py

@@ -1,6 +1,9 @@
 import os, sys, re, shutil
 
-SRC = 'updates/2025-09-04/upstream/jdk.internal.util.json'
+# Refreshed 2026-08-30 for the upstream jdk.incubator.json module layout (issue #154).
+# Scope: impl files fetched into the snapshot dir (see RefreshFromUpstream.java).
+# Public API files follow the manual process in json-java21/AGENTS.md.
+SRC = 'updates/2025-09-04/upstream/jdk.incubator.json.impl'
 DST = 'json-java21/src/main/java/jdk/incubator/internal/util/json'
 
 def read(path):
@@ -27,10 +30,11 @@ def write_safe(path, text):
     return True
 
 def transform(text, name):
-    # package
-    text = re.sub(r'^package\s+jdk\.internal\.util\.json;', 'package jdk.incubator.internal.util.json;', text, flags=re.M)
-    # imports for public API
-    text = re.sub(r'^(\s*import\s+)java\.util\.json\.', r'\1jdk.incubator.java.util.json.', text, flags=re.M)
+    # package: upstream impl package -> our internal package
+    text = re.sub(r'^package\s+jdk\.incubator\.json\.impl;', 'package jdk.incubator.internal.util.json;', text, flags=re.M)
+    # imports: impl-internal first (defensive; upstream impl rarely imports itself), then public API
+    text = re.sub(r'^(\s*import\s+)jdk\.incubator\.json\.impl\.', r'\1jdk.incubator.internal.util.json.', text, flags=re.M)
+    text = re.sub(r'^(\s*import\s+)jdk\.incubator\.json\.', r'\1jdk.incubator.java.util.json.', text, flags=re.M)
     # annotations (single-line)
     text = re.sub(r'^\s*@(?:jdk\.internal\..*|ValueBased|StableValue).*\n', '', text, flags=re.M)
     # remove import of ValueBased if present
@@ -57,9 +61,6 @@ def main():
     for name in os.listdir(SRC):
         if not name.endswith('.java'):
             continue
-        if name in ('StableValue.java', 'Utils.java'):
-            # Keep local backport helper and existing Utils for now
-            continue
         src_path = os.path.join(SRC, name)
         dst_path = os.path.join(DST, name)
         data = read(src_path)
@@ -69,6 +70,7 @@ def main():
     if not ok:
         sys.exit(2)
     print('Transform complete')
+    print('Reminder: after transforming, re-append the Utils.powExact polyfill (Java 21 lacks Math.powExact) and keep LazyConstant.java untouched (local polyfill).')
 
 if __name__ == '__main__':
     main()