fix(#9): native builds stage the jars asked for on every build

scripts/build-native.sh

@@ -28,9 +28,11 @@ PACKAGE_NAME="${2:-arcadedb-embedded}"
 PACKAGE_DESCRIPTION="${3:-ArcadeDB embedded multi-model database with bundled JRE}"
 ARCADEDB_TAG="${4:-}"
 BUILD_VERSION="${5:-}"
+# The jars to bundle; empty pulls them from arcadedata/arcadedb:ARCADEDB_TAG
+JARS_SOURCE="${6:-}"
 
 if [[ -z "$PLATFORM" ]] || [[ -z "$ARCADEDB_TAG" ]]; then
-    echo -e "${RED}Usage: $0 PLATFORM PACKAGE_NAME PACKAGE_DESCRIPTION ARCADEDB_TAG [BUILD_VERSION]${NC}"
+    echo -e "${RED}Usage: $0 PLATFORM PACKAGE_NAME PACKAGE_DESCRIPTION ARCADEDB_TAG [BUILD_VERSION] [JARS_SOURCE]${NC}"
     exit 1
 fi
 
@@ -168,39 +170,14 @@ fi
 
 echo ""
 
-# Step 1: Download ArcadeDB JARs (if not already present)
+# Step 1: Stage the ArcadeDB JARs, on every build (scripts/stage_jars.sh)
 JARS_DIR="$PY_BINDINGS_DIR/src/arcadedb_embedded/jars"
-if [[ -d "$JARS_DIR" ]] && [[ $(ls -1 "$JARS_DIR"/*.jar 2> /dev/null | wc -l) -gt 0 ]]; then
-    echo -e "${GREEN}✅ Using existing JARs from: $JARS_DIR${NC}"
-    JAR_COUNT=$(ls -1 "$JARS_DIR"/*.jar | wc -l)
-    echo -e "${CYAN}📦 Found $JAR_COUNT JAR files${NC}"
-else
-    # Check for Docker (needed to download JARs from ArcadeDB image)
-    if ! command -v docker &> /dev/null; then
-        echo -e "${RED}❌ Docker not found and JARs not present${NC}"
-        echo -e "${YELLOW}💡 Either:${NC}"
-        echo -e "${YELLOW}   1. Install Docker to download JARs: https://www.docker.com/get-started${NC}"
-        echo -e "${YELLOW}   2. Or manually place JARs in: $JARS_DIR${NC}"
-        exit 1
-    fi
-
-    echo -e "${CYAN}📥 Downloading ArcadeDB JARs from Docker image...${NC}"
-    TEMP_JARS=$(mktemp -d)
-
-    if ! docker run --rm arcadedata/arcadedb:${ARCADEDB_TAG} tar -cf - -C /home/arcadedb lib | tar -xf - -C "$TEMP_JARS"; then
-        echo -e "${RED}❌ Failed to download JARs from Docker image${NC}"
-        echo -e "${YELLOW}💡 Make sure Docker is running and you have internet access${NC}"
-        rm -rf "$TEMP_JARS"
-        exit 1
-    fi
-
-    mkdir -p "$JARS_DIR"
-    cp "$TEMP_JARS/lib"/*.jar "$JARS_DIR/"
-    rm -rf "$TEMP_JARS"
-    echo -e "${GREEN}✅ JARs downloaded to: $JARS_DIR${NC}"
-fi
+# shellcheck source=stage_jars.sh
+source "$SCRIPT_DIR/stage_jars.sh"
+stage_jars "$JARS_DIR" "$JARS_SOURCE" "$ARCADEDB_TAG"
+echo -e "${CYAN}📦 $(ls -1 "$JARS_DIR"/*.jar | wc -l | tr -d ' ') JAR files${NC}"
 
-# Always filter excluded jars, even when reusing an existing jars directory from a prior build.
+# Always filter excluded jars, whichever source staged them.
 apply_jar_exclusions "$JARS_DIR" "$JAR_EXCLUSIONS_FILE"
 
 # Build the Python-bindings bridge jar (batched row transport; see

scripts/build.sh

@@ -279,7 +279,13 @@ echo ""
 if [[ "$USE_NATIVE" == true ]]; then
     # Native build
     echo -e "${YELLOW} Building natively on ${PLATFORM}...${NC}"
-    "$SCRIPT_DIR/build-native.sh" "$PLATFORM" "$PACKAGE_NAME" "$DESCRIPTION" "$DOCKER_TAG" "${BUILD_VERSION:-}"
+    # The jars build.sh was given (staged into local-jars); none: the native build
+    # pulls arcadedata/arcadedb:$DOCKER_TAG itself, never reusing a previous build's (#9)
+    NATIVE_JARS_SOURCE=""
+    if [[ -n "$JAR_LIB_DIR" ]]; then
+        NATIVE_JARS_SOURCE="$LOCAL_JARS_DIR"
+    fi
+    "$SCRIPT_DIR/build-native.sh" "$PLATFORM" "$PACKAGE_NAME" "$DESCRIPTION" "$DOCKER_TAG" "${BUILD_VERSION:-}" "$NATIVE_JARS_SOURCE"
 else
     # Docker build
     echo -e "${YELLOW} 🐳 Building in Docker...${NC}"

scripts/stage_jars.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Stage the ArcadeDB jars a native build bundles. Sourced by build-native.sh.
+#
+# stage_jars JARS_DIR SOURCE_DIR ARCADEDB_TAG
+#
+# JARS_DIR ends up holding exactly the jars asked for, on every build, its
+# earlier jars (a previous build's, its arcadedb-python-bridge.jar) removed first:
+#   - SOURCE_DIR when given: build.sh passes local-jars/lib, where it copied its
+#     jars directory (in CI, the downloaded artifact). The jars come as given, so
+#     a pinned ARCADEDB_TAG that names another engine version only warns.
+#   - otherwise arcadedata/arcadedb:ARCADEDB_TAG, pulled again: the tags follow
+#     upstream snapshots, so a cached image or a previous build's jars would ship
+#     an old engine under a new build (#9).
+# SOURCE_DIR being JARS_DIR itself (build-native.sh run by hand on its own jars)
+# leaves them as they are: there is nothing to copy.
+
+stage_jars() {
+    local jars_dir="$1"
+    local source_dir="$2"
+    local tag="$3"
+
+    mkdir -p "$jars_dir"
+
+    if [[ -n "$source_dir" ]] && [[ "$source_dir" -ef "$jars_dir" ]]; then
+        echo "📦 Using the jars already in: $jars_dir"
+        return 0
+    fi
+
+    if [[ -n "$source_dir" ]]; then
+        if ! compgen -G "$source_dir/*.jar" > /dev/null; then
+            echo "❌ No JAR files in: $source_dir" >&2
+            return 1
+        fi
+        rm -f "$jars_dir"/*.jar
+        cp "$source_dir"/*.jar "$jars_dir"/
+        echo "📦 Jars staged from: $source_dir"
+        local engine
+        engine=$(python3 "$(dirname "${BASH_SOURCE[0]}")/extract_version.py" "$jars_dir" --format=docker 2> /dev/null || true)
+        if [[ "$tag" != "latest" ]] && [[ -n "$engine" ]] && [[ "$engine" != "$tag" ]]; then
+            echo "⚠️  ARCADEDB_TAG is $tag, but the jars given carry engine $engine: building $engine" >&2
+        fi
+        return 0
+    fi
+
+    if ! command -v docker > /dev/null 2>&1; then
+        echo "❌ Docker not found: it pulls the jars from arcadedata/arcadedb:$tag" >&2
+        echo "💡 Or pass a jars directory: ./scripts/build.sh <platform> <python> <jars-dir>" >&2
+        return 1
+    fi
+
+    local temp_jars
+    temp_jars=$(mktemp -d)
+    echo "📥 Pulling the jars from arcadedata/arcadedb:$tag..."
+    if ! docker run --rm --pull always "arcadedata/arcadedb:$tag" tar -cf - -C /home/arcadedb lib | tar -xf - -C "$temp_jars"; then
+        echo "❌ Failed to get the jars from arcadedata/arcadedb:$tag" >&2
+        rm -rf "$temp_jars"
+        return 1
+    fi
+    rm -f "$jars_dir"/*.jar
+    cp "$temp_jars/lib"/*.jar "$jars_dir"/
+    rm -rf "$temp_jars"
+    echo "✅ Jars from arcadedata/arcadedb:$tag in: $jars_dir"
+}

tests/conftest.py

@@ -16,6 +16,9 @@
 if sys.platform == "win32":
     # test_sigint.py sends SIGINT to a child process, which Windows cannot deliver
     collect_ignore.append("test_sigint.py")
+    # runs scripts/stage_jars.sh under bash with a fake docker; the bash a Windows
+    # subprocess finds may be WSL's rather than Git Bash
+    collect_ignore.append("test_stage_jars.py")
 if not os.path.isfile(
     os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md")
 ):

tests/test_stage_jars.py

@@ -0,0 +1,153 @@
+"""A native build bundles the jars it was asked for, not the last build's (#9).
+
+build-native.sh reused whatever was in src/arcadedb_embedded/jars: with no
+jars directory given it never pulled ``arcadedata/arcadedb:${ARCADEDB_TAG}``,
+so a wheel built after ``latest`` moved carried the old engine; with one given
+(build.sh's third argument) it ignored it. scripts/stage_jars.sh now stages
+the jars on every build: from the directory build.sh passes, or freshly from
+the image. These tests run it under bash with a fake ``docker`` on PATH.
+"""
+
+from __future__ import annotations
+
+import os
+import stat
+import subprocess  # nosec B404 - test-controlled child process
+import tarfile
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts"
+STAGE_JARS = SCRIPTS_DIR / "stage_jars.sh"
+
+
+def _jars(directory: Path, *names: str) -> Path:
+    directory.mkdir(parents=True, exist_ok=True)
+    for name in names:
+        (directory / name).write_text(name)
+    return directory
+
+
+def _fake_docker(tmp_path: Path, *lib_jars: str, fail: bool = False) -> Path:
+    """A docker that logs its argv and streams a tar of lib/ holding lib_jars."""
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    image_home = _jars(tmp_path / "image" / "lib", *lib_jars).parent
+    tar_path = tmp_path / "lib.tar"
+    with tarfile.open(tar_path, "w") as tar:
+        tar.add(image_home / "lib", arcname="lib")
+    docker = bin_dir / "docker"
+    body = "exit 1" if fail else f'cat "{tar_path}"'
+    docker.write_text(
+        f'#!/usr/bin/env bash\necho "$*" >> "{tmp_path / "docker.log"}"\n{body}\n'
+    )
+    docker.chmod(docker.stat().st_mode | stat.S_IEXEC)
+    return bin_dir
+
+
+def _stage(jars_dir: Path, source: str, tag: str, bin_dir=None):
+    env = dict(os.environ)
+    if bin_dir is not None:
+        env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
+    return subprocess.run(  # nosec B603 B607 - fixed argv, no shell
+        [
+            "bash",
+            "-c",
+            'set -euo pipefail; source "$1"; stage_jars "$2" "$3" "$4"',
+            "stage",
+            str(STAGE_JARS),
+            str(jars_dir),
+            source,
+            tag,
+        ],
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+
+def _names(directory: Path) -> set[str]:
+    return {p.name for p in directory.glob("*.jar")}
+
+
+def test_no_source_pulls_the_tag_and_replaces_stale_jars(tmp_path):
+    jars = _jars(
+        tmp_path / "jars",
+        "arcadedb-engine-26.10.2.jar",
+        "arcadedb-python-bridge.jar",
+    )
+    bin_dir = _fake_docker(tmp_path, "arcadedb-engine-26.11.1-SNAPSHOT.jar", "x-1.jar")
+    out = _stage(jars, "", "26.11.1-SNAPSHOT", bin_dir)
+    assert out.returncode == 0, out.stderr
+    assert _names(jars) == {"arcadedb-engine-26.11.1-SNAPSHOT.jar", "x-1.jar"}
+    log = (tmp_path / "docker.log").read_text()
+    assert "--pull always" in log
+    assert "arcadedata/arcadedb:26.11.1-SNAPSHOT" in log
+
+
+def test_source_dir_replaces_stale_jars(tmp_path):
+    jars = _jars(tmp_path / "jars", "arcadedb-engine-26.10.2.jar", "stale-1.jar")
+    source = _jars(tmp_path / "local", "arcadedb-engine-26.11.1-RC1.jar", "y-2.jar")
+    out = _stage(jars, str(source), "latest")
+    assert out.returncode == 0, out.stderr
+    assert _names(jars) == {"arcadedb-engine-26.11.1-RC1.jar", "y-2.jar"}
+    assert _names(source) == {"arcadedb-engine-26.11.1-RC1.jar", "y-2.jar"}
+
+
+def test_source_dir_that_is_the_jars_dir_is_kept(tmp_path):
+    # build-native.sh run by hand on its own jars dir: nothing to copy
+    jars = _jars(tmp_path / "jars", "arcadedb-engine-26.11.1-SNAPSHOT.jar")
+    out = _stage(jars, str(jars), "latest")
+    assert out.returncode == 0, out.stderr
+    assert _names(jars) == {"arcadedb-engine-26.11.1-SNAPSHOT.jar"}
+
+
+def test_failed_pull_fails_the_build(tmp_path):
+    jars = _jars(tmp_path / "jars", "arcadedb-engine-26.10.2.jar")
+    bin_dir = _fake_docker(tmp_path, "unused.jar", fail=True)
+    out = _stage(jars, "", "latest", bin_dir)
+    assert out.returncode != 0
+
+
+def test_build_sh_hands_its_jars_dir_to_the_native_build():
+    # The staged local-jars directory, when build.sh was given one
+    call = [
+        line
+        for line in (SCRIPTS_DIR / "build.sh").read_text(encoding="utf-8").splitlines()
+        if '"$SCRIPT_DIR/build-native.sh"' in line
+    ]
+    assert len(call) == 1, call
+    assert "NATIVE_JARS_SOURCE" in call[0]
+
+
+def test_ci_path_restages_the_artifact_from_local_jars(tmp_path):
+    # CI: build.sh copies its jars dir (the artifact, src/arcadedb_embedded/jars)
+    # into local-jars/lib and hands that over; a previous build's leftovers in
+    # the jars dir go, the artifact's jars come back.
+    jars = _jars(
+        tmp_path / "jars",
+        "arcadedb-engine-26.11.1-SNAPSHOT.jar",
+        "arcadedb-python-bridge.jar",
+        "arcadedb-engine-26.10.2.jar",
+    )
+    local = _jars(
+        tmp_path / "local-jars" / "lib", "arcadedb-engine-26.11.1-SNAPSHOT.jar"
+    )
+    out = _stage(jars, str(local), "26.11.1-SNAPSHOT")
+    assert out.returncode == 0, out.stderr
+    assert _names(jars) == {"arcadedb-engine-26.11.1-SNAPSHOT.jar"}
+
+
+def test_pinned_tag_that_differs_from_the_given_jars_warns(tmp_path):
+    jars = tmp_path / "jars"
+    source = _jars(tmp_path / "local", "arcadedb-engine-26.10.2.jar")
+    out = _stage(jars, str(source), "26.11.1-SNAPSHOT")
+    assert out.returncode == 0, out.stderr
+    assert "26.11.1-SNAPSHOT" in out.stderr and "26.10.2" in out.stderr
+
+
+def test_matching_or_latest_tag_does_not_warn(tmp_path):
+    source = _jars(tmp_path / "local", "arcadedb-engine-26.10.2.jar")
+    for tag in ("26.10.2", "latest"):
+        out = _stage(tmp_path / f"jars-{tag}", str(source), tag)
+        assert out.returncode == 0, out.stderr
+        assert "⚠️" not in out.stderr