From d955df29ed965ad83a4e6ae22e8b7e834318dfff Mon Sep 17 00:00:00 2001 From: robfrank Date: Fri, 23 Oct 2026 01:32:58 +0200 Subject: [PATCH] fix(#9): native builds stage the jars asked for on every build build-native.sh reused whatever was in src/arcadedb_embedded/jars: with no jars directory it never pulled arcadedata/arcadedb:$ARCADEDB_TAG, and with one it ignored it. scripts/stage_jars.sh now clears and stages the jars on every build, from the jars directory build.sh passes (local-jars/lib) or freshly from the image (docker run --pull always). A pinned tag that names another engine than the given jars warns. Co-Authored-By: Claude Opus 5.5 --- scripts/build-native.sh | 41 +++-------- scripts/build.sh | 8 +- scripts/stage_jars.sh | 63 ++++++++++++++++ tests/conftest.py | 3 + tests/test_stage_jars.py | 153 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 235 insertions(+), 33 deletions(-) create mode 100644 scripts/stage_jars.sh create mode 100644 tests/test_stage_jars.py diff --git a/scripts/build-native.sh b/scripts/build-native.sh index 214e1925..9957f957 100755 --- a/scripts/build-native.sh +++ b/scripts/build-native.sh @@ -28,9 +28,11 @@ PACKAGE_NAME="${2:-arcadedb-embedded}" PACKAGE_DESCRIPTION="${3:-ArcadeDB embedded multi-model database with bundled JRE}" ARCADEDB_TAG="${4:-}" BUILD_VERSION="${5:-}" +# The jars to bundle; empty pulls them from arcadedata/arcadedb:ARCADEDB_TAG +JARS_SOURCE="${6:-}" if [[ -z "$PLATFORM" ]] || [[ -z "$ARCADEDB_TAG" ]]; then - echo -e "${RED}Usage: $0 PLATFORM PACKAGE_NAME PACKAGE_DESCRIPTION ARCADEDB_TAG [BUILD_VERSION]${NC}" + echo -e "${RED}Usage: $0 PLATFORM PACKAGE_NAME PACKAGE_DESCRIPTION ARCADEDB_TAG [BUILD_VERSION] [JARS_SOURCE]${NC}" exit 1 fi @@ -168,39 +170,14 @@ fi echo "" -# Step 1: Download ArcadeDB JARs (if not already present) +# Step 1: Stage the ArcadeDB JARs, on every build (scripts/stage_jars.sh) JARS_DIR="$PY_BINDINGS_DIR/src/arcadedb_embedded/jars" -if [[ -d "$JARS_DIR" ]] && [[ $(ls -1 "$JARS_DIR"/*.jar 2> /dev/null | wc -l) -gt 0 ]]; then - echo -e "${GREEN}✅ Using existing JARs from: $JARS_DIR${NC}" - JAR_COUNT=$(ls -1 "$JARS_DIR"/*.jar | wc -l) - echo -e "${CYAN}📦 Found $JAR_COUNT JAR files${NC}" -else - # Check for Docker (needed to download JARs from ArcadeDB image) - if ! command -v docker &> /dev/null; then - echo -e "${RED}❌ Docker not found and JARs not present${NC}" - echo -e "${YELLOW}💡 Either:${NC}" - echo -e "${YELLOW} 1. Install Docker to download JARs: https://www.docker.com/get-started${NC}" - echo -e "${YELLOW} 2. Or manually place JARs in: $JARS_DIR${NC}" - exit 1 - fi - - echo -e "${CYAN}📥 Downloading ArcadeDB JARs from Docker image...${NC}" - TEMP_JARS=$(mktemp -d) - - if ! docker run --rm arcadedata/arcadedb:${ARCADEDB_TAG} tar -cf - -C /home/arcadedb lib | tar -xf - -C "$TEMP_JARS"; then - echo -e "${RED}❌ Failed to download JARs from Docker image${NC}" - echo -e "${YELLOW}💡 Make sure Docker is running and you have internet access${NC}" - rm -rf "$TEMP_JARS" - exit 1 - fi - - mkdir -p "$JARS_DIR" - cp "$TEMP_JARS/lib"/*.jar "$JARS_DIR/" - rm -rf "$TEMP_JARS" - echo -e "${GREEN}✅ JARs downloaded to: $JARS_DIR${NC}" -fi +# shellcheck source=stage_jars.sh +source "$SCRIPT_DIR/stage_jars.sh" +stage_jars "$JARS_DIR" "$JARS_SOURCE" "$ARCADEDB_TAG" +echo -e "${CYAN}📦 $(ls -1 "$JARS_DIR"/*.jar | wc -l | tr -d ' ') JAR files${NC}" -# Always filter excluded jars, even when reusing an existing jars directory from a prior build. +# Always filter excluded jars, whichever source staged them. apply_jar_exclusions "$JARS_DIR" "$JAR_EXCLUSIONS_FILE" # Build the Python-bindings bridge jar (batched row transport; see diff --git a/scripts/build.sh b/scripts/build.sh index 4ac30bf4..48cd0da6 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -279,7 +279,13 @@ echo "" if [[ "$USE_NATIVE" == true ]]; then # Native build echo -e "${YELLOW} Building natively on ${PLATFORM}...${NC}" - "$SCRIPT_DIR/build-native.sh" "$PLATFORM" "$PACKAGE_NAME" "$DESCRIPTION" "$DOCKER_TAG" "${BUILD_VERSION:-}" + # The jars build.sh was given (staged into local-jars); none: the native build + # pulls arcadedata/arcadedb:$DOCKER_TAG itself, never reusing a previous build's (#9) + NATIVE_JARS_SOURCE="" + if [[ -n "$JAR_LIB_DIR" ]]; then + NATIVE_JARS_SOURCE="$LOCAL_JARS_DIR" + fi + "$SCRIPT_DIR/build-native.sh" "$PLATFORM" "$PACKAGE_NAME" "$DESCRIPTION" "$DOCKER_TAG" "${BUILD_VERSION:-}" "$NATIVE_JARS_SOURCE" else # Docker build echo -e "${YELLOW} 🐳 Building in Docker...${NC}" diff --git a/scripts/stage_jars.sh b/scripts/stage_jars.sh new file mode 100644 index 00000000..066aa800 --- /dev/null +++ b/scripts/stage_jars.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Stage the ArcadeDB jars a native build bundles. Sourced by build-native.sh. +# +# stage_jars JARS_DIR SOURCE_DIR ARCADEDB_TAG +# +# JARS_DIR ends up holding exactly the jars asked for, on every build, its +# earlier jars (a previous build's, its arcadedb-python-bridge.jar) removed first: +# - SOURCE_DIR when given: build.sh passes local-jars/lib, where it copied its +# jars directory (in CI, the downloaded artifact). The jars come as given, so +# a pinned ARCADEDB_TAG that names another engine version only warns. +# - otherwise arcadedata/arcadedb:ARCADEDB_TAG, pulled again: the tags follow +# upstream snapshots, so a cached image or a previous build's jars would ship +# an old engine under a new build (#9). +# SOURCE_DIR being JARS_DIR itself (build-native.sh run by hand on its own jars) +# leaves them as they are: there is nothing to copy. + +stage_jars() { + local jars_dir="$1" + local source_dir="$2" + local tag="$3" + + mkdir -p "$jars_dir" + + if [[ -n "$source_dir" ]] && [[ "$source_dir" -ef "$jars_dir" ]]; then + echo "📦 Using the jars already in: $jars_dir" + return 0 + fi + + if [[ -n "$source_dir" ]]; then + if ! compgen -G "$source_dir/*.jar" > /dev/null; then + echo "❌ No JAR files in: $source_dir" >&2 + return 1 + fi + rm -f "$jars_dir"/*.jar + cp "$source_dir"/*.jar "$jars_dir"/ + echo "📦 Jars staged from: $source_dir" + local engine + engine=$(python3 "$(dirname "${BASH_SOURCE[0]}")/extract_version.py" "$jars_dir" --format=docker 2> /dev/null || true) + if [[ "$tag" != "latest" ]] && [[ -n "$engine" ]] && [[ "$engine" != "$tag" ]]; then + echo "⚠️ ARCADEDB_TAG is $tag, but the jars given carry engine $engine: building $engine" >&2 + fi + return 0 + fi + + if ! command -v docker > /dev/null 2>&1; then + echo "❌ Docker not found: it pulls the jars from arcadedata/arcadedb:$tag" >&2 + echo "💡 Or pass a jars directory: ./scripts/build.sh " >&2 + return 1 + fi + + local temp_jars + temp_jars=$(mktemp -d) + echo "📥 Pulling the jars from arcadedata/arcadedb:$tag..." + if ! docker run --rm --pull always "arcadedata/arcadedb:$tag" tar -cf - -C /home/arcadedb lib | tar -xf - -C "$temp_jars"; then + echo "❌ Failed to get the jars from arcadedata/arcadedb:$tag" >&2 + rm -rf "$temp_jars" + return 1 + fi + rm -f "$jars_dir"/*.jar + cp "$temp_jars/lib"/*.jar "$jars_dir"/ + rm -rf "$temp_jars" + echo "✅ Jars from arcadedata/arcadedb:$tag in: $jars_dir" +} diff --git a/tests/conftest.py b/tests/conftest.py index 8cdb08d1..b04109dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,9 @@ if sys.platform == "win32": # test_sigint.py sends SIGINT to a child process, which Windows cannot deliver collect_ignore.append("test_sigint.py") + # runs scripts/stage_jars.sh under bash with a fake docker; the bash a Windows + # subprocess finds may be WSL's rather than Git Bash + collect_ignore.append("test_stage_jars.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): diff --git a/tests/test_stage_jars.py b/tests/test_stage_jars.py new file mode 100644 index 00000000..f5926ad1 --- /dev/null +++ b/tests/test_stage_jars.py @@ -0,0 +1,153 @@ +"""A native build bundles the jars it was asked for, not the last build's (#9). + +build-native.sh reused whatever was in src/arcadedb_embedded/jars: with no +jars directory given it never pulled ``arcadedata/arcadedb:${ARCADEDB_TAG}``, +so a wheel built after ``latest`` moved carried the old engine; with one given +(build.sh's third argument) it ignored it. scripts/stage_jars.sh now stages +the jars on every build: from the directory build.sh passes, or freshly from +the image. These tests run it under bash with a fake ``docker`` on PATH. +""" + +from __future__ import annotations + +import os +import stat +import subprocess # nosec B404 - test-controlled child process +import tarfile +from pathlib import Path + +SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts" +STAGE_JARS = SCRIPTS_DIR / "stage_jars.sh" + + +def _jars(directory: Path, *names: str) -> Path: + directory.mkdir(parents=True, exist_ok=True) + for name in names: + (directory / name).write_text(name) + return directory + + +def _fake_docker(tmp_path: Path, *lib_jars: str, fail: bool = False) -> Path: + """A docker that logs its argv and streams a tar of lib/ holding lib_jars.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + image_home = _jars(tmp_path / "image" / "lib", *lib_jars).parent + tar_path = tmp_path / "lib.tar" + with tarfile.open(tar_path, "w") as tar: + tar.add(image_home / "lib", arcname="lib") + docker = bin_dir / "docker" + body = "exit 1" if fail else f'cat "{tar_path}"' + docker.write_text( + f'#!/usr/bin/env bash\necho "$*" >> "{tmp_path / "docker.log"}"\n{body}\n' + ) + docker.chmod(docker.stat().st_mode | stat.S_IEXEC) + return bin_dir + + +def _stage(jars_dir: Path, source: str, tag: str, bin_dir=None): + env = dict(os.environ) + if bin_dir is not None: + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + return subprocess.run( # nosec B603 B607 - fixed argv, no shell + [ + "bash", + "-c", + 'set -euo pipefail; source "$1"; stage_jars "$2" "$3" "$4"', + "stage", + str(STAGE_JARS), + str(jars_dir), + source, + tag, + ], + env=env, + capture_output=True, + text=True, + ) + + +def _names(directory: Path) -> set[str]: + return {p.name for p in directory.glob("*.jar")} + + +def test_no_source_pulls_the_tag_and_replaces_stale_jars(tmp_path): + jars = _jars( + tmp_path / "jars", + "arcadedb-engine-26.10.2.jar", + "arcadedb-python-bridge.jar", + ) + bin_dir = _fake_docker(tmp_path, "arcadedb-engine-26.11.1-SNAPSHOT.jar", "x-1.jar") + out = _stage(jars, "", "26.11.1-SNAPSHOT", bin_dir) + assert out.returncode == 0, out.stderr + assert _names(jars) == {"arcadedb-engine-26.11.1-SNAPSHOT.jar", "x-1.jar"} + log = (tmp_path / "docker.log").read_text() + assert "--pull always" in log + assert "arcadedata/arcadedb:26.11.1-SNAPSHOT" in log + + +def test_source_dir_replaces_stale_jars(tmp_path): + jars = _jars(tmp_path / "jars", "arcadedb-engine-26.10.2.jar", "stale-1.jar") + source = _jars(tmp_path / "local", "arcadedb-engine-26.11.1-RC1.jar", "y-2.jar") + out = _stage(jars, str(source), "latest") + assert out.returncode == 0, out.stderr + assert _names(jars) == {"arcadedb-engine-26.11.1-RC1.jar", "y-2.jar"} + assert _names(source) == {"arcadedb-engine-26.11.1-RC1.jar", "y-2.jar"} + + +def test_source_dir_that_is_the_jars_dir_is_kept(tmp_path): + # build-native.sh run by hand on its own jars dir: nothing to copy + jars = _jars(tmp_path / "jars", "arcadedb-engine-26.11.1-SNAPSHOT.jar") + out = _stage(jars, str(jars), "latest") + assert out.returncode == 0, out.stderr + assert _names(jars) == {"arcadedb-engine-26.11.1-SNAPSHOT.jar"} + + +def test_failed_pull_fails_the_build(tmp_path): + jars = _jars(tmp_path / "jars", "arcadedb-engine-26.10.2.jar") + bin_dir = _fake_docker(tmp_path, "unused.jar", fail=True) + out = _stage(jars, "", "latest", bin_dir) + assert out.returncode != 0 + + +def test_build_sh_hands_its_jars_dir_to_the_native_build(): + # The staged local-jars directory, when build.sh was given one + call = [ + line + for line in (SCRIPTS_DIR / "build.sh").read_text(encoding="utf-8").splitlines() + if '"$SCRIPT_DIR/build-native.sh"' in line + ] + assert len(call) == 1, call + assert "NATIVE_JARS_SOURCE" in call[0] + + +def test_ci_path_restages_the_artifact_from_local_jars(tmp_path): + # CI: build.sh copies its jars dir (the artifact, src/arcadedb_embedded/jars) + # into local-jars/lib and hands that over; a previous build's leftovers in + # the jars dir go, the artifact's jars come back. + jars = _jars( + tmp_path / "jars", + "arcadedb-engine-26.11.1-SNAPSHOT.jar", + "arcadedb-python-bridge.jar", + "arcadedb-engine-26.10.2.jar", + ) + local = _jars( + tmp_path / "local-jars" / "lib", "arcadedb-engine-26.11.1-SNAPSHOT.jar" + ) + out = _stage(jars, str(local), "26.11.1-SNAPSHOT") + assert out.returncode == 0, out.stderr + assert _names(jars) == {"arcadedb-engine-26.11.1-SNAPSHOT.jar"} + + +def test_pinned_tag_that_differs_from_the_given_jars_warns(tmp_path): + jars = tmp_path / "jars" + source = _jars(tmp_path / "local", "arcadedb-engine-26.10.2.jar") + out = _stage(jars, str(source), "26.11.1-SNAPSHOT") + assert out.returncode == 0, out.stderr + assert "26.11.1-SNAPSHOT" in out.stderr and "26.10.2" in out.stderr + + +def test_matching_or_latest_tag_does_not_warn(tmp_path): + source = _jars(tmp_path / "local", "arcadedb-engine-26.10.2.jar") + for tag in ("26.10.2", "latest"): + out = _stage(tmp_path / f"jars-{tag}", str(source), tag) + assert out.returncode == 0, out.stderr + assert "⚠️" not in out.stderr