diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..f0a6d38e --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,12 @@ +# actionlint configuration, picked up automatically by the pre-commit hook (.pre-commit-config.yaml). +# +# The ignores below are the shellcheck findings the two workflows carried when they moved here from +# ArcadeData/arcadedb, scoped to the exact file and rule. They are style findings in `run:` scripts, not +# schema errors, and the list is meant to shrink: fix the script and delete its entry. +paths: + .github/workflows/test-python-bindings.yml: + ignore: + - 'shellcheck reported issue in this script: SC(2001|2010|2012|2086|2129):' + .github/workflows/test-python-examples.yml: + ignore: + - 'shellcheck reported issue in this script: SC(2010|2012|2086|2129|2188):' diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5debcbd1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,26 @@ +version: 2 + +# Every update block carries `cooldown: default-days: 3`, so Dependabot will not +# propose a release until it has been public for three days: a compromised +# release is usually pulled well inside that window. Security updates ignore +# the cooldown and still open the moment an advisory lands. +updates: + # pyproject.toml is the package definition (`[tool.uv] managed = false`, no + # uv.lock), so it is tracked as a pip project rather than a uv one. + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: weekly + day: "sunday" + cooldown: + default-days: 3 + open-pull-requests-limit: 10 + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: weekly + day: "sunday" + cooldown: + default-days: 3 + open-pull-requests-limit: 10 diff --git a/.github/workflows/test-python-bindings.yml b/.github/workflows/test-python-bindings.yml new file mode 100644 index 00000000..1f5ab6be --- /dev/null +++ b/.github/workflows/test-python-bindings.yml @@ -0,0 +1,571 @@ +name: Test Python Bindings + +on: + push: + branches: [main] + pull_request: + + # A new upstream snapshot is published as arcadedata/arcadedb:latest without + # any change here, so build and test against it every night as well. + schedule: + - cron: "0 3 * * *" + + # Allow being called by other workflows (e.g., release workflow) + workflow_call: + inputs: + build-version: + description: "Override package version (PEP 440) for build.sh" + required: false + type: string + + # Allow manual trigger + workflow_dispatch: + +permissions: + contents: read + +jobs: + bandit: + name: Bandit security scan + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install Bandit + run: python -m pip install "bandit==1.9.4" + + - name: Run Bandit on src and tests (must be clean) + run: python -m bandit -c pyproject.toml -r src tests --severity-level low --confidence-level low + + - name: Run Bandit on examples (must be clean at medium+/high-confidence) + run: python -m bandit -c pyproject.toml -r examples --severity-level medium --confidence-level high + + # Meterian cannot scan this folder: `uv lock` refuses on a project carrying + # `[tool.uv] managed = false`, so it produces no lockfile to read (see #5616). + # This audits the dependency floors declared in pyproject.toml instead. + dependency-floors: + name: Dependency floor audit + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + # Pinned rather than tracking latest, unlike the other uv installs in this + # repo: this job's own output depends on how uv resolves, so a silent + # resolver change would silently change what gets audited. + - name: Setup UV package manager + shell: bash + run: | + # Retry transient network failures (a connection reset failed a whole examples job on #8897); the + # script is downloaded first so a retried transfer is never piped half-written into sh. + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/0.12.0/install.sh + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + uv --version + + - name: Install pip-audit + run: python -m pip install "pip-audit==2.10.1" + + # --resolution lowest-direct pins every declared dependency to the oldest + # release its own constraint admits, which is the version this manifest + # actually promises to work with. The default resolution would pick the + # newest instead, and a floor left open to a vulnerable release would + # never show up. + # + # The dev extra is excluded on purpose: black, isort, mypy and pytest-cov + # declare no lower bound, so lowest-direct drops them onto their earliest + # PyPI releases (mypy 0.1, which pulls Python 2-era sdists that fail to + # build). It is contributor tooling and is never installed from the wheel. + # + # Every interpreter the package claims to support is resolved, because the + # closure is interpreter-dependent even though the declared floors are not: + # 3.10 pulls exceptiongroup, tomli and typing-extensions that 3.12 does not, + # and 3.14 drops one more. --python-version selects the resolution target + # without installing anything, so this stays one job rather than a matrix. + # The list is read from the classifiers rather than repeated here, so adding + # an interpreter to the package cannot leave it unaudited. + # + # --disable-pip reads the pinned file directly rather than asking pip to + # resolve it. Without it, a floor whose version has no wheel for the + # running interpreter fails as an sdist build error instead of reporting + # the advisory that caused the regression. + - name: Audit declared dependency floors on every supported interpreter + run: | + # The trailing dot in the prefix is load-bearing: the classifiers also + # carry a bare "Programming Language :: Python :: 3", which without it + # yields --python-version 3 and fails. + versions=$(python -c 'import tomllib;p="Programming Language :: Python :: 3.";print(" ".join(c.rsplit(" ",1)[1] for c in tomllib.load(open("pyproject.toml","rb"))["project"]["classifiers"] if c.startswith(p)))') + # No classifiers would audit nothing and still pass, which is the kind + # of silent gap this job exists to close. + [ -n "$versions" ] || { echo "no Python version classifiers found in pyproject.toml"; exit 1; } + echo "auditing declared floors for: $versions" + for v in $versions; do + echo "::group::Resolve floors for Python $v" + uv pip compile pyproject.toml \ + --extra test --extra vector --extra examples --extra arrow --extra pandas \ + --resolution lowest-direct \ + --python-version "$v" \ + -o "$RUNNER_TEMP/floors-$v.txt" + cat "$RUNNER_TEMP/floors-$v.txt" + echo "::endgroup::" + pip-audit --no-deps --disable-pip -r "$RUNNER_TEMP/floors-$v.txt" + done + + # First job: Download ArcadeDB JARs (platform-agnostic) + download-jars: + name: Download ArcadeDB JARs + runs-on: ubuntu-latest + outputs: + arcadedb-tag: ${{ steps.jars.outputs.arcadedb-tag }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Download JARs from ArcadeDB Docker image + id: jars + shell: bash + run: | + # Download JARs from the official Docker image. `latest` follows + # upstream's current snapshot. Every build job, Linux included, builds + # from these jars (the artifact below), not from its own pull: upstream + # re-pushes the snapshot tag on every deploy, so a pull per job could put + # two different engine builds under one version in a single run. + echo "๐Ÿ“ฆ Downloading JARs from arcadedata/arcadedb:latest..." + + # Create output directory + mkdir -p src/arcadedb_embedded/jars + + # Create a temporary container and copy JARs from it + CONTAINER_ID=$(docker create arcadedata/arcadedb:latest) + docker cp $CONTAINER_ID:/home/arcadedb/lib/. src/arcadedb_embedded/jars/ + docker rm $CONTAINER_ID + + # Verify JARs were downloaded + ls -lh src/arcadedb_embedded/jars/ + JAR_COUNT=$(ls -1 src/arcadedb_embedded/jars/*.jar 2>/dev/null | wc -l) + echo "โœ… Downloaded $JAR_COUNT JAR files" + + ARCADEDB_TAG=$(python3 scripts/extract_version.py src/arcadedb_embedded/jars --format=docker) + echo "๐Ÿ“Œ ArcadeDB version: $ARCADEDB_TAG" + echo "arcadedb-tag=$ARCADEDB_TAG" >> "$GITHUB_OUTPUT" + + # Unfiltered on purpose: build-native.sh applies scripts/jar_exclusions.txt + # to whatever lands in the jars directory, so the list lives in one place. + - name: Upload JARs as artifact + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: arcadedb-jars + path: src/arcadedb_embedded/jars/*.jar + retention-days: 1 + + # Second job: Build and test on each platform + test: + name: Test arcadedb-embedded (${{ matrix.platform }}, Python ${{ matrix.python-version }}) + runs-on: ${{ matrix.runs-on }} + needs: download-jars + env: + BUILD_VERSION: ${{ inputs.build-version }} + strategy: + # Don't cancel other matrix jobs if one fails + fail-fast: false + matrix: + python-version: + - '3.10' + - '3.11' + - '3.12' + - '3.13' + - '3.14' + # Temporarily limit to four platforms (skip macOS x86_64, Windows ARM64) + # platform: ['linux/amd64', 'linux/arm64', 'darwin/amd64', 'darwin/arm64', 'windows/amd64', 'windows/arm64'] + platform: ['linux/amd64', 'linux/arm64', 'darwin/arm64', 'windows/amd64'] + include: + - platform: linux/amd64 + runs-on: ubuntu-24.04 + - platform: linux/arm64 + runs-on: ubuntu-24.04-arm + - platform: darwin/arm64 + runs-on: macos-15 + - platform: windows/amd64 + runs-on: windows-2025 + # - platform: darwin/amd64 + # runs-on: macos-15-intel + # - platform: windows/arm64 + # runs-on: windows-11-arm + # macOS x86_64 and Windows ARM64 temporarily disabled + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup UV package manager + shell: bash + run: | + # Retry transient network failures (a connection reset failed a whole examples job on #8897); the + # script is downloaded first so a retried transfer is never piped half-written into sh. + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/install.sh + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + uv --version + + - name: Download ArcadeDB JARs artifact + # Every platform builds from this artifact, see download-jars. + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: arcadedb-jars + path: src/arcadedb_embedded/jars + + - name: Set up Java (for native builds on macOS/Windows) + if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/arm64' + uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: 'corretto' + java-version: '25' + + - name: Set up Docker Buildx (Linux only) + if: matrix.platform == 'linux/amd64' || matrix.platform == 'linux/arm64' + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Install Python build dependencies + shell: bash + run: | + uv pip install --system build wheel setuptools + + - name: Set UTF-8 encoding (Windows only) + if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' + shell: bash + run: | + echo "PYTHONIOENCODING=utf-8" >> $GITHUB_ENV + echo "PYTHONUTF8=1" >> $GITHUB_ENV + + # Windows currently enabled, no symlink needed + # - name: Create python3 symlink (Windows only) + # if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' + # shell: bash + # run: | + # PYTHON_DIR=$(dirname "$(which python)") + # ln -s "$PYTHON_DIR/python.exe" "$PYTHON_DIR/python3.exe" || true + # python3 --version + + - name: Build and test arcadedb-embedded (${{ matrix.platform }}) + shell: bash + env: + # The version download-jars resolved from `latest`; the Docker build still + # takes its base image by tag, but the jars come from the artifact + ARCADEDB_TAG: ${{ needs.download-jars.outputs.arcadedb-tag }} + run: | + echo "๐Ÿ”จ Building arcadedb-embedded for ${{ matrix.platform }} with Python ${{ matrix.python-version }}..." + # The jars download-jars fetched, on every platform, so all wheels of this + # run carry the same engine build (and build.sh checks the integration jar) + ./scripts/build.sh ${{ matrix.platform }} ${{ matrix.python-version }} src/arcadedb_embedded/jars + + - name: Extract wheel for additional testing + shell: bash + run: | + mkdir -p test-install + cp dist/*.whl test-install/ + + - name: Set up Python for host testing + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + # Note: Java is NOT required for arcadedb-embedded (JRE is bundled) + # This package works without any external Java installation! + + - name: Install wheel and test dependencies + shell: bash + run: | + # This list is hand-maintained and mirrors the `test` extra. It had + # already drifted: pyarrow and pandas were missing, so to_arrow() and + # to_dataframe() tests skipped themselves in this job while it + # reported green. The guard step below now fails the build if any + # test skips for a missing import, so the next drift is loud. + uv pip install --system test-install/*.whl pytest pytest-cov requests numpy "psycopg[binary]" redis neo4j pyarrow pandas "adbc-driver-postgresql>=1.12" + + - name: Run pytest on host + id: pytest + # A HUNG TEST ENDS THE STEP, NOT THE DAY. On 2026-09-25 the windows/3.13 + # job sat in test_lsm_vector_quantization_int8_comprehensive for 86 minutes + # (re-run: passed in 7), and with no limit it would have held the run for + # GitHub's 6-hour default. The step takes 108-319 s on every platform. + timeout-minutes: 30 + shell: bash + run: | + echo "๐Ÿงช Testing arcadedb-embedded (with bundled JRE)..." + + # Run pytest with JUnit XML output for structured results + # WINDOWS RUNS WITH --capture=sys (humemai/arcadedb-embedded-python#10). pytest's default fd capture swaps + # fds 1 and 2 with dup2 around every test; on Windows that closes the OS handle + # the JVM cached for System.out/err at start-up, a later pipe or socket can be + # handed the same handle value, and the next Java log line then blocks forever + # in WriteFile once that pipe is full. The Java dump caught it on 2026-09-29: + # main thread in FileOutputStream.writeBytes under LSMVectorIndex.build's + # progress log. sys capture never touches the fds, so the JVM keeps writing to + # the runner's own pipes (its log lines appear in this job's log). No test uses + # capfd. + CAPTURE="" + if [ "$RUNNER_OS" = "Windows" ]; then CAPTURE="--capture=sys"; fi + set +e # Don't exit on test failures + pytest tests/ -v --tb=short --color=yes --junitxml=test-results.xml $CAPTURE + PYTEST_EXIT_CODE=$? + set -e + + # Parse JUnit XML for test counts (cross-platform XML parsing) + if [ -f "test-results.xml" ]; then + # Extract counts from testsuite element + TOTAL=$(grep -oE 'tests="[0-9]+"' test-results.xml | grep -oE '[0-9]+' | head -1) + FAILED=$(grep -oE 'failures="[0-9]+"' test-results.xml | grep -oE '[0-9]+' | head -1) + SKIPPED=$(grep -oE 'skipped="[0-9]+"' test-results.xml | grep -oE '[0-9]+' | head -1) + ERRORS=$(grep -oE 'errors="[0-9]+"' test-results.xml | grep -oE '[0-9]+' | head -1) + # JUnit counts an expected failure (a strict xfail: an open upstream bug the suite tracks) + # as skipped, so a run with 14 of them read "14 skipped". Count them apart from real skips. + # grep exits 1 when nothing matches, and this shell runs with pipefail and -e: a suite with no expected failures + # (every upstream bug fixed, no strict xfail left) used to abort the step here, silently, after all tests passed. + XFAILED=$({ grep -oE ']*type="pytest.xfail"' test-results.xml || true; } | wc -l | tr -d ' ') + + # Default to 0 if not found + TOTAL=${TOTAL:-0} + FAILED=${FAILED:-0} + SKIPPED=${SKIPPED:-0} + ERRORS=${ERRORS:-0} + XFAILED=${XFAILED:-0} + + # Calculate passed tests (JUnit's skipped total includes the expected failures) + PASSED=$((TOTAL - FAILED - SKIPPED - ERRORS)) + SKIPPED=$((SKIPPED - XFAILED)) + + echo "passed=$PASSED" >> $GITHUB_OUTPUT + echo "skipped=$SKIPPED" >> $GITHUB_OUTPUT + echo "xfailed=$XFAILED" >> $GITHUB_OUTPUT + echo "failed=$FAILED" >> $GITHUB_OUTPUT + + echo "โœ… Test results: $PASSED passed, $XFAILED expected failures (xfail), $SKIPPED skipped, $FAILED failed" + + # A test that skips because a dependency is missing is not a + # passing test, and it does not look like a failure either: the + # suite goes green having exercised nothing. + # + # This has now happened three times. pyarrow was dropped by an + # install command and the to_arrow() tests skipped; pandas was + # never in the `test` extra at all, so to_dataframe() had no + # executed coverage in any CI run this package has ever had; and + # the hand-written dependency list in this very job was missing + # both. Each time the signal was a green build. + # + # So make it loud. importorskip failures name the module, and a + # missing dependency is a CI bug rather than an expected condition. + IMPORT_SKIPS=$(grep -oE 'could not import .[a-zA-Z0-9_.]+.' test-results.xml | sort -u || true) + if [ -n "$IMPORT_SKIPS" ]; then + echo "โŒ Tests skipped because dependencies are missing:" + echo "$IMPORT_SKIPS" | sed 's/^/ /' + echo "" + echo " These tests did not run. Add the module to the 'test' extra in" + echo " pyproject.toml AND to the install step above," + echo " or delete the test if the integration is no longer supported." + exit 1 + fi + + # Any other skip must be one somebody accepted (scripts/check_test_skips.py + # names each, with the platform it applies to and why). A skip because the + # runtime "lacks" a feature the wheel always ships is how a wrong empty answer + # once passed as a skip; the import grep above is the special case. + python scripts/check_test_skips.py test-results.xml --platform "${{ matrix.platform }}" || exit 1 + + # Force failure if there are failures or errors + if [ "$FAILED" -gt 0 ] || [ "$ERRORS" -gt 0 ]; then + echo "โŒ Tests failed (failures=$FAILED, errors=$ERRORS)" + exit 1 + fi + else + echo "โš ๏ธ test-results.xml not found" + exit 1 + fi + + # Exit with pytest's exit code + exit $PYTEST_EXIT_CODE + + - name: Generate test summary + if: always() + shell: python + run: | + import os + import zipfile + import glob + from pathlib import Path + + summary_file = os.environ.get('GITHUB_STEP_SUMMARY', '/dev/null') + + with open(summary_file, 'a', encoding='utf-8') as f: + f.write("## ๐Ÿงช Test Results: arcadedb-embedded (${{ matrix.platform }})\n") + f.write("\n") + + # Check test status + if "${{ steps.pytest.outcome }}" == "success": + f.write("โœ… **Status**: PASSED\n") + else: + f.write("โŒ **Status**: FAILED\n") + + f.write("\n") + f.write("| Metric | Count |\n") + f.write("|--------|-------|\n") + + # Show test results + f.write("| โœ… Passed | ${{ steps.pytest.outputs.passed || 'N/A' }} |\n") + f.write("| โญ๏ธ Skipped | ${{ steps.pytest.outputs.skipped || 'N/A' }} |\n") + f.write("| ๐Ÿ”– Expected failures (xfail) | ${{ steps.pytest.outputs.xfailed || 'N/A' }} |\n") + f.write("| โŒ Failed | ${{ steps.pytest.outputs.failed || 'N/A' }} |\n") + f.write("\n") + + f.write("### ๐Ÿ“ฆ Package Info:\n") + f.write("\n") + + # Get wheel size and analyze contents + wheel_files = glob.glob("dist/*.whl") + if wheel_files: + wheel_file = wheel_files[0] + + # Get wheel size + wheel_size_bytes = os.path.getsize(wheel_file) + wheel_size_mb = wheel_size_bytes / 1024 / 1024 + + # Analyze wheel contents + jre_size_mb = "N/A" + jar_size_mb = "N/A" + installed_size_mb = "N/A" + + try: + # Extract and analyze (cross-platform) + import tempfile + import shutil + + with tempfile.TemporaryDirectory() as temp_dir: + with zipfile.ZipFile(wheel_file, 'r') as whl: + whl.extractall(temp_dir) + + # Calculate JRE size + jre_dir = None + for root, dirs, files in os.walk(temp_dir): + if 'jre' in dirs: + jre_dir = os.path.join(root, 'jre') + break + + if jre_dir and os.path.isdir(jre_dir): + jre_size_kb = 0 + for root, dirs, files in os.walk(jre_dir): + for file in files: + jre_size_kb += os.path.getsize(os.path.join(root, file)) + jre_size_mb = jre_size_kb / 1024 / 1024 + + # Calculate JAR size + jar_files = [] + for root, dirs, files in os.walk(temp_dir): + for file in files: + if file.endswith('.jar'): + jar_files.append(os.path.join(root, file)) + + if jar_files: + jar_size_kb = sum(os.path.getsize(f) for f in jar_files) / 1024 + jar_size_mb = jar_size_kb / 1024 + + # Calculate total size + total_size_kb = 0 + for root, dirs, files in os.walk(temp_dir): + for file in files: + total_size_kb += os.path.getsize(os.path.join(root, file)) + installed_size_mb = total_size_kb / 1024 / 1024 + except Exception as e: + print(f"Warning: Could not analyze wheel contents: {e}") + + f.write(f"- **Wheel Size**: {wheel_size_mb:.1f}M (compressed)\n") + if isinstance(jre_size_mb, str): + f.write(f"- **JRE Size**: {jre_size_mb} (uncompressed)\n") + else: + f.write(f"- **JRE Size**: {jre_size_mb:.1f}M (uncompressed)\n") + if isinstance(jar_size_mb, str): + f.write(f"- **JARs Size**: {jar_size_mb} (uncompressed)\n") + else: + f.write(f"- **JARs Size**: {jar_size_mb:.1f}M (uncompressed)\n") + if isinstance(installed_size_mb, str): + f.write(f"- **Installed Size**: {installed_size_mb} (total uncompressed)\n") + else: + f.write(f"- **Installed Size**: ~{installed_size_mb:.0f}M (total uncompressed)\n") + + f.write("- **Platform**: ${{ matrix.platform }}\n") + f.write("- **Features**: All ArcadeDB features (some JARs excluded, see jar_exclusions.txt)\n") + + - name: Upload test results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-results-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }}-py${{ matrix.python-version }} + path: | + pytest-output.txt + .coverage + retention-days: 7 + + - name: Upload wheel artifact + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }}-py${{ matrix.python-version }} + path: dist/*.whl + retention-days: 7 + + - name: Upload wheel artifact for release + if: matrix.python-version == '3.12' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }}-test + path: dist/*.whl + retention-days: 7 + + # Summary job that checks all platforms + test-summary: + name: Test Summary + needs: test + runs-on: ubuntu-24.04 + if: always() + steps: + - name: Check test results + run: | + echo "## ๐ŸŽฏ Overall Test Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ needs.test.result }}" = "success" ]; then + echo "โœ… **All platforms passed testing!**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The arcadedb-embedded package has been successfully built and tested." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Package**: arcadedb-embedded" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "โ„น๏ธ **Note**: Some platform/Python combinations are excluded from testing:" >> $GITHUB_STEP_SUMMARY + echo "- Windows ARM64 (no GitHub-hosted runners available)" >> $GITHUB_STEP_SUMMARY + echo "- macOS x86_64 (temporarily disabled)" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ **Some platforms failed testing**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Please check the individual test jobs for details." >> $GITHUB_STEP_SUMMARY + exit 1 + fi diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml new file mode 100644 index 00000000..17f2c4d5 --- /dev/null +++ b/.github/workflows/test-python-examples.yml @@ -0,0 +1,727 @@ +name: Test Python Examples + +on: + push: + branches: [main] + pull_request: + + # A new upstream snapshot is published as arcadedata/arcadedb:latest without + # any change here, so build and test against it every night as well. + schedule: + - cron: "0 4 * * *" + + # Allow being called by other workflows (e.g., release workflow) + workflow_call: + inputs: + examples: + description: "Glob pattern(s) for examples to run (space-separated, relative to examples)." + required: false + type: string + default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" + build-version: + description: "Override package version (PEP 440) for build.sh" + required: false + type: string + + # Allow manual trigger + workflow_dispatch: + inputs: + examples: + description: "Glob pattern(s) for examples to run (space-separated, relative to examples)." + required: false + default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" + +# SUPERSEDE, do not race. +# +# A force-push to a PR branch left the OLD run going alongside the new one: on +# #6889 both 118b347ce and 75293c7a2 ran at once, and the stale one reported a +# failure that the newer commit had already fixed. Anyone reading the PR sees a +# red check against work that is done. +# +# cancel-in-progress is scoped to pull_request ONLY. On a push to a long-lived +# branch a superseded run is often the one you still want -- it is the record +# for that commit -- so those are left to finish. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + EXAMPLES: ${{ inputs.examples || '0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py' }} + +# Note: Example 21 runs in CI at reduced scale; its arguments and 15-minute +# limit are set in the per-example case in the "Run all examples" step below. + +permissions: + contents: read + +jobs: + # First job: Download ArcadeDB JARs (platform-agnostic) + download-jars: + name: Download ArcadeDB JARs + runs-on: ubuntu-24.04 + outputs: + arcadedb-tag: ${{ steps.jars.outputs.arcadedb-tag }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Download JARs from ArcadeDB Docker image + id: jars + shell: bash + run: | + # Download JARs from the official Docker image. `latest` follows + # upstream's current snapshot. Every build job, Linux included, builds + # from these jars (the artifact below), not from its own pull: upstream + # re-pushes the snapshot tag on every deploy, so a pull per job could put + # two different engine builds under one version in a single run. + echo "๐Ÿ“ฆ Downloading JARs from arcadedata/arcadedb:latest..." + + # Create output directory + mkdir -p src/arcadedb_embedded/jars + + # Create a temporary container and copy JARs from it + CONTAINER_ID=$(docker create arcadedata/arcadedb:latest) + docker cp $CONTAINER_ID:/home/arcadedb/lib/. src/arcadedb_embedded/jars/ + docker rm $CONTAINER_ID + + echo "โœ… Downloaded $(ls -1 src/arcadedb_embedded/jars/*.jar | wc -l) JARs" + + ARCADEDB_TAG=$(python3 scripts/extract_version.py src/arcadedb_embedded/jars --format=docker) + echo "๐Ÿ“Œ ArcadeDB version: $ARCADEDB_TAG" + echo "arcadedb-tag=$ARCADEDB_TAG" >> "$GITHUB_OUTPUT" + + # Unfiltered on purpose: build-native.sh applies scripts/jar_exclusions.txt + # to whatever lands in the jars directory, so the list lives in one place. + - name: Upload JARs as artifact + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: arcadedb-jars-examples + path: src/arcadedb_embedded/jars/*.jar + retention-days: 1 + + # Second job: Test examples on each platform + test-examples: + name: Test Python Examples (${{ matrix.platform }}, Python ${{ matrix.python-version }}) + runs-on: ${{ matrix.runs-on }} + needs: download-jars + env: + BUILD_VERSION: ${{ inputs.build-version }} + # Pin the Hugging Face cache inside the workspace so it is cacheable with + # one path on every platform (the default differs per OS). The examples + # embed with sentence-transformers, and this matrix is 20 jobs, so + # uncached that is 20 independent model downloads per release and 20 + # chances for a transient hub outage to fail a release. It has happened. + HF_HOME: ${{ github.workspace }}/.hf-cache + strategy: + fail-fast: false + matrix: + python-version: + - '3.10' + - '3.11' + - '3.12' + - '3.13' + - '3.14' + # Temporarily limit to four platforms (skip macOS x86_64, Windows ARM64) + # platform: ['linux/amd64', 'linux/arm64', 'darwin/amd64', 'darwin/arm64', 'windows/amd64', 'windows/arm64'] + platform: ['linux/amd64', 'linux/arm64', 'darwin/arm64', 'windows/amd64'] + include: + - platform: linux/amd64 + runs-on: ubuntu-24.04 + - platform: linux/arm64 + runs-on: ubuntu-24.04-arm + - platform: darwin/arm64 + runs-on: macos-15 + - platform: windows/amd64 + runs-on: windows-2025 + # - platform: darwin/amd64 + # runs-on: macos-15-intel + # - platform: windows/arm64 + # runs-on: windows-11-arm + # macOS x86_64 and Windows ARM64 temporarily disabled + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Download JARs artifact + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: arcadedb-jars-examples + path: src/arcadedb_embedded/jars + + - name: Set up Java (for native builds on macOS/Windows) + if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/arm64' + uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: 'corretto' + java-version: '25' + + - name: Set up Docker Buildx (Linux only) + if: matrix.platform == 'linux/amd64' || matrix.platform == 'linux/arm64' + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup UV package manager + shell: bash + run: | + # Retry transient network failures (a connection reset failed a whole examples job on #8897); the + # script is downloaded first so a retried transfer is never piped half-written into sh. + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/install.sh + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + uv --version + + - name: Install Python build dependencies + shell: bash + run: | + uv pip install --system build wheel setuptools + + # Windows currently enabled, no symlink needed + # - name: Create python3 symlink (Windows only) + # if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' + # shell: bash + # run: | + # PYTHON_DIR=$(dirname "$(which python)") + # ln -s "$PYTHON_DIR/python.exe" "$PYTHON_DIR/python3.exe" || true + # python3 --version + + - name: Build arcadedb-embedded (${{ matrix.platform }}) + shell: bash + env: + # The version download-jars resolved from `latest`; the Docker build still + # takes its base image by tag, but the jars come from the artifact + ARCADEDB_TAG: ${{ needs.download-jars.outputs.arcadedb-tag }} + run: | + echo "๐Ÿ”จ Building arcadedb-embedded for ${{ matrix.platform }} with Python ${{ matrix.python-version }}..." + # The jars download-jars fetched, on every platform, so all wheels of this + # run carry the same engine build (and build.sh checks the integration jar) + ./scripts/build.sh ${{ matrix.platform }} ${{ matrix.python-version }} src/arcadedb_embedded/jars + + # Note: Java is NOT required - arcadedb-embedded has bundled JRE! + + - name: Install ArcadeDB Python bindings + shell: bash + run: | + uv pip install --system dist/*embed*.whl + + - name: Install example dependencies + shell: bash + run: | + # Install dependencies needed by examples + # PyTorch is required by sentence-transformers, even on macOS + # For Python 3.13+, wheels might only be on PyTorch official index + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS: Install CPU-only PyTorch (supports MPS acceleration) + uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu + uv pip install --system "numpy<2.0" requests sentence-transformers + else + # Linux & Windows: Install CPU-only PyTorch to save space (avoid CUDA) + uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu + uv pip install --system numpy requests sentence-transformers + fi + + # Restored before the dataset step, which is what triggers the model + # download. Keyed on the model name so bumping the model invalidates it. + - name: Cache Hugging Face model + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.hf-cache + key: hf-all-MiniLM-L6-v2-${{ runner.os }}-${{ runner.arch }} + restore-keys: | + hf-all-MiniLM-L6-v2-${{ runner.os }}- + + # Pre-warm with bounded retries, separately from the dataset step, so a + # transient hub failure is retried here rather than surfacing mid-example + # as an OSError that reads like a bindings bug. Retrying is appropriate + # for this step specifically: it exercises third-party I/O, not our code. + # Deliberately NOT fail-fast: if the hub is truly down the dataset step + # below still runs and fails with its own message, so this cannot turn a + # real breakage into a silent skip. + - name: Pre-warm embedding model + shell: bash + continue-on-error: true + run: | + uv pip install --system sentence-transformers >/dev/null 2>&1 || true + for attempt in 1 2 3; do + if python3 -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2'); print('model ready')"; then + echo "โœ… model cached on attempt $attempt" + exit 0 + fi + echo "โš ๏ธ model fetch failed (attempt $attempt/3)" + [ "$attempt" -lt 3 ] && sleep $((attempt * 20)) + done + echo "::warning::could not pre-warm the embedding model after 3 attempts; the dataset step will retry" + + - name: Download datasets + shell: bash + run: | + uv pip install --system tqdm py7zr lxml + cd examples + + # Determine which examples are selected + examples=$(ls $EXAMPLES 2>/dev/null || true) + + # EXIT 75 MEANS "THE DATASET HOST IS DOWN", NOT "THE BINDINGS BROKE". + # + # On 2026-08-29 the TLS certificate for files.grouplens.org expired and + # every job here went red on BOTH this fork and upstream's main -- 11 + # and 12 jobs, every OS, Python 3.10 through 3.14 -- at this step, + # before a single example ran. Re-running does not help: a re-run + # happens now, and the certificate is still expired. + # + # The cost is not the lost coverage, it is that upstream's main showed + # a failing run titled with a Python bindings PR. A third party letting + # a certificate lapse must not be indistinguishable from a regression + # we introduced. + # + # So: skip the examples that need the dataset, keep running the rest, + # and say plainly why. Any other non-zero exit is still a hard failure. + SKIPPED_DATASETS="" + fetch() { # fetch