Repository navigation
JVM: keep NaN, inf and -inf in to_json_list and in nested values of the columnar readers #330
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs examples/ against the `arcadedb-jvm` wheel. The wheel is py3-none-<platform tag> (setup.py): the `build` job builds it once for each | |
| # platform, and the `test-examples` job installs that one file into each Python and runs the examples. Nothing is built per Python. | |
| name: Test Python Examples | |
| on: | |
| push: | |
| branches: [main] | |
| paths-ignore: | |
| # Changes that cannot alter what these jobs build or run. README.md and | |
| # tests/README.md are deliberately NOT here: the wheel build copies README.md | |
| # (scripts/Dockerfile.build, pyproject `readme`) and test_no_monorepo_wording.py | |
| # reads both. docs/** is safe to skip while tests/conftest.py collect-ignores | |
| # test_docs_examples.py (it needs docs/index.md); if that Markdown site comes | |
| # back, take docs/** out of this list. Schedule, dispatch and workflow_call | |
| # ignore these filters and always run everything. | |
| - "docs/**" | |
| - "**/*.adoc" | |
| - "CLAUDE.md" | |
| - "examples/README.md" | |
| - "LICENSE*" | |
| # the native build has its own workflows; nothing under native/ is in the JVM wheel | |
| - "native/**" | |
| - ".github/workflows/test-native.yml" | |
| - ".github/workflows/release-native.yml" | |
| pull_request: | |
| # ready_for_review: draft PRs skip the platform matrices (see the job conditions), | |
| # so marking one ready is what starts them. | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| paths-ignore: | |
| # Same list as under push. | |
| - "docs/**" | |
| - "**/*.adoc" | |
| - "CLAUDE.md" | |
| - "examples/README.md" | |
| - "LICENSE*" | |
| # the native build has its own workflows; nothing under native/ is in the JVM wheel | |
| - "native/**" | |
| - ".github/workflows/test-native.yml" | |
| - ".github/workflows/release-native.yml" | |
| # A new upstream snapshot is published as arcadedata/arcadedb:latest without | |
| # any change here, so build and test against it every night as well. The tag | |
| # moves (ARCADEDB_IMAGE_TAG defaults to latest), so a night with no commit | |
| # here still tests a new engine; that is why there is no skip-if-main-unchanged | |
| # guard. One nightly per workflow, staggered an hour apart. | |
| schedule: | |
| - cron: "0 4 * * *" | |
| # Allow being called by other workflows (release-jvm.yml) | |
| workflow_call: | |
| inputs: | |
| arcadedb-tag: | |
| description: "arcadedata/arcadedb image tag to take the jars from (e.g. 26.10.2); default latest" | |
| required: false | |
| type: string | |
| default: "latest" | |
| examples: | |
| description: "Glob pattern(s) for examples to run (space-separated, relative to examples)." | |
| required: false | |
| type: string | |
| default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" | |
| build-version: | |
| description: "Override package version (PEP 440) for build.sh" | |
| required: false | |
| type: string | |
| # Allow manual trigger | |
| workflow_dispatch: | |
| inputs: | |
| arcadedb-tag: | |
| description: "arcadedata/arcadedb image tag to take the jars from (e.g. 26.10.2); default latest" | |
| required: false | |
| default: "latest" | |
| examples: | |
| description: "Glob pattern(s) for examples to run (space-separated, relative to examples)." | |
| required: false | |
| default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" | |
| # SUPERSEDE, do not race (same rule in test-jvm.yml). | |
| # | |
| # A force-push to a PR branch left the OLD run going alongside the new one: on | |
| # #6889 both 118b347ce and 75293c7a2 ran at once, and the stale one reported a | |
| # failure that the newer commit had already fixed. Anyone reading the PR sees a | |
| # red check against work that is done. The stale run also holds macOS/Windows | |
| # runners the organisation shares with ArcadeData/arcadedb. | |
| # | |
| # cancel-in-progress is scoped to pull_request ONLY. On a push to a long-lived | |
| # branch a superseded run is often the one you still want -- it is the record | |
| # for that commit -- so those are left to finish, and they get a group of their | |
| # own (run_id) so a queued push or nightly run is not replaced by a newer one | |
| # either. The group names the workflow literally: github.workflow is the | |
| # CALLER's name in a workflow_call, which would put this workflow and its | |
| # sibling in one group. | |
| concurrency: | |
| group: test-python-examples-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| # The image download-jars takes the jars from. Unset (push, pull_request, schedule) | |
| # means latest; a release build passes its tag. | |
| ARCADEDB_IMAGE_TAG: ${{ inputs.arcadedb-tag || 'latest' }} | |
| EXAMPLES: ${{ inputs.examples || '0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py' }} | |
| # Note: Example 21 runs in CI at reduced scale; its arguments and 15-minute | |
| # limit are set in the per-example case in the "Run all examples" step below. | |
| permissions: | |
| contents: read | |
| jobs: | |
| # First job: Download ArcadeDB JARs (platform-agnostic) | |
| download-jars: | |
| name: Download ArcadeDB JARs | |
| runs-on: ubuntu-24.04 | |
| # Draft PRs skip the whole workflow; marking the PR ready starts it. | |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false | |
| outputs: | |
| arcadedb-tag: ${{ steps.jars.outputs.arcadedb-tag }} | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| wheel-matrix: ${{ steps.matrix.outputs.wheel-matrix }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| # The platform x Python legs of this run (scripts/select_matrix.py): a small matrix on | |
| # pull requests, the full one on every other event. The organisation shares one runner | |
| # cap with ArcadeData/arcadedb, so a pull request does not run every combination. | |
| # The wheel matrix is the platforms of those legs, one build each: the wheel is | |
| # py3-none-<platform>, the same file for every Python. | |
| # Done first, so a failure here costs no Docker pull. | |
| - name: Select the test matrix | |
| id: matrix | |
| shell: bash | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| run: | | |
| matrix="$(python3 scripts/select_matrix.py examples --event "$EVENT_NAME")" | |
| wheel_matrix="$(python3 scripts/select_matrix.py examples --event "$EVENT_NAME" --wheels)" | |
| echo "$matrix" | |
| echo "$wheel_matrix" | |
| echo "matrix=$matrix" >> "$GITHUB_OUTPUT" | |
| echo "wheel-matrix=$wheel_matrix" >> "$GITHUB_OUTPUT" | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 | |
| - name: Download JARs from ArcadeDB Docker image | |
| id: jars | |
| shell: bash | |
| run: | | |
| # Download JARs from the official Docker image: ARCADEDB_IMAGE_TAG, | |
| # `latest` (upstream's current snapshot) unless the caller pinned a tag. Every build job, Linux included, builds | |
| # from these jars (the artifact below), not from its own pull: upstream | |
| # re-pushes the snapshot tag on every deploy, so a pull per job could put | |
| # two different engine builds under one version in a single run. | |
| echo "📦 Downloading JARs from arcadedata/arcadedb:${ARCADEDB_IMAGE_TAG}..." | |
| # Create output directory | |
| mkdir -p src/arcadedb/jars | |
| # Create a temporary container and copy JARs from it | |
| CONTAINER_ID=$(docker create "arcadedata/arcadedb:${ARCADEDB_IMAGE_TAG}") | |
| docker cp $CONTAINER_ID:/home/arcadedb/lib/. src/arcadedb/jars/ | |
| docker rm $CONTAINER_ID | |
| echo "✅ Downloaded $(ls -1 src/arcadedb/jars/*.jar | wc -l) JARs" | |
| ARCADEDB_TAG=$(python3 scripts/extract_version.py src/arcadedb/jars --format=docker) | |
| echo "📌 ArcadeDB version: $ARCADEDB_TAG" | |
| # The build jobs pull arcadedata/arcadedb:<jar version> as their base image, | |
| # so a pinned tag must name that version exactly (26.10.2, not an alias). | |
| if [ "$ARCADEDB_IMAGE_TAG" != "latest" ] && [ "$ARCADEDB_IMAGE_TAG" != "$ARCADEDB_TAG" ]; then | |
| echo "❌ arcadedb-tag '$ARCADEDB_IMAGE_TAG' holds engine $ARCADEDB_TAG; pass the exact version tag" | |
| exit 1 | |
| fi | |
| echo "arcadedb-tag=$ARCADEDB_TAG" >> "$GITHUB_OUTPUT" | |
| # Unfiltered on purpose: build-native.sh applies scripts/jar_exclusions.txt | |
| # to whatever lands in the jars directory, so the list lives in one place. | |
| - name: Upload JARs as artifact | |
| uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 | |
| with: | |
| name: arcadedb-jars-examples | |
| path: src/arcadedb/jars/*.jar | |
| retention-days: 1 | |
| # Second job: build the wheel of each platform, once. The wheel is py3-none-<platform tag> (setup.py), so the file built here is | |
| # the one every Python of the test-examples job below installs: nothing is built per Python. build.sh takes a Python only for the | |
| # Linux container that builds and smoke-tests the wheel; any of them gives the same file. It also checks the size, the licences | |
| # and the tag of the wheel, as in test-jvm.yml. | |
| build: | |
| name: Build wheel for examples (${{ matrix.platform }}) | |
| runs-on: ${{ matrix.runs-on }} | |
| needs: download-jars | |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false | |
| env: | |
| BUILD_VERSION: ${{ inputs.build-version }} | |
| strategy: | |
| fail-fast: false | |
| # One leg per platform of the test matrix, from the "Select the test matrix" step of download-jars | |
| # (scripts/select_matrix.py --wheels): {"include": [{"platform": ..., "runs-on": ...}, ...]}. | |
| matrix: ${{ fromJSON(needs.download-jars.outputs.wheel-matrix) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Download JARs artifact | |
| uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 | |
| with: | |
| name: arcadedb-jars-examples | |
| path: src/arcadedb/jars | |
| - name: Set up Java (for native builds on macOS/Windows) | |
| if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/arm64' | |
| uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 | |
| with: | |
| distribution: 'corretto' | |
| java-version: '25' | |
| - name: Set up Docker Buildx (Linux only) | |
| if: matrix.platform == 'linux/amd64' || matrix.platform == 'linux/arm64' | |
| uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 | |
| - name: Set up Python | |
| uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 | |
| with: | |
| python-version: "3.12" | |
| - name: Setup UV package manager | |
| shell: bash | |
| run: | | |
| # Retry transient network failures (a connection reset failed a whole examples job on #8897); the | |
| # script is downloaded first so a retried transfer is never piped half-written into sh. | |
| installer="$(mktemp)" | |
| curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/0.12.0/install.sh | |
| sh "$installer" | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| export PATH="$HOME/.local/bin:$PATH" | |
| uv --version | |
| - name: Set UTF-8 encoding (Windows only) | |
| if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' | |
| shell: bash | |
| run: | | |
| echo "PYTHONIOENCODING=utf-8" >> $GITHUB_ENV | |
| echo "PYTHONUTF8=1" >> $GITHUB_ENV | |
| # Windows currently enabled, no symlink needed | |
| # - name: Create python3 symlink (Windows only) | |
| # if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' | |
| # shell: bash | |
| # run: | | |
| # PYTHON_DIR=$(dirname "$(which python)") | |
| # ln -s "$PYTHON_DIR/python.exe" "$PYTHON_DIR/python3.exe" || true | |
| # python3 --version | |
| - name: Build arcadedb (${{ matrix.platform }}) | |
| shell: bash | |
| env: | |
| # The engine version download-jars read from the ARCADEDB_IMAGE_TAG jars; | |
| # the Docker build takes its base image by this tag, the jars from the artifact | |
| ARCADEDB_TAG: ${{ needs.download-jars.outputs.arcadedb-tag }} | |
| run: | | |
| echo "🔨 Building arcadedb-jvm for ${{ matrix.platform }}..." | |
| # The jars download-jars fetched, on every platform, so all wheels of this | |
| # run carry the same engine build (and build.sh checks the integration jar) | |
| ./scripts/build.sh ${{ matrix.platform }} 3.12 src/arcadedb/jars | |
| # build.sh already ran this (and the size check); it is repeated here as in test-jvm.yml, so the tag shows in this job's log and | |
| # summary and a change to build.sh cannot drop it unnoticed. The wheel must be py3-none-<platform>, the file every Python installs. | |
| - name: Check the wheel tag | |
| shell: bash | |
| run: | | |
| python scripts/verify_wheel_tag.py dist/*.whl | |
| { | |
| echo "## 📦 Wheel: arcadedb-jvm (${{ matrix.platform }})" | |
| echo | |
| for w in dist/*.whl; do | |
| echo "- \`$(basename "$w")\`, $(du -m "$w" | cut -f1) MB" | |
| done | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| # One wheel per platform, for the test-examples legs of that platform. Named examples-wheel-<platform>, not wheel-<platform>: | |
| # a release runs this workflow and test-jvm.yml in one run, and release-jvm.yml publishes every wheel-* artifact. | |
| - name: Upload wheel artifact | |
| uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 | |
| with: | |
| name: examples-wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }} | |
| path: dist/*.whl | |
| if-no-files-found: error | |
| retention-days: 1 | |
| # Third job: install the wheel of the platform into each Python and run the examples. Nothing is built here: the wheel is | |
| # the file the build job made (py3-none-<platform tag>), so each Python runs the examples against the same file that is published. | |
| test-examples: | |
| name: Test Python Examples (${{ matrix.platform }}, Python ${{ matrix.python-version }}) | |
| runs-on: ${{ matrix.runs-on }} | |
| needs: [download-jars, build] | |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false | |
| env: | |
| # Pin the Hugging Face cache inside the workspace so it is cacheable with | |
| # one path on every platform (the default differs per OS). The examples | |
| # embed with sentence-transformers, and the full matrix is 16 jobs, so | |
| # uncached that is 16 independent model downloads per release and 16 | |
| # chances for a transient hub outage to fail a release. It has happened. | |
| HF_HOME: ${{ github.workspace }}/.hf-cache | |
| strategy: | |
| fail-fast: false | |
| # Chosen by the "Select the test matrix" step of download-jars from the event that | |
| # started the run (scripts/select_matrix.py holds the platform and Python lists): | |
| # small on pull requests, the full 4 platforms x 4 Pythons on every other event. | |
| # Every leg carries platform, runs-on and python-version, so the job names stay | |
| # "Test Python Examples (<platform>, Python <version>)". | |
| matrix: ${{ fromJSON(needs.download-jars.outputs.matrix) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Set up Python | |
| uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup UV package manager | |
| shell: bash | |
| run: | | |
| # Retry transient network failures (a connection reset failed a whole examples job on #8897); the | |
| # script is downloaded first so a retried transfer is never piped half-written into sh. | |
| installer="$(mktemp)" | |
| curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/0.12.0/install.sh | |
| sh "$installer" | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| export PATH="$HOME/.local/bin:$PATH" | |
| uv --version | |
| # The wheel of this platform, built once by the build job and the same file for every Python. | |
| - name: Download the wheel of ${{ matrix.platform }} | |
| uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 | |
| with: | |
| name: examples-wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }} | |
| path: test-install | |
| # Note: Java is NOT required - arcadedb has bundled JRE! | |
| # The wheel and the example dependencies live in a uv project of their own (examples-env/, not the repo's pyproject.toml, | |
| # which is a package definition and not a uv project), synced with `uv sync`. Nothing is installed into the runner's Python. | |
| # torch comes from the CPU-only PyTorch index (explicit: only torch is looked up there), which saves the CUDA download. | |
| # PyTorch is required by sentence-transformers, even on macOS. | |
| # numpy is not capped: the old macOS `numpy<2.0` was for Intel Macs, where PyTorch stops at 2.2 (built against | |
| # NumPy 1.x). That leg (darwin/amd64) is disabled; re-enabling it needs the cap back for that platform only. | |
| # On Apple Silicon the cap breaks pyarrow, which requires NumPy 2 (example 27). | |
| - name: Install ArcadeDB Python bindings and example dependencies | |
| shell: bash | |
| env: | |
| PYVER: ${{ matrix.python-version }} | |
| run: | | |
| wheels=(test-install/arcadedb_jvm-*.whl) | |
| [ "${#wheels[@]}" -eq 1 ] || { echo "expected one wheel in test-install, found ${#wheels[@]}" >&2; exit 1; } | |
| mkdir -p examples-env | |
| cat > examples-env/pyproject.toml <<TOML | |
| [project] | |
| name = "examples-env" | |
| version = "0" | |
| requires-python = ">=3.11" | |
| dependencies = [ | |
| "arcadedb-jvm", | |
| "torch", | |
| "numpy", | |
| "requests", | |
| "sentence-transformers", | |
| "pandas", | |
| "polars", | |
| "pyarrow", | |
| "tqdm", | |
| "py7zr", | |
| "lxml", | |
| ] | |
| [tool.uv.sources] | |
| arcadedb-jvm = { path = "../${wheels[0]}" } | |
| torch = { index = "pytorch-cpu" } | |
| [[tool.uv.index]] | |
| name = "pytorch-cpu" | |
| url = "https://download.pytorch.org/whl/cpu" | |
| explicit = true | |
| TOML | |
| uv sync --project examples-env --python "$PYVER" | |
| # The examples run with this interpreter directly, so a timeout kills the example itself and not a `uv run` wrapper. | |
| ex_py="$(uv run --project examples-env --no-sync python -c 'import sys; print(sys.executable)')" | |
| echo "EX_PY=$ex_py" >> "$GITHUB_ENV" | |
| "$ex_py" -c "import arcadedb; print('arcadedb', arcadedb.__version__)" | |
| # Restored before the dataset step, which is what triggers the model | |
| # download. Keyed on the model name so bumping the model invalidates it. | |
| - name: Cache Hugging Face model | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: ${{ github.workspace }}/.hf-cache | |
| key: hf-all-MiniLM-L6-v2-${{ runner.os }}-${{ runner.arch }} | |
| restore-keys: | | |
| hf-all-MiniLM-L6-v2-${{ runner.os }}- | |
| # Pre-warm with bounded retries, separately from the dataset step, so a | |
| # transient hub failure is retried here rather than surfacing mid-example | |
| # as an OSError that reads like a bindings bug. Retrying is appropriate | |
| # for this step specifically: it exercises third-party I/O, not our code. | |
| # Deliberately NOT fail-fast: if the hub is truly down the dataset step | |
| # below still runs and fails with its own message, so this cannot turn a | |
| # real breakage into a silent skip. | |
| - name: Pre-warm embedding model | |
| shell: bash | |
| continue-on-error: true | |
| run: | | |
| for attempt in 1 2 3; do | |
| if "$EX_PY" -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2'); print('model ready')"; then | |
| echo "✅ model cached on attempt $attempt" | |
| exit 0 | |
| fi | |
| echo "⚠️ model fetch failed (attempt $attempt/3)" | |
| [ "$attempt" -lt 3 ] && sleep $((attempt * 20)) | |
| done | |
| echo "::warning::could not pre-warm the embedding model after 3 attempts; the dataset step will retry" | |
| - name: Download datasets | |
| shell: bash | |
| run: | | |
| cd examples | |
| # Determine which examples are selected | |
| examples=$(ls $EXAMPLES 2>/dev/null || true) | |
| # EXIT 75 MEANS "THE DATASET HOST IS DOWN", NOT "THE BINDINGS BROKE". | |
| # | |
| # On 2026-08-29 the TLS certificate for files.grouplens.org expired and | |
| # every job here went red on BOTH this fork and upstream's main -- 11 | |
| # and 12 jobs, every OS, Python 3.10 through 3.14 -- at this step, | |
| # before a single example ran. Re-running does not help: a re-run | |
| # happens now, and the certificate is still expired. | |
| # | |
| # The cost is not the lost coverage, it is that upstream's main showed | |
| # a failing run titled with a Python bindings PR. A third party letting | |
| # a certificate lapse must not be indistinguishable from a regression | |
| # we introduced. | |
| # | |
| # So: skip the examples that need the dataset, keep running the rest, | |
| # and say plainly why. Any other non-zero exit is still a hard failure. | |
| SKIPPED_DATASETS="" | |
| fetch() { # fetch <label> <download_data.py args...> | |
| local label="$1"; shift | |
| # rc MUST be captured in the else branch. After `if cmd; then ...; fi` | |
| # with no else, the IF STATEMENT returns 0, so a `$?` read after `fi` | |
| # is the if's status and not the command's -- which silently turned | |
| # every failure into "exited 0" and would have made this skip | |
| # everything, including real ones. | |
| local rc=0 | |
| if "$EX_PY" download_data.py "$@"; then | |
| return 0 | |
| else | |
| rc=$? | |
| fi | |
| if [ "$rc" -eq 75 ]; then | |
| echo "::warning title=Dataset unavailable::$label host unreachable; skipping examples that need it" | |
| SKIPPED_DATASETS="$SKIPPED_DATASETS $label" | |
| return 0 | |
| fi | |
| echo "::error title=Dataset download failed::$label exited $rc" | |
| return "$rc" | |
| } | |
| if echo "$examples" | grep -qE '(^|[[:space:]])(04_csv_import_documents|05_csv_import_graph|06_vector_search_recommendations)\.py([[:space:]]|$)'; then | |
| echo "📥 Downloading MovieLens Small dataset..." | |
| fetch movielens-small movielens-small | |
| fi | |
| if echo "$examples" | grep -qE '(^|[[:space:]])(07_stackoverflow_tables_oltp|08_stackoverflow_tables_olap|09_stackoverflow_graph_oltp|10_stackoverflow_graph_olap|13_stackoverflow_hybrid_queries)\.py([[:space:]]|$)'; then | |
| echo "📥 Downloading Stack Overflow Tiny dataset (no vectors)..." | |
| fetch stackoverflow-tiny-novec stackoverflow-tiny --no-vectors | |
| fi | |
| if echo "$examples" | grep -qE '(^|[[:space:]])(11_vector_index_build|12_vector_search)\.py([[:space:]]|$)'; then | |
| echo "📥 Downloading Stack Overflow Tiny dataset (with vectors)..." | |
| fetch stackoverflow-tiny stackoverflow-tiny | |
| fi | |
| # HAND THE SKIP LIST TO THE NEXT STEP. | |
| # | |
| # Each step is its own shell, so SKIPPED_DATASETS dies here. Without | |
| # this the download skips politely and the example loop then runs | |
| # 04_csv_import_documents.py --dataset movielens-small against data | |
| # that was never fetched -- the job still fails, just later and with a | |
| # worse error. Caught in review on #6889. | |
| echo "SKIPPED_DATASETS=$(echo $SKIPPED_DATASETS | xargs)" >> "$GITHUB_ENV" | |
| if [ -n "$(echo $SKIPPED_DATASETS | xargs)" ]; then | |
| echo "::notice title=Datasets unavailable::skipping examples that need:$SKIPPED_DATASETS" | |
| fi | |
| - name: Install timeout command (macOS only) | |
| if: matrix.platform == 'darwin/amd64' || matrix.platform == 'darwin/arm64' | |
| shell: bash | |
| run: | | |
| # macOS doesn't have timeout command by default, use coreutils | |
| brew install coreutils | |
| - name: Set UTF-8 encoding (Windows only) | |
| if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' | |
| shell: bash | |
| run: | | |
| echo "PYTHONIOENCODING=utf-8" >> $GITHUB_ENV | |
| echo "PYTHONUTF8=1" >> $GITHUB_ENV | |
| - name: Run all examples | |
| id: run_examples | |
| shell: bash | |
| run: | | |
| cd examples | |
| echo "🚀 Running Python Examples..." | |
| echo "" | |
| # Initialize counters | |
| total=0 | |
| passed=0 | |
| failed=0 | |
| skipped=0 | |
| # Create results file | |
| results_file="example-results.txt" | |
| > $results_file | |
| # Select example files by pattern(s) in EXAMPLES (space-separated globs) | |
| examples=$(ls $EXAMPLES 2>/dev/null | sort) | |
| if [ -z "$examples" ]; then | |
| echo "❌ No example files found!" | |
| exit 1 | |
| fi | |
| # Detect timeout command (macOS uses gtimeout, Linux uses timeout) | |
| if command -v gtimeout &> /dev/null; then | |
| TIMEOUT_CMD="gtimeout" | |
| else | |
| TIMEOUT_CMD="timeout" | |
| fi | |
| # WHICH EXAMPLES NEED WHICH DATASET. Same grouping the download step | |
| # uses to decide what to fetch, kept in the same file so the two | |
| # cannot drift into disagreeing about what a dataset is for. | |
| needs_dataset() { # <example.py> -> prints the label, or nothing | |
| case "$1" in | |
| 04_csv_import_documents.py|05_csv_import_graph.py|06_vector_search_recommendations.py) | |
| echo "movielens-small" ;; | |
| 07_stackoverflow_tables_oltp.py|08_stackoverflow_tables_olap.py|09_stackoverflow_graph_oltp.py|10_stackoverflow_graph_olap.py|13_stackoverflow_hybrid_queries.py) | |
| echo "stackoverflow-tiny-novec" ;; | |
| 11_vector_index_build.py|12_vector_search.py) | |
| echo "stackoverflow-tiny" ;; | |
| esac | |
| } | |
| # Run each example | |
| for example in $examples; do | |
| total=$((total + 1)) | |
| # Skip, do not fail, when the dataset host was unreachable. Counted | |
| # as skipped and named, so a reader can tell "not run" from "passed". | |
| need=$(needs_dataset "$(basename "$example")") | |
| if [ -n "$need" ] && echo " ${SKIPPED_DATASETS:-} " | grep -q " $need "; then | |
| echo "SKIP $example - dataset '$need' unavailable (third-party host)" | |
| skipped=$((skipped + 1)) | |
| continue | |
| fi | |
| # Set example-specific parameters and timeout | |
| case "$example" in | |
| "21_graph_analytical_view_sql.py") | |
| example_args="--base-cities 1200 --stale-growth-cities 300 --sync-growth-cities 200 --regions 12 --batch-size 300 --sample-limit 5" | |
| example_name="$example (reduced scale for CI)" | |
| timeout_duration=900 # 15 minutes | |
| example_jvm_args="" | |
| ;; | |
| "22_numpy_bulk_io.py") | |
| example_args="--rows 50000 --points 100000 --vectors 5000" | |
| example_name="$example (reduced scale for CI)" | |
| timeout_duration=600 # 10 minutes | |
| example_jvm_args="" | |
| ;; | |
| "04_csv_import_documents.py") | |
| example_args="--dataset movielens-small --export" | |
| example_name="$example (movielens-small dataset with export)" | |
| timeout_duration=900 # 15 minutes | |
| example_jvm_args="-Xmx8g -Xms8g" | |
| ;; | |
| "05_csv_import_graph.py") | |
| example_args="--dataset movielens-small --method sql --import-jsonl ./exports/movielens_small_db.jsonl.tgz --export" | |
| example_name="$example (movielens-small dataset, embedded sql method, import/export)" | |
| timeout_duration=900 # 15 minutes | |
| example_jvm_args="" | |
| ;; | |
| "06_vector_search_recommendations.py") | |
| example_args="--import-jsonl ./exports/movielens_graph_small_db.jsonl.tgz" | |
| example_name="$example (vector search, import from JSONL)" | |
| timeout_duration=900 # 15 minutes | |
| example_jvm_args="" | |
| ;; | |
| "07_stackoverflow_tables_oltp.py") | |
| example_args="--dataset stackoverflow-tiny --db arcadedb_sql --threads 1 --transactions 1000 --batch-size 500 --mem-limit 2g --run-label ci07" | |
| example_name="$example (stackoverflow-tiny, arcadedb_sql, minimal oltp)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| "08_stackoverflow_tables_olap.py") | |
| example_args="--dataset stackoverflow-tiny --db arcadedb_sql --threads 1 --batch-size 500 --query-runs 1 --query-order fixed --mem-limit 2g --run-label ci08" | |
| example_name="$example (stackoverflow-tiny, arcadedb_sql, minimal olap)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| "09_stackoverflow_graph_oltp.py") | |
| example_args="--dataset stackoverflow-tiny --db arcadedb_cypher --threads 1 --transactions 1000 --batch-size 500 --mem-limit 2g --run-label ci09" | |
| example_name="$example (stackoverflow-tiny graph oltp, arcadedb cypher, minimal)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| "10_stackoverflow_graph_olap.py") | |
| example_args="--dataset stackoverflow-tiny --db arcadedb_cypher --threads 1 --batch-size 500 --query-runs 1 --query-order fixed --mem-limit 2g --run-label ci10" | |
| example_name="$example (stackoverflow-tiny graph olap, arcadedb cypher, minimal)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| "11_vector_index_build.py") | |
| example_args="--backend arcadedb_sql --dataset stackoverflow-tiny --threads 1 --mem-limit 2g --batch-size 500 --beam-width 100 --quantization NONE --run-label ci11_arcadedb_sql" | |
| example_name="$example (vector build, arcadedb_sql backend, minimal)" | |
| timeout_duration=1200 | |
| example_jvm_args="" | |
| ;; | |
| "12_vector_search.py") | |
| db_path=$(find ./my_test_databases -maxdepth 1 -type d -name 'backend=arcadedb_sql_dataset=stackoverflow-tiny_*run=ci11_arcadedb_sql' | head -n 1) | |
| if [ -z "$db_path" ]; then | |
| echo "❌ Missing build output for 12_vector_search.py. Ensure 11_vector_index_build.py ran first." | tee -a $results_file | |
| failed=$((failed + 1)) | |
| echo "" | |
| continue | |
| fi | |
| example_args="--backend arcadedb_sql --dataset stackoverflow-tiny --db-path $db_path --ef-search-values 100 --k 10 --query-limit 100 --query-runs 1 --query-order fixed --threads 1 --mem-limit 2g --run-label ci12_arcadedb_sql" | |
| example_name="$example (vector search, arcadedb_sql backend, minimal)" | |
| timeout_duration=1200 | |
| example_jvm_args="" | |
| ;; | |
| "13_stackoverflow_hybrid_queries.py") | |
| example_args="--dataset stackoverflow-tiny --batch-size 500 --encode-batch-size 64 --top-k 5 --candidate-limit 100 --min-reputation 100 --infer-sample-limit 5000 --run-label ci13" | |
| example_name="$example (hybrid pipeline, tiny + minimal limits)" | |
| timeout_duration=1800 | |
| example_jvm_args="" | |
| ;; | |
| "14_lifecycle_timing.py") | |
| example_args="--runs 1 --table-records 2000 --graph-vertices 500 --vector-records 500 --vector-dimensions 32 --query-runs 10 --jvm-heap 4g" | |
| example_name="$example (lifecycle benchmark, minimal)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| "15_import_database_vs_transactional_table_ingest.py") | |
| example_args="--rows-per-table 2000 --tables 2 --columns 6 --string-size 32 --batch-size 500 --async-parallel 1 --parallel 1 --heap-size 4g --work-dir ./my_test_databases/import_vs_txn_dummy_ci" | |
| example_name="$example (table ingest benchmark, minimal)" | |
| timeout_duration=1200 | |
| example_jvm_args="" | |
| ;; | |
| "16_import_database_vs_transactional_graph_ingest.py") | |
| example_args="--vertices 2000 --edges 4000 --vertex-int-props 3 --vertex-str-props 2 --edge-int-props 1 --edge-str-props 1 --string-size 32 --batch-size 500 --async-parallel 1 --parallel 1 --heap-size 4g --work-dir ./my_test_databases/import_vs_txn_graph_ci" | |
| example_name="$example (graph ingest benchmark, minimal)" | |
| timeout_duration=1200 | |
| example_jvm_args="" | |
| ;; | |
| "17_timeseries_end_to_end.py") | |
| example_args="--hours 2 --interval-minutes 10" | |
| example_name="$example (time-series end-to-end, minimal)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| "18_geo_predicates_wkt.py") | |
| example_args="" | |
| example_name="$example (geo predicates with WKT points and polygons)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| "19_hash_index_exact_match.py") | |
| example_args="" | |
| example_name="$example (hash index exact-match workflow)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| "20_graph_algorithms_route_planning.py") | |
| example_args="" | |
| example_name="$example (graph algorithms route planning workflow)" | |
| timeout_duration=900 | |
| example_jvm_args="" | |
| ;; | |
| *) | |
| example_args="" | |
| example_name="$example" | |
| timeout_duration=900 # 15 minutes default | |
| example_jvm_args="" | |
| ;; | |
| esac | |
| log_file="${example%.py}.log" | |
| echo "----------------------------------------" | |
| echo "📝 Running: $example_name" | |
| echo "----------------------------------------" | |
| # Run the example with appropriate parameters | |
| if [ -n "$example_jvm_args" ]; then | |
| if ARCADEDB_JVM_ARGS="$example_jvm_args" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $example_args > "$log_file" 2>&1; then | |
| exit_code=0 | |
| else | |
| exit_code=$? | |
| fi | |
| else | |
| if ARCADEDB_JVM_ARGS="" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $example_args > "$log_file" 2>&1; then | |
| exit_code=0 | |
| else | |
| exit_code=$? | |
| fi | |
| fi | |
| # ANSWERS, NOT ONLY A CLEAN EXIT (humemai/arcadedb-embedded-python#12). Example 10 once returned 0 rows for | |
| # 5 of its 10 queries on every platform and passed. After a clean run, the | |
| # same arguments on the pure-Python reference backend must give the same | |
| # row count and hash for every query. | |
| if [ $exit_code -eq 0 ] && [ "$example" = "10_stackoverflow_graph_olap.py" ]; then | |
| ref_args="${example_args/--db arcadedb_cypher/--db python_memory}" | |
| ref_args="${ref_args/--run-label ci10/--run-label ci10ref}" | |
| ref_log="${example%.py}-reference.log" | |
| if ! $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $ref_args > "$ref_log" 2>&1; then | |
| echo "❌ reference run (python_memory) of $example failed; last 20 lines:" | |
| tail -n 20 "$ref_log" | |
| exit_code=1 | |
| elif ! "$EX_PY" scripts/compare_query_hashes.py "$log_file" "$ref_log"; then | |
| echo "❌ $example: answers differ from the python_memory reference (above)" | |
| exit_code=1 | |
| fi | |
| fi | |
| if [ $exit_code -eq 0 ]; then | |
| echo "✅ PASSED: $example_name" | tee -a $results_file | |
| passed=$((passed + 1)) | |
| else | |
| if [ $exit_code -eq 124 ]; then | |
| echo "⏱️ TIMEOUT: $example_name (exceeded $((timeout_duration/60)) minutes)" | tee -a $results_file | |
| failed=$((failed + 1)) | |
| else | |
| echo "❌ FAILED: $example_name (exit code: $exit_code)" | tee -a $results_file | |
| failed=$((failed + 1)) | |
| fi | |
| # Show last 20 lines of error log | |
| echo "Last 20 lines of output:" | |
| tail -n 20 "$log_file" | |
| fi | |
| echo "" | |
| done | |
| # Print summary | |
| echo "========================================" | |
| echo "📊 EXAMPLE TEST SUMMARY" | |
| echo "========================================" | |
| echo "Total: $total" | |
| echo "Passed: $passed ✅" | |
| echo "Failed: $failed ❌" | |
| echo "Skipped: $skipped ⏭️" | |
| echo "========================================" | |
| echo "" | |
| echo "Examples pattern(s): $EXAMPLES" | |
| # Output to GitHub Actions | |
| echo "total=$total" >> $GITHUB_OUTPUT | |
| echo "passed=$passed" >> $GITHUB_OUTPUT | |
| echo "failed=$failed" >> $GITHUB_OUTPUT | |
| echo "skipped=$skipped" >> $GITHUB_OUTPUT | |
| # Show detailed results | |
| echo "Detailed Results:" | |
| cat $results_file | |
| # Save list of examples for summary (before the failure exit, so | |
| # the summary is populated even when some examples fail) | |
| echo "$examples" > examples-ran.txt | |
| # Exit with error if any failed | |
| if [ $failed -gt 0 ]; then | |
| echo "❌ Some examples failed!" | |
| exit 1 | |
| else | |
| echo "✅ All examples passed!" | |
| fi | |
| - name: Generate test summary | |
| if: always() | |
| shell: bash | |
| run: | | |
| cd examples | |
| echo "## 🎮 Python Examples Test Results (${{ matrix.platform }})" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| total="${{ steps.run_examples.outputs.total || '0' }}" | |
| passed="${{ steps.run_examples.outputs.passed || '0' }}" | |
| failed="${{ steps.run_examples.outputs.failed || '0' }}" | |
| if [ "${{ steps.run_examples.outcome }}" = "success" ]; then | |
| echo "✅ **Status**: ALL EXAMPLES PASSED ($passed/$total)" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ **Status**: SOME EXAMPLES FAILED ($passed/$total passed)" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|------:|" >> $GITHUB_STEP_SUMMARY | |
| echo "| 📝 Total | $total |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ✅ Passed | $passed |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ❌ Failed | $failed |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ⏭️ Skipped | ${{ steps.run_examples.outputs.skipped || '0' }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Add detailed results if available | |
| if [ -f example-results.txt ]; then | |
| echo "### Detailed Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| cat example-results.txt >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Examples Tested" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Pattern(s)**: $EXAMPLES" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f examples-ran.txt ]; then | |
| while IFS= read -r example; do | |
| [ -n "$example" ] && echo "- **$example**" >> $GITHUB_STEP_SUMMARY | |
| done < examples-ran.txt | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| - name: Upload example logs | |
| if: always() | |
| uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 | |
| with: | |
| name: example-logs-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }}-py${{ matrix.python-version }} | |
| path: | | |
| examples/*.log | |
| examples/example-results.txt | |
| retention-days: 7 | |
| - name: Upload example databases | |
| if: failure() | |
| uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 | |
| with: | |
| name: example-databases-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }}-py${{ matrix.python-version }} | |
| path: examples/my_test_databases/ | |
| retention-days: 3 | |
| # Summary job that checks all platforms | |
| test-examples-summary: | |
| name: Examples Test Summary | |
| needs: [build, test-examples] | |
| runs-on: ubuntu-latest | |
| # Skipped (not red) when the matrix was skipped for a draft PR. | |
| if: always() && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) | |
| steps: | |
| - name: Check test results | |
| shell: bash | |
| run: | | |
| echo "## 🎯 Overall Examples Test Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ needs.build.result }}" = "success" ] && [ "${{ needs.test-examples.result }}" = "success" ]; then | |
| echo "✅ **All platforms passed example testing!**" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "All examples ran successfully across all enabled platforms, against one wheel built for each platform." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Platforms tested**: linux/amd64, linux/arm64, darwin/arm64, windows/amd64" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ **Some platforms failed example testing**" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Please check the individual platform jobs for details." >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| fi |