From 31cba7c23c8d97c20f6271510fae343cd43d7ae0 Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Mon, 28 Sep 2026 00:27:54 +0800 Subject: [PATCH 1/2] [python] Keep Python upsert streaming checks on the Python writer --- paimon-python/pypaimon/tests/table_upsert_by_key_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py index 68d26e86e8db..8833b1fc8855 100644 --- a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py +++ b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py @@ -22,6 +22,7 @@ import pyarrow as pa import pyarrow.parquet as pq +import pytest from pypaimon.read.table_read import TableRead from pypaimon.table.special_fields import SpecialFields @@ -140,6 +141,7 @@ def test_invalid_row_group_size_fails_before_opening_output(self): _RowIdUpdateFileWriter(table, (), ['id']) output_stream.assert_not_called() + @pytest.mark.python_write @mock.patch.object(_RowIdUpdateFileWriter, '_ROW_GROUP_MAX_ROWS', 2) def test_partial_upsert_streams_original_file_group(self): schema = pa.schema([ From 96560cc94752cf2c04303d8624ee6c128a01c981 Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Sun, 27 Sep 2026 20:19:45 +0800 Subject: [PATCH 2/2] [python] Centralize PyPaimon packaging configuration --- .github/workflows/ci-python.yml | 153 +++----- .github/workflows/release-python-publish.yml | 2 +- .github/workflows/release-python.yml | 16 +- .github/workflows/release.yml | 2 +- .gitignore | 1 + docs/docs/project/creating-a-release.md | 9 +- docs/docs/project/releases.md | 2 +- .../project/verifying-a-release-candidate.md | 14 +- docs/docs/pypaimon/installation.md | 18 +- paimon-python/MANIFEST.in | 2 +- paimon-python/README.md | 37 +- paimon-python/conftest.py | 3 +- paimon-python/dev/check-licensing.sh | 2 +- paimon-python/dev/install.py | 71 ++++ paimon-python/dev/lint-python.sh | 6 +- paimon-python/dev/requirements-dev.txt | 41 --- paimon-python/dev/requirements.txt | 44 --- paimon-python/dev/run_mixed_tests.sh | 25 +- paimon-python/dev/test_packaging.py | 212 +++++++++++ paimon-python/pypaimon/_version.py | 19 + paimon-python/pypaimon/build_info.py | 48 ++- .../pypaimon/tests/build_info_test.py | 7 +- .../ray_data_evolution_merge_into_test.py | 3 +- .../tests/rest/rest_read_write_test.py | 17 +- paimon-python/pyproject.toml | 211 +++++++++++ paimon-python/setup.py | 341 +++--------------- 26 files changed, 719 insertions(+), 587 deletions(-) create mode 100644 paimon-python/dev/install.py delete mode 100644 paimon-python/dev/requirements-dev.txt delete mode 100644 paimon-python/dev/requirements.txt create mode 100644 paimon-python/dev/test_packaging.py create mode 100644 paimon-python/pypaimon/_version.py create mode 100644 paimon-python/pyproject.toml diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index fddb1a4a6d17..952a1173a559 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -27,7 +27,6 @@ permissions: env: JDK_VERSION: 8 MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true - LUMINA_DATA_VERSION: 0.1.0 jobs: @@ -95,19 +94,26 @@ jobs: - name: Verify Python version run: python --version - - name: Verify requirements.txt dependencies can be installed + - name: Verify Python distributions + run: | + # Build tools and legacy Flake8 require incompatible importlib-metadata versions. + python -m venv "${RUNNER_TEMP}/paimon-packaging" + "${RUNNER_TEMP}/paimon-packaging/bin/python" -m pip install --upgrade pip setuptools wheel packaging tomli build + "${RUNNER_TEMP}/paimon-packaging/bin/python" paimon-python/dev/test_packaging.py + + - name: Verify core package dependencies can be installed shell: bash run: | cd paimon-python python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { - echo "ERROR: Failed to resolve dependencies from dev/requirements.txt" + python -m pip install . --target "$TEMP_DIR" || { + echo "ERROR: Failed to install core PyPaimon dependencies" rm -rf "$TEMP_DIR" exit 1 } rm -rf "$TEMP_DIR" - echo "✓ dev/requirements.txt can be resolved on this Python" + echo "✓ Core PyPaimon dependencies resolve on this Python" - name: Build Java run: | @@ -118,57 +124,20 @@ jobs: - name: Install Python dependencies shell: bash run: | - df -h - if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then - python -m pip install --upgrade pip==21.3.1 - python --version - python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 datasketches==4.1.0 2>&1 >/dev/null - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ - elif [[ "${{ matrix.python-version }}" == "3.7" ]]; then - # 3.7 installs the version-pinned set declared for 3.7 in dev/requirements.txt. - # ray/lance/daft/torch have no 3.7 wheels; those tests are importorskip-ed (as on 3.6.15). - python -m pip install --upgrade pip - python -m pip install -r paimon-python/dev/requirements.txt - python -m pip install flake8==4.0.1 'pytest~=7.0' py4j==0.10.9.9 parameterized==0.9.0 datasketches==4.1.0 - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ + python -m pip install --upgrade pip + if python -c "import sys; sys.exit(0 if sys.version_info < (3, 8) else 1)"; then + # Old pip has no --group; read the same TOML with the compatibility bridge. + python -m pip install tomli + python paimon-python/dev/install.py legacy else - python -m pip install --upgrade pip - pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 'isal>=1.8,<2' zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 pylance==0.39.0 cramjam pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' 'datafusion>=54,<55' datasketches - if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"; then - python -m pip install pyarrow==24.0.0 numpy==2.4.6 pandas==2.3.3 flake8==7.1.2 - else - python -m pip install pyarrow==16.0.0 numpy==1.24.3 pandas==2.0.3 flake8==4.0.1 - fi - python -m pip install 'h5py>=3,<4' Pillow - python -c "import h5py; print('h5py', h5py.__version__)" - if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)"; then - python -m pip install './paimon-python[rosbag]' - python -c "import rosbags; print('rosbags installed')" - fi + python -m pip install torch --index-url https://download.pytorch.org/whl/cpu + dependency_group=dev if [[ "${{ matrix.python-version }}" == "3.10" ]]; then - python -m pip install './paimon-python[lerobot,video]' - python -c "import datasets, lerobot; print('datasets', datasets.__version__, 'lerobot', lerobot.__version__)" - fi - - # Required by SQL tests; native planner tests run against main below. - python -m pip install pypaimon-rust - if [[ "${{ matrix.python-version }}" == "3.11" ]]; then - # Run the RoboMIND pipeline tests with synthetic local HDF5 data. - python -m pip install "./paimon-python[sql]" - fi - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ - if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)"; then - # Vortex requires pyarrow >= 17, while Pypaimon requires pyarrow < 20 and != 19. - python -m pip install vortex-data==0.70.0 pyarrow==18.1.0 - fi - python -m pip install 'paimon-mosaic>=0.1.0' - if [[ "${{ matrix.python-version }}" == "3.11" ]]; then - python -m pip check + dependency_group=torch fi + python -m pip install -e ./paimon-python --group "./paimon-python/pyproject.toml:${dependency_group}" fi - python -m pip install 'oss2>=2.18,<3' - df -h + python -m pip check - name: Run lint-python.sh shell: bash @@ -204,13 +173,7 @@ jobs: - name: Install Python test dependencies run: | python -m pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install "./paimon-python[sql,rosbag]" \ - readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 \ - ray==2.54.0 fastavro==1.11.1 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 \ - pylance==0.39.0 'pytest~=7.0' py4j==0.10.9.9 parameterized==0.9.0 \ - 'daft>=0.7.6' datasketches pyarrow==18.1.0 numpy==1.24.3 pandas==2.0.3 \ - 'h5py>=3,<4' Pillow 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' \ - vortex-data==0.70.0 'paimon-mosaic>=0.1.0' + python -m pip install -e ./paimon-python --group ./paimon-python/pyproject.toml:dev python -m pip check - name: Verify native wheel provenance @@ -262,21 +225,20 @@ jobs: run: python --version - name: Install Python dependencies - shell: bash run: | - python -m pip install --upgrade pip - pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 datasketches 'datasets>=4,<4.1' - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ + python -m pip install --upgrade pip + python -m pip install torch --index-url https://download.pytorch.org/whl/cpu + python -m pip install -e ./paimon-python --group ./paimon-python/pyproject.toml:torch + python -m pip check - name: Run lint-python.sh shell: bash run: | chmod +x paimon-python/dev/lint-python.sh ./paimon-python/dev/lint-python.sh -i pytest_torch - # One job: check dev/requirements.txt on each Python version in sequence, then Ray version test on 3.10. + # Check core installation across Python versions, then basic Ray compatibility on 3.10. compatibility: - name: Requirements and Ray compatibility + name: Package and Ray compatibility timeout-minutes: 60 runs-on: ubuntu-latest steps: @@ -289,71 +251,71 @@ jobs: build-essential git curl cmake \ && sudo rm -rf /var/lib/apt/lists/* - - name: Verify dev/requirements.txt on Python 3.8 + - name: Verify core installation on Python 3.8 uses: actions/setup-python@v5 with: python-version: '3.8' - - name: Resolve requirements (3.8) + - name: Install core package (3.8) run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.8" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.8" - - name: Verify dev/requirements.txt on Python 3.9 + - name: Verify core installation on Python 3.9 uses: actions/setup-python@v5 with: python-version: '3.9' - - name: Resolve requirements (3.9) + - name: Install core package (3.9) run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.9" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.9" - - name: Verify dev/requirements.txt on Python 3.10 + - name: Verify core installation on Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' - - name: Resolve requirements (3.10) + - name: Install core package (3.10) run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.10" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.10" - - name: Verify dev/requirements.txt on Python 3.11 + - name: Verify core installation on Python 3.11 uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Resolve requirements (3.11) + - name: Install core package (3.11) run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.11" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.11" - - name: Verify dev/requirements.txt on Python 3.12 + - name: Verify core installation on Python 3.12 uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Resolve requirements (3.12) + - name: Install core package (3.12) run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.12" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.12" - - name: Verify dev/requirements.txt on Python 3.13 + - name: Verify core installation on Python 3.13 uses: actions/setup-python@v5 with: python-version: '3.13' - - name: Resolve requirements (3.13) + - name: Install core package (3.13) run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.13" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.13" - name: Setup Python 3.10 for Ray test uses: actions/setup-python@v5 @@ -362,13 +324,10 @@ jobs: - name: Install base Python dependencies run: | python -m pip install --upgrade pip - pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install --no-cache-dir \ - pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 \ - fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 \ - numpy==1.24.3 pandas==2.0.3 cramjam pytest~=7.0 py4j==0.10.9.9 requests \ - parameterized==0.9.0 packaging datasketches - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ + python -m pip install torch --index-url https://download.pytorch.org/whl/cpu + # Use only test tools here: the dev group's Ray >=2.50 floor is for full-feature tests. + python -m pip install './paimon-python[ray,oss,lumina,vindex,full-text,theta-sketch]' \ + --group ./paimon-python/pyproject.toml:test - name: Test Ray version compatibility run: | cd paimon-python diff --git a/.github/workflows/release-python-publish.yml b/.github/workflows/release-python-publish.yml index a153ff2d4ce4..bb0fec4b70fc 100644 --- a/.github/workflows/release-python-publish.yml +++ b/.github/workflows/release-python-publish.yml @@ -61,7 +61,7 @@ jobs: source_version="$( sed -n 's/^VERSION = "\(.*\)"/\1/p' \ - paimon-python/setup.py + paimon-python/pypaimon/_version.py )" release_version="${{ inputs.release_version }}" if [[ "${source_version}" != "${release_version}" ]]; then diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index c8d3ef32e2d7..f0bab24cbf44 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -52,7 +52,7 @@ jobs: with: python-version: "3.11" cache: pip - cache-dependency-path: paimon-python/dev/requirements.txt + cache-dependency-path: paimon-python/pyproject.toml - name: Set package version id: package_version @@ -62,7 +62,7 @@ jobs: source_version="$( sed -n 's/^VERSION = "\(.*\)"/\1/p' \ - paimon-python/setup.py + paimon-python/pypaimon/_version.py )" release_version="${{ inputs.release_version }}" if [[ "${source_version}" != "${release_version}" ]]; then @@ -74,19 +74,18 @@ jobs: if [[ "${GITHUB_REF}" == refs/tags/*-rc* ]]; then rc_number="${GITHUB_REF_NAME##*-rc}" package_version="${release_version}rc${rc_number}" - sed -i \ - "s/^VERSION = \".*\"/VERSION = \"${package_version}\"/" \ - paimon-python/setup.py fi echo "version=${package_version}" >> "${GITHUB_OUTPUT}" + echo "PYPAIMON_BUILD_VERSION=${package_version}" >> "${GITHUB_ENV}" echo "Building PyPaimon ${package_version}" - name: Build distributions working-directory: paimon-python run: | - python -m pip install --upgrade pip setuptools wheel twine - python setup.py sdist bdist_wheel + python -m pip install --upgrade pip + python -m pip install --group build + python -m build python -m twine check dist/* - name: Verify distribution contents @@ -111,7 +110,8 @@ jobs: with tarfile.open(sdist, "r:gz") as archive: names = archive.getnames() prefix = "pypaimon-{}/".format(expected) - for required in ("LICENSE", "NOTICE", "README.md", "setup.py"): + for required in ("LICENSE", "NOTICE", "README.md", "setup.py", "pyproject.toml", + "pypaimon/_version.py", "pypaimon/_full_version"): if prefix + required not in names: raise SystemExit("Missing {} from sdist".format(required)) sample_data = "pypaimon/sample/data/data.jsonl" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index caaf86a89e4d..2316ae69684d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,7 +81,7 @@ jobs: python_release_version="$( sed -n 's/^VERSION = "\(.*\)"/\1/p' \ - paimon-python/setup.py + paimon-python/pypaimon/_version.py )" if [[ "${python_release_version}" != "${release_version}" ]]; then echo "PyPaimon version ${python_release_version:-missing} does not match Paimon ${release_version}" >&2 diff --git a/.gitignore b/.gitignore index 54381f82eb3e..9f1331ac3d61 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ metastore_db/ ### Python ### *.pyc +paimon-python/build/ paimon-python/dist/ paimon-python/*.egg-info/ paimon-python/dev/log diff --git a/docs/docs/project/creating-a-release.md b/docs/docs/project/creating-a-release.md index c8c15c41e252..f42e2060c548 100644 --- a/docs/docs/project/creating-a-release.md +++ b/docs/docs/project/creating-a-release.md @@ -169,7 +169,7 @@ NEW_VERSION="${PAIMON_VERSION}" \ ./tools/releasing/update_branch_version.sh ``` -Set `VERSION` in `paimon-python/setup.py` to the final +Set `VERSION` in `paimon-python/pypaimon/_version.py` to the final `PAIMON_VERSION`, without `.dev` or an RC suffix. The release workflow derives the TestPyPI version by appending `rcRC_NUMBER`; the source candidate keeps the final version. @@ -180,9 +180,9 @@ candidate contains no snapshot or development version: ```shell mvn -q -DforceStdout help:evaluate -Dexpression=project.version -python3 paimon-python/setup.py --version +python3 -c "import runpy; print(runpy.run_path('paimon-python/pypaimon/_version.py')['VERSION'])" rg --glob 'pom.xml' "${PAIMON_VERSION}-SNAPSHOT" -rg '^VERSION = ".*\.dev' paimon-python/setup.py +rg '^VERSION = ".*\.dev' paimon-python/pypaimon/_version.py git status --short ``` @@ -316,7 +316,8 @@ RELEASE_VERSION="${PAIMON_VERSION}" \ ./tools/releasing/create_source_release.sh cd paimon-python -python3 setup.py sdist +python3 -m pip install --group build +python3 -m build --sdist gpg --armor --detach-sig \ "dist/pypaimon-${PAIMON_VERSION}.tar.gz" diff --git a/docs/docs/project/releases.md b/docs/docs/project/releases.md index 3e23b79fb72a..5e72367d3f75 100644 --- a/docs/docs/project/releases.md +++ b/docs/docs/project/releases.md @@ -65,7 +65,7 @@ PyPI package is built from the approved commit with the final version. ## Release model The combined Paimon and PyPaimon release uses one shared version number. The -Maven project version and `paimon-python/setup.py` version must be equal. +Maven project version and `paimon-python/pypaimon/_version.py` version must be equal. | Deliverable | Candidate | Published location | | --- | --- | --- | diff --git a/docs/docs/project/verifying-a-release-candidate.md b/docs/docs/project/verifying-a-release-candidate.md index f3cd609281b8..53c4d5455ed9 100644 --- a/docs/docs/project/verifying-a-release-candidate.md +++ b/docs/docs/project/verifying-a-release-candidate.md @@ -189,7 +189,7 @@ Check at least the following: - All bundled third-party material is compatible with the Apache License 2.0 and is recorded in `LICENSE` or `NOTICE` where required. - Maven POMs use `PAIMON_VERSION` without `-SNAPSHOT`. -- `paimon-python/setup.py` and PyPaimon package metadata use +- `paimon-python/pypaimon/_version.py` and PyPaimon package metadata use `PAIMON_VERSION` without `.dev`. Extract the Paimon candidate: @@ -324,10 +324,11 @@ First build the official PyPaimon source candidate in an isolated environment: ```shell python3 -m venv pypaimon-rc-venv . pypaimon-rc-venv/bin/activate -python -m pip install --upgrade pip setuptools wheel +python -m pip install --upgrade pip cd "pypaimon-${PAIMON_VERSION}" -python setup.py sdist bdist_wheel +python -m pip install --group build +python -m build python -m pip install "dist/pypaimon-${PAIMON_VERSION}"*.whl python -c "import pypaimon; print('PyPaimon import OK')" cd .. @@ -349,8 +350,7 @@ than assuming that a matrix from another release applies: ```shell ( cd "paimon-${PAIMON_VERSION}/paimon-python" - python -m pip install -r dev/requirements.txt - python -m pip install -r dev/requirements-dev.txt + python -m pip install -e . --group dev python -m pytest pypaimon/tests -v ) ``` @@ -358,6 +358,10 @@ than assuming that a matrix from another release applies: Some optional integration tests need additional services or packages. Record which tests ran, skipped, or failed. +For Python 3.6/3.7, install a compatible `tomli`, run `python dev/install.py legacy`, +and use `bash dev/lint-python.sh -i pytest` to select the supported core test subset. +The complete dev group is intended for a modern Python environment. + ## Verify the TestPyPI candidate Install the exact RC version in a new environment: diff --git a/docs/docs/pypaimon/installation.md b/docs/docs/pypaimon/installation.md index a4efd18b6ba7..d0a664758341 100644 --- a/docs/docs/pypaimon/installation.md +++ b/docs/docs/pypaimon/installation.md @@ -90,22 +90,24 @@ Python module with: python -m pip install ./paimon-python ``` -For local Python development, use an editable installation: +For local Python development, use pip with dependency-group support to install +the editable project and development tools together: ```shell -python -m pip install -e './paimon-python' +cd paimon-python +python -m pip install -e . --group dev ``` -To produce a source archive: +To produce a source archive and wheel, run from `paimon-python/`: ```shell -cd paimon-python -python setup.py sdist +python -m pip install --group build +python -m build ``` -Archives are written to `paimon-python/dist/`. Development builds can produce -both a base development archive and a dated archive; select one archive when -installing instead of passing `dist/*.tar.gz` to pip. +The source archive and wheel are written to `dist/` with the same version. +Development versions append the Git commit date, for example `2.2.dev20260927`. +Install the wheel with `python -m pip install dist/*.whl`. ## Verify the installation diff --git a/paimon-python/MANIFEST.in b/paimon-python/MANIFEST.in index d644332dce82..35a33f1e5f03 100644 --- a/paimon-python/MANIFEST.in +++ b/paimon-python/MANIFEST.in @@ -17,7 +17,7 @@ ################################################################################ include LICENSE include NOTICE -include dev/requirements.txt +include pyproject.toml include README.md recursive-include pypaimon *.py recursive-include pypaimon/sample/data *.jsonl diff --git a/paimon-python/README.md b/paimon-python/README.md index f1654fa0e796..6489241823b0 100644 --- a/paimon-python/README.md +++ b/paimon-python/README.md @@ -10,10 +10,25 @@ This PyPi package contains the Python APIs for using Paimon. Pypaimon requires Python 3.6+. -# Dependencies +# Build + +Run from `paimon-python/` with pip supporting dependency groups. + +Normal build: + +```shell +python -m pip install --group build +python -m build +``` -The core dependencies are listed in `dev/requirements.txt`. -The development dependencies are listed in `dev/requirements-dev.txt`. +Development build (editable installation with development dependencies): + +```shell +python -m pip install -e . --group dev +python -m build +``` + +Both produce a source archive and wheel in `dist/`. # OSS metadata commits @@ -89,22 +104,6 @@ All concurrent writers must use conditional creation. Older Python clients or other clients that overwrite snapshot objects can still overwrite a successful commit. This change does not add conditional writes for other object stores. -# Build - -You can build the source package by executing the following command: - -```commandline -python3 setup.py sdist -``` - -The package is under `dist/`. Then you can install the package by executing the following command: - -```commandline -pip3 install dist/*.tar.gz -``` - -The command will install the package and core dependencies to your local Python environment. - # Row ID column updates For a batch update of selected columns in a data-evolution table with row diff --git a/paimon-python/conftest.py b/paimon-python/conftest.py index ef2e3b5ec1e3..7aa11d1fc6a4 100644 --- a/paimon-python/conftest.py +++ b/paimon-python/conftest.py @@ -198,7 +198,8 @@ def enable_native_backends(request, monkeypatch): python_read = request.node.get_closest_marker("python_read") is not None python_write = request.node.get_closest_marker("python_write") is not None python_commit = request.node.get_closest_marker("python_commit") is not None - native_plan_test = request.path.name in ( + # request.path requires pytest 7; Python 3.6 uses pytest 6. + native_plan_test = os.path.basename(request.node.location[0]) in ( "native_plan_test.py", "native_plan_integration_test.py", "native_plan_capabilities_test.py") force_plan = _native_plan_enabled() and not python_plan and not native_plan_test diff --git a/paimon-python/dev/check-licensing.sh b/paimon-python/dev/check-licensing.sh index c867a0d7b6e4..19bb7f013bfc 100755 --- a/paimon-python/dev/check-licensing.sh +++ b/paimon-python/dev/check-licensing.sh @@ -48,7 +48,7 @@ if [ -z "${SOURCE_PACKAGE}" ]; then cd ${PROJECT_ROOT} # Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root - if [ ! -f ${PROJECT_ROOT}/setup.py ]; then + if [ ! -f ${PROJECT_ROOT}/pyproject.toml ]; then echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory." exit 1 fi diff --git a/paimon-python/dev/install.py b/paimon-python/dev/install.py new file mode 100644 index 000000000000..d1100606ceca --- /dev/null +++ b/paimon-python/dev/install.py @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Install TOML dependency groups on old Python/pip without --group support. + +Modern development environments use ``pip install -e . --group dev`` directly. +This bridge reads the same configuration for the Python 3.6/3.7 CI lanes. +""" +import argparse +import os +import re +import subprocess +import sys + +try: + import tomllib +except ImportError: + import tomli as tomllib + + +def _normalize(name): + return re.sub(r"[-_.]+", "-", name).lower() + + +def resolve_group(groups, name, parents=()): + name = _normalize(name) + if name in parents: + raise ValueError("Cyclic dependency group: {}".format(name)) + requirements = [] + for item in groups[name]: + if isinstance(item, str): + requirements.append(item) + elif isinstance(item, dict) and set(item) == {"include-group"}: + requirements.extend(resolve_group( + groups, item["include-group"], parents + (name,))) + else: + raise ValueError("Invalid dependency group item: {!r}".format(item)) + return requirements + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("group") + args = parser.parse_args() + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + with open(os.path.join(root, "pyproject.toml"), "rb") as project_file: + raw_groups = tomllib.load(project_file)["dependency-groups"] + groups = {_normalize(name): value for name, value in raw_groups.items()} + if len(groups) != len(raw_groups): + raise ValueError("Duplicate normalized dependency group names") + subprocess.check_call( + [sys.executable, "-m", "pip", "install", root] + + resolve_group(groups, args.group)) + + +if __name__ == "__main__": + main() diff --git a/paimon-python/dev/lint-python.sh b/paimon-python/dev/lint-python.sh index 1795379f1758..86d77a6782b7 100755 --- a/paimon-python/dev/lint-python.sh +++ b/paimon-python/dev/lint-python.sh @@ -152,7 +152,7 @@ function license_check() { # Flake8 check function flake8_check() { - local PRUNE_PATHS="\( -path ./dev -o -path ./.tox -o -path ./.venv" + local PRUNE_PATHS="\( -path ./dev -o -path ./.tox -o -path ./.venv -o -path ./build -o -path ./dist" if python -c "import sys; sys.exit(0 if sys.version_info < (3, 10) else 1)" 2>/dev/null; then PRUNE_PATHS="$PRUNE_PATHS -o -path ./pypaimon/daft -o -path ./pypaimon/tests/daft" fi @@ -364,9 +364,5 @@ done # collect checks according to the options collect_checks -if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)"; then - python -m pip install 'paimon-ftindex==0.1.0' || exit 1 -fi - # run checks check_stage diff --git a/paimon-python/dev/requirements-dev.txt b/paimon-python/dev/requirements-dev.txt deleted file mode 100644 index 711b0ee62a10..000000000000 --- a/paimon-python/dev/requirements-dev.txt +++ /dev/null @@ -1,41 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -# Core dependencies for pypaimon are in requirements.txt - -# Test dependencies for pypaimon are as follows -duckdb==1.3.2 -flake8==4.0.1 -oss2>=2.18,<3 -pytest~=7.0 -# HDF5 ingestion tests run in the supported h5py wheel lanes (Python 3.8+). -h5py>=3,<4; python_version >= "3.8" -# merge_into needs Dataset.join (added in Ray 2.50). Python 3.8 has no 2.50 wheel. -ray>=2.10.0; python_version < "3.9" -ray>=2.50.0; python_version >= "3.9" -requests -parameterized -# Vortex 0.71.0 regresses native predicate pushdown on single-row files. -vortex-data==0.70.0; python_version >= "3.11" -# merge_into condition expressions (optional, for condition tests) -datafusion>=52; python_version >= "3.10" -# Lumina vector search (optional, for lumina index tests) -lumina-data>=0.1.0 -# paimon-vindex vector search (optional, for vindex index tests) -paimon-vindex==0.5.0; python_version >= "3.9" -# paimon-ftindex full-text search (optional, for full-text index tests) -paimon-ftindex==0.1.0; python_version >= "3.8" diff --git a/paimon-python/dev/requirements.txt b/paimon-python/dev/requirements.txt deleted file mode 100644 index dc9b4b4e911a..000000000000 --- a/paimon-python/dev/requirements.txt +++ /dev/null @@ -1,44 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -cachetools>=4.2,<6; python_version=="3.6" -cachetools>=5,<6; python_version>"3.6" -dataclasses>=0.8; python_version < "3.7" -fastavro>=1.4,<2 -isal>=1.8,<2; python_version >= "3.9" and (platform_machine == "x86_64" or platform_machine == "AMD64" or platform_machine == "aarch64" or platform_machine == "arm64") -fsspec>=2021.10,<2026; python_version<"3.8" -fsspec>=2023,<2026; python_version>="3.8" -packaging>=21,<26 -pandas>=1.1,<2; python_version < "3.7" -pandas>=1.3,<3; python_version >= "3.7" and python_version < "3.9" -pandas>=1.5,<3; python_version >= "3.9" -polars>=0.9,<1; python_version<"3.8" -polars>=1,<2; python_version=="3.8" -polars>=1.32,<2; python_version>="3.9" -pyarrow>=6,<7; python_version < "3.7" -pyarrow>=7,<13; python_version >= "3.7" and python_version < "3.8" -pyarrow>=16,!=19.0.0,<20; python_version >= "3.8" -pyroaring<=0.3.3; python_version < "3.7" -pyroaring<=0.4.5; python_version == "3.7" -pyroaring>=1.0.0; python_version >= "3.8" -readerwriterlock>=1,<2 -requests>=2.21.0,<3 -urllib3>=1.26,<3 -zstandard>=0.19,<1 -backports.zstd>=1.0.0,<1.4.0; python_version >= "3.9" and python_version < "3.14" -cramjam>=1.3.0,<3; python_version>="3.7" -pyyaml>=5.4,<7 diff --git a/paimon-python/dev/run_mixed_tests.sh b/paimon-python/dev/run_mixed_tests.sh index fc84202a0d54..84a54e21dbae 100755 --- a/paimon-python/dev/run_mixed_tests.sh +++ b/paimon-python/dev/run_mixed_tests.sh @@ -660,29 +660,8 @@ ensure_paimon_vindex() { return 0 fi - echo "Installing Python paimon-vindex dependency..." - if python -m pip install 'paimon-vindex==0.5.0'; then - return 0 - fi - - echo -e "${YELLOW}Direct pip install failed; installing paimon-vindex into a temporary target directory...${NC}" - local target_dir="${TMPDIR:-/tmp}/paimon-vindex-site" - rm -rf "$target_dir" - if python -m pip install --target "$target_dir" 'paimon-vindex==0.5.0'; then - export PYTHONPATH="$target_dir:${PYTHONPATH:-}" - return 0 - fi - - if python -c "import numpy" >/dev/null 2>&1; then - echo -e "${YELLOW}Dependency install failed but numpy is already available; retrying paimon-vindex without dependencies...${NC}" - rm -rf "$target_dir" - if python -m pip install --target "$target_dir" --no-deps 'paimon-vindex==0.5.0'; then - export PYTHONPATH="$target_dir:${PYTHONPATH:-}" - return 0 - fi - fi - - echo -e "${RED}✗ Failed to install paimon-vindex${NC}" + echo -e "${RED}Missing paimon-vindex; install the Python dev dependency group first.${NC}" + echo "From paimon-python: python -m pip install -e . --group dev" return 1 } diff --git a/paimon-python/dev/test_packaging.py b/paimon-python/dev/test_packaging.py new file mode 100644 index 000000000000..e9d0e5ea52d5 --- /dev/null +++ b/paimon-python/dev/test_packaging.py @@ -0,0 +1,212 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Exercise actual distributions without importing runtime/native dependencies. + +Run with ``python dev/test_packaging.py`` after installing the build tools. +The same suite runs on Python 3.6 and modern Python in CI. +""" +import email +import os +from pathlib import Path +import runpy +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +import zipfile + +from packaging.markers import default_environment +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +try: + import tomllib +except ImportError: + import tomli as tomllib + + +def command(args, cwd, env=None): + result = subprocess.run(args, cwd=str(cwd), env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode: + raise AssertionError(result.stdout.decode("utf-8", errors="replace")[-8000:]) + return result.stdout.decode("utf-8") + + +def wheel_contents(path): + with zipfile.ZipFile(str(path)) as archive: + names = archive.namelist() + metadata = email.message_from_bytes(archive.read(next( + name for name in names if name.endswith(".dist-info/METADATA")))) + full_version = archive.read("pypaimon/_full_version").decode().strip() + return names, metadata, full_version + + +class PackagingTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp = tempfile.TemporaryDirectory(prefix="pypaimon-packaging-") + cls.root = Path(cls.temp.name) + cls.source = cls.root / "source" + project = Path(__file__).resolve().parent.parent + shutil.copytree(str(project), str(cls.source), ignore=shutil.ignore_patterns( + "build", "dist", "*.egg-info", "__pycache__", ".pytest_cache", ".venv")) + cls.env = os.environ.copy() + for key in ("GIT_DIR", "GIT_WORK_TREE", "PYPAIMON_BUILD_VERSION"): + cls.env.pop(key, None) + cls.env.update(GIT_AUTHOR_DATE="2020-01-02T03:04:05+00:00", + GIT_COMMITTER_DATE="2020-01-02T03:04:05+00:00") + cls.commit = cls.init_git(cls.source) + base = runpy.run_path(str(cls.source / "pypaimon/_version.py"))["VERSION"] + cls.version = str(Version(base + "20200102" if base.endswith(".dev") else base)) + with (cls.source / "pyproject.toml").open("rb") as config: + cls.config = tomllib.load(config) + cls.dist = cls.root / "custom-output" + cls.build(cls.source, cls.dist) + cls.wheel = next(cls.dist.glob("*.whl")) + cls.sdist = next(cls.dist.glob("*.tar.gz")) + + @classmethod + def tearDownClass(cls): + cls.temp.cleanup() + + @classmethod + def init_git(cls, root): + for args in (["init", "-q"], ["config", "user.name", "Packaging Test"], + ["config", "user.email", "test@example.com"], ["add", "."], + ["commit", "-qm", "test source"]): + command(["git"] + args, root, cls.env) + return command(["git", "rev-parse", "HEAD"], root, cls.env).strip() + + @classmethod + def build(cls, source, output, extra=(), env=None): + command([sys.executable, "-m", "build", "--no-isolation", "--outdir", + str(output)] + list(extra), source, env or cls.env) + + def test_sdist_and_wheel_share_version_and_commit(self): + self.assertEqual(2, len(list(self.dist.iterdir()))) + self.assertEqual("pypaimon-{}.tar.gz".format(self.version), self.sdist.name) + self.assertEqual("pypaimon-{}-py3-none-any.whl".format(self.version), self.wheel.name) + _, metadata, full_version = wheel_contents(self.wheel) + self.assertEqual(self.version, metadata["Version"]) + self.assertEqual("python-{}-{}".format(self.version, self.commit), full_version) + with tarfile.open(str(self.sdist)) as archive: + prefix = "pypaimon-{}/".format(self.version) + self.assertEqual(full_version, archive.extractfile( + prefix + "pypaimon/_full_version").read().decode().strip()) + + def test_distribution_contents(self): + names, metadata, _ = wheel_contents(self.wheel) + self.assertEqual(">=3.6", metadata["Requires-Python"]) + self.assertFalse(any(name.startswith(("pypaimon/tests/", "pypaimon/acceptance/")) + for name in names)) + self.assertFalse(any(name.endswith((".so", ".pyd", ".jar")) for name in names)) + for name in ("pypaimon/sample/data/data.jsonl", "pypaimon/_version.py", + "pypaimon/benchmark/act/default_experiment.json"): + self.assertIn(name, names) + for suffix in ("/LICENSE", "/NOTICE"): + self.assertTrue(any(".dist-info/" in name and name.endswith(suffix) + for name in names)) + with tarfile.open(str(self.sdist)) as archive: + names = archive.getnames() + prefix = "pypaimon-{}/".format(self.version) + for name in ("pyproject.toml", "setup.py", "LICENSE", "NOTICE", "README.md"): + self.assertIn(prefix + name, names) + self.assertFalse(any("requirements.txt" in name or "requirements-dev.txt" in name + or name.startswith((prefix + "pypaimon/tests/", + prefix + "pypaimon/acceptance/")) + for name in names)) + + @staticmethod + def active_requirements(values, environment): + result = set() + for value in values: + requirement = Requirement(value) + if requirement.marker is None or requirement.marker.evaluate(environment): + result.add((canonicalize_name(requirement.name), str(requirement.specifier))) + return result + + def test_metadata_preserves_core_and_extra_conditions(self): + _, metadata, _ = wheel_contents(self.wheel) + project = self.config["project"] + self.assertEqual(set(project["optional-dependencies"]), + set(metadata.get_all("Provides-Extra"))) + for python_version in ("3.6", "3.7", "3.8", "3.11", "3.13"): + for system in ("Linux", "Windows"): + for extra in [""] + list(project["optional-dependencies"]): + with self.subTest(python=python_version, system=system, extra=extra): + environment = default_environment() + environment.update(python_version=python_version, + python_full_version=python_version + ".0", + platform_system=system, extra=extra) + expected = project["dependencies"] + project[ + "optional-dependencies"].get(extra, []) + self.assertEqual(self.active_requirements(expected, environment), + self.active_requirements( + metadata.get_all("Requires-Dist"), environment)) + + def test_dev_group_reuses_product_extras(self): + resolve = runpy.run_path(str(self.source / "dev/install.py"))["resolve_group"] + requirements = [Requirement(value) for value in resolve( + self.config["dependency-groups"], "dev")] + project = next(requirement for requirement in requirements if requirement.name == "pypaimon") + self.assertTrue({"ray", "sql", "vindex", "full-text"}.issubset(project.extras)) + # Product constraints belong to extras, not a second developer copy. + names = {requirement.name for requirement in requirements} + self.assertFalse(names.intersection({"datafusion", "paimon-vindex", "requests", "h5py"})) + + def test_sdist_rebuild_preserves_provenance_in_another_git_repo(self): + outer = self.root / "downstream" + outer.mkdir() + (outer / "README").write_text("Unrelated downstream repository") + downstream_commit = self.init_git(outer) + self.assertNotEqual(self.commit, downstream_commit) + with tarfile.open(str(self.sdist)) as archive: + # This archive was just built from our test source. + archive.extractall(str(outer)) + output = self.root / "rebuilt" + self.build(outer / "pypaimon-{}".format(self.version), output, extra=("--wheel",)) + _, metadata, full_version = wheel_contents(next(output.glob("*.whl"))) + self.assertEqual(self.version, metadata["Version"]) + self.assertEqual("python-{}-{}".format(self.version, self.commit), full_version) + + def test_release_override_applies_to_both_formats(self): + env = dict(self.env, PYPAIMON_BUILD_VERSION="2.2rc1") + output = self.root / "release-candidate" + self.build(self.source, output, env=env) + self.assertEqual({"pypaimon-2.2rc1.tar.gz", "pypaimon-2.2rc1-py3-none-any.whl"}, + {path.name for path in output.iterdir()}) + _, metadata, full_version = wheel_contents(next(output.glob("*.whl"))) + self.assertEqual("2.2rc1", metadata["Version"]) + self.assertEqual("python-2.2rc1-{}".format(self.commit), full_version) + self.assertEqual("", command(["git", "diff", "--", "pypaimon/_version.py"], + self.source, self.env)) + + def test_invalid_release_override_fails(self): + env = dict(self.env, PYPAIMON_BUILD_VERSION="not a version") + result = subprocess.run([sys.executable, "-m", "build", "--no-isolation", "--sdist"], + cwd=str(self.source), env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + self.assertNotEqual(0, result.returncode) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/paimon-python/pypaimon/_version.py b/paimon-python/pypaimon/_version.py new file mode 100644 index 000000000000..7948185ec76a --- /dev/null +++ b/paimon-python/pypaimon/_version.py @@ -0,0 +1,19 @@ +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## +# Base version; development distributions append the Git commit date. +VERSION = "2.2.dev" diff --git a/paimon-python/pypaimon/build_info.py b/paimon-python/pypaimon/build_info.py index e034f356fd4e..d042e028f01a 100644 --- a/paimon-python/pypaimon/build_info.py +++ b/paimon-python/pypaimon/build_info.py @@ -16,12 +16,12 @@ # under the License. import os -import re +import runpy import subprocess _UNKNOWN = "UNKNOWN" _FULL_VERSION_FILE = os.path.join(os.path.dirname(__file__), "_full_version") -_SETUP_FILE = os.path.join(os.path.dirname(os.path.dirname(__file__)), "setup.py") +_VERSION_FILE = os.path.join(os.path.dirname(__file__), "_version.py") def _repository_root(): @@ -33,36 +33,33 @@ def _repository_root(): return python_root -def git_commit_id(): - """Return the current Paimon Git revision without discovering an outer repo.""" +def _git_output(args): repository_root = _repository_root() env = os.environ.copy() env["GIT_CEILING_DIRECTORIES"] = os.path.dirname(repository_root) try: return subprocess.check_output( - ["git", "-C", repository_root, "rev-parse", "HEAD"], + ["git", "-C", repository_root] + args, stderr=subprocess.DEVNULL, env=env, ).decode("utf-8").strip() except Exception: - return _UNKNOWN + return None + + +def git_commit_id(): + """Return the current Paimon Git revision without discovering an outer repo.""" + return _git_output(["rev-parse", "HEAD"]) or _UNKNOWN def _source_version(): try: - with open(_SETUP_FILE, "r") as setup_file: - match = re.search( - r'^VERSION = ["\']([^"\']+)["\']$', - setup_file.read(), - re.MULTILINE, - ) - return None if match is None else match.group(1) + return runpy.run_path(_VERSION_FILE)["VERSION"] except OSError: return None -def _load_full_version(): - """Return the embedded full version, or derive it from the checkout.""" +def _embedded_full_version(): try: with open(_FULL_VERSION_FILE, "r") as full_version_file: value = full_version_file.read().strip() @@ -70,7 +67,28 @@ def _load_full_version(): return value except OSError: pass + return None + + +def package_version(): + """Resolve one version for metadata, sdist and wheel, even outside Git.""" + embedded = _embedded_full_version() + if embedded: + return embedded[len("python-"):].rsplit("-", 1)[0] version = _source_version() + if version and version.endswith(".dev"): + # Commit date, rather than build date, keeps source rebuilds stable. + date = _git_output(["log", "-1", "--format=%cd", "--date=format:%Y%m%d"]) + return version + (date or "0") + return version + + +def _load_full_version(): + """Return the embedded full version, or derive it from the checkout.""" + embedded = _embedded_full_version() + if embedded: + return embedded + version = package_version() return ( _UNKNOWN if version is None diff --git a/paimon-python/pypaimon/tests/build_info_test.py b/paimon-python/pypaimon/tests/build_info_test.py index a3dc11627e39..b7a08aea6926 100644 --- a/paimon-python/pypaimon/tests/build_info_test.py +++ b/paimon-python/pypaimon/tests/build_info_test.py @@ -34,7 +34,7 @@ def test_full_version(self): self.assertRegex( build_info.full_version(), r"^python-{}-(UNKNOWN|[0-9a-f]{{40}})$".format( - re.escape(build_info._source_version())), + re.escape(build_info.package_version())), ) def test_embedded_full_version(self): @@ -106,9 +106,10 @@ def test_sdist_provenance_survives_downstream_git_repository(self): embedded_file = os.path.join(extracted, "pypaimon", "_full_version") with open(embedded_file, "r") as full_version_file: embedded = full_version_file.read().strip() + self.assertTrue(embedded.endswith("-" + upstream_commit)) self.assertEqual( - "python-{}-{}".format(build_info._source_version(), upstream_commit), - embedded, + "pypaimon-{}".format(embedded[len("python-"):].rsplit("-", 1)[0]), + top_level, ) downstream_commit = self._init_git_repository(extracted, "downstream") diff --git a/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py b/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py index f43c354b6ee7..f30bd443ca06 100644 --- a/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py +++ b/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py @@ -68,7 +68,8 @@ def setUpClass(cls): cls.catalog = CatalogFactory.create(cls.catalog_options) cls.catalog.create_database('default', True) if not ray.is_initialized(): - ray.init(ignore_reinit_error=True, num_cpus=2) + # Nested joins and the final shuffle retain actors while scheduling tasks. + ray.init(ignore_reinit_error=True, num_cpus=4) @classmethod def tearDownClass(cls): diff --git a/paimon-python/pypaimon/tests/rest/rest_read_write_test.py b/paimon-python/pypaimon/tests/rest/rest_read_write_test.py index 4c9a9fd06635..eb5d4b459456 100644 --- a/paimon-python/pypaimon/tests/rest/rest_read_write_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_read_write_test.py @@ -546,10 +546,8 @@ def test_reader_ray_data(self): self.assertIsNotNone(ray_dataset, "Ray dataset should not be None") self.assertEqual(ray_dataset.count(), 8, "Should have 8 rows") - df = ray_dataset.to_pandas() - expect = pd.DataFrame(self.raw_data) - pd.testing.assert_frame_equal(df.sort_values(by='user_id').reset_index(drop=True), - expect.sort_values(by='user_id').reset_index(drop=True)) + actual = pa.concat_tables(ray.get(ray_dataset.to_arrow_refs())) + self.assertEqual(actual.sort_by('user_id'), self.expected.sort_by('user_id')) def test_ray_data_write_and_read(self): if not ray.is_initialized(): @@ -587,15 +585,8 @@ def test_ray_data_write_and_read(self): self.assertIsNotNone(ray_dataset, "Ray dataset should not be None") self.assertEqual(ray_dataset.count(), 3, "Should have 3 rows") - df = ray_dataset.to_pandas() - expected_df = pd.DataFrame({ - 'id': [1, 2, 3], - 'name': ['Alice', 'Bob', 'Charlie'], - 'value': [100, 200, 300], - }) - expected_df['id'] = expected_df['id'].astype('int32') - pd.testing.assert_frame_equal(df.sort_values(by='id').reset_index(drop=True), - expected_df.sort_values(by='id').reset_index(drop=True)) + actual = pa.concat_tables(ray.get(ray_dataset.to_arrow_refs())) + self.assertEqual(actual.sort_by('id'), test_data.sort_by('id')) def test_write_wide_table_large_data(self): logging.basicConfig(level=logging.INFO) diff --git a/paimon-python/pyproject.toml b/paimon-python/pyproject.toml new file mode 100644 index 000000000000..d39be4f839df --- /dev/null +++ b/paimon-python/pyproject.toml @@ -0,0 +1,211 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[build-system] +requires = [ + "setuptools>=40.8,<60; python_version < '3.7'", + "setuptools>=61; python_version >= '3.7'", + "wheel", + "packaging>=21", + "tomli>=1.2.3; python_version < '3.11'", +] +build-backend = "setuptools.build_meta" + +[project] +name = "pypaimon" +dynamic = ["version"] +description = "Apache Paimon Python API" +readme = "README.md" +# Keep the table form for setuptools versions supporting Python 3.7/3.8. +license = {text = "Apache-2.0"} +authors = [{name = "Apache Software Foundation", email = "dev@paimon.apache.org"}] +requires-python = ">=3.6" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "cachetools>=4.2,<6; python_version == '3.6'", + "cachetools>=5,<6; python_version > '3.6'", + "dataclasses>=0.8; python_version < '3.7'", + "fastavro>=1.4,<2", + "isal>=1.8,<2; python_version >= '3.9' and (platform_machine == 'x86_64' or platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'arm64')", + "fsspec>=2021.10,<2026; python_version < '3.8'", + "fsspec>=2023,<2026; python_version >= '3.8'", + "packaging>=21", + "pandas>=1.1,<2; python_version < '3.7'", + "pandas>=1.3,<3; python_version >= '3.7' and python_version < '3.9'", + "pandas>=1.5,<3; python_version >= '3.9'", + "polars>=0.9,<1; python_version < '3.8'", + "polars>=1,<2; python_version == '3.8'", + "polars>=1.32,<2; python_version >= '3.9'", + "pyarrow>=6,<7; python_version < '3.7'", + "pyarrow>=7,<13; python_version >= '3.7' and python_version < '3.8'", + "pyarrow>=16,!=19.0.0,<20; python_version >= '3.8'", + "pyroaring<=0.3.3; python_version < '3.7'", + "pyroaring<=0.4.5; python_version == '3.7'", + "pyroaring>=1.0.0; python_version >= '3.8'", + "readerwriterlock>=1,<2", + "requests>=2.21.0,<3", + "urllib3>=1.26,<3", + "zstandard>=0.19,<1", + "backports.zstd>=1.0.0,<1.4.0; python_version >= '3.9' and python_version < '3.14'", + "cramjam>=1.3.0,<3; python_version >= '3.7'", + "pyyaml>=5.4,<7", +] + +[project.urls] +Homepage = "https://paimon.apache.org" +Documentation = "https://paimon.apache.org/docs/master/pypaimon/python-api/" + +[project.scripts] +paimon = "pypaimon.cli:main" + +[project.optional-dependencies] +video = [ + "av>=12,<19; python_version >= '3.10'", + "Pillow; python_version >= '3.10'", +] +hdf5 = [ + "h5py>=3,<4; python_version >= '3.8'", +] +rosbag = [ + "rosbags>=0.11.5,<0.12; python_version >= '3.10'", +] +# datasets 4.1+ may require PyArrow 21+, outside the supported core range. +# Pandas 2.2.2+ supports NumPy 2 selected by the media dependencies. +lerobot = [ + "datasets>=4,<4.1; python_version >= '3.10'", + "pandas>=2.2.2,<3; python_version >= '3.10'", + "torch>=2.3; python_version >= '3.10'", + "lerobot>=0.4.4,<0.5; python_version >= '3.10'", +] +ray = [ + "ray>=2.10,<3; python_version >= '3.8'", +] +torch = [ + "torch", + "Pillow", +] +# ACT uses the same LeRobot stack; do not maintain its versions twice. +act = [ + "pypaimon[lerobot]; python_version >= '3.10'", + "Pillow; python_version >= '3.10'", +] +daft = [ + "daft>=0.7.6; python_version >= '3.10'", +] +oss = [ + "oss2>=2.18,<3", + "ossfs>=2021.8; python_version < '3.8'", + "ossfs>=2023; python_version >= '3.8'", +] +jindo = [ + "oss2>=2.18,<3", + "pyjindosdk>=6.10.4", +] +lance = [ + "pylance>=0.20,<1; python_version >= '3.9'", + "pylance>=0.10,<1; python_version >= '3.8' and python_version < '3.9'", +] +# Vortex 0.71 regresses native predicate pushdown on single-row files. +vortex = [ + "vortex-data==0.70.0; python_version >= '3.11'", +] +mosaic = [ + "paimon-mosaic>=0.1.0; python_version >= '3.9'", +] +lumina = [ + "lumina-data>=0.1.0", +] +vindex = [ + "paimon-vindex==0.5.0; python_version >= '3.9'", +] +full-text = [ + "paimon-ftindex==0.1.0; python_version >= '3.8'", +] +theta-sketch = [ + "datasketches>=4,<5; python_version < '3.9'", + "datasketches>=5,<6; python_version >= '3.9'", +] +hll-sketch = [ + "datasketches>=4,<5; python_version < '3.9'", + "datasketches>=5,<6; python_version >= '3.9'", +] +datafusion = [ + "datafusion>=54,<55; python_version >= '3.10'", +] +sql = [ + "pypaimon[datafusion]; python_version >= '3.10'", + "pypaimon-rust>=0.3.0; python_version >= '3.10'", +] +hdfs = [ + "hdfs-native>=0.13,<1; python_version >= '3.10' and platform_system != 'Windows'", +] + +[dependency-groups] +# Tools are separated from optional integrations so old-Python core tests and +# targeted compatibility jobs do not install unrelated native dependencies. +test = [ + "pytest>=6.2,<7; python_version < '3.7'", + "pytest>=7,<8; python_version >= '3.7'", + "parameterized>=0.8,<0.9; python_version < '3.7'", + "parameterized>=0.9,<1; python_version >= '3.7'", + "py4j==0.10.9.9", +] +lint = [ + "flake8>=4,<5; python_version < '3.8'", + "flake8>=7,<8; python_version >= '3.8'", +] +dev = [ + {include-group = "test"}, + {include-group = "lint"}, + {include-group = "build"}, + "pypaimon[oss,lance,ray,daft,sql,hdf5,rosbag,video,vortex,mosaic,lumina,vindex,full-text,theta-sketch,hll-sketch]", + "duckdb>=1.3,<1.4; python_version >= '3.9'", + # merge_into / row-id operations require Dataset.join; basic Ray I/O does not. + "ray>=2.50; python_version >= '3.9'", +] +torch = [ + {include-group = "dev"}, + "pypaimon[torch,lerobot]", +] +legacy = [ + {include-group = "test"}, + {include-group = "lint"}, + "pypaimon[oss,lumina,theta-sketch,hll-sketch]", +] +build = ["build", "twine"] + +[tool.setuptools] +include-package-data = true +license-files = ["LICENSE", "NOTICE"] + +[tool.setuptools.packages.find] +include = ["pypaimon*"] +exclude = ["pypaimon.tests*", "pypaimon.acceptance*"] + +[tool.setuptools.package-data] +pypaimon = ["_full_version", "benchmark/act/default_experiment.json"] diff --git a/paimon-python/setup.py b/paimon-python/setup.py index 47e600343ef1..e673fe50bf70 100644 --- a/paimon-python/setup.py +++ b/paimon-python/setup.py @@ -15,68 +15,29 @@ # See the License for the specific language governing permissions and # limitations under the License. ########################################################################## -import atexit +"""Setuptools hooks and the Python 3.6 bridge for pyproject.toml metadata.""" import os -import shutil -import subprocess +import runpy import sys -import tarfile -import tempfile +from packaging.version import Version from setuptools import find_packages, setup from setuptools.command.build_py import build_py from setuptools.command.sdist import sdist PYTHON_ROOT = os.path.dirname(os.path.abspath(__file__)) -FULL_VERSION_FILE = os.path.join(PYTHON_ROOT, "pypaimon", "_full_version") -UNKNOWN_COMMIT_ID = "UNKNOWN" - -VERSION = "2.2.dev" - - -def _repository_root(): - parent = os.path.dirname(PYTHON_ROOT) - if os.path.basename(PYTHON_ROOT) == "paimon-python" and os.path.exists( - os.path.join(parent, "pom.xml")): - return parent - return PYTHON_ROOT - - -def _git_output(args): - repository_root = _repository_root() - env = os.environ.copy() - env["GIT_CEILING_DIRECTORIES"] = os.path.dirname(repository_root) - try: - return subprocess.check_output( - ["git", "-C", repository_root] + args, - stderr=subprocess.DEVNULL, - env=env, - ).decode("utf-8").strip() - except Exception: - return None - - -def _full_version(): - try: - with open(FULL_VERSION_FILE, "r") as full_version_file: - embedded = full_version_file.read().strip() - if embedded: - return embedded - except OSError: - pass - - git_commit_id = _git_output(["rev-parse", "HEAD"]) - if git_commit_id is None: - git_commit_id = UNKNOWN_COMMIT_ID - return "python-{}-{}".format(VERSION, git_commit_id) +BUILD_INFO = runpy.run_path(os.path.join(PYTHON_ROOT, "pypaimon", "build_info.py")) +VERSION = str(Version(os.environ.get("PYPAIMON_BUILD_VERSION") or + BUILD_INFO["package_version"]())) +COMMIT_ID = BUILD_INFO["full_version"]().rsplit("-", 1)[-1] +FULL_VERSION = "python-{}-{}".format(VERSION, COMMIT_ID) def _write_full_version(root): package_dir = os.path.join(root, "pypaimon") - if not os.path.exists(package_dir): - os.makedirs(package_dir) - with open(os.path.join(package_dir, "_full_version"), "w") as full_version_file: - full_version_file.write(_full_version() + "\n") + os.makedirs(package_dir, exist_ok=True) + with open(os.path.join(package_dir, "_full_version"), "w") as version_file: + version_file.write(FULL_VERSION + "\n") class PaimonBuildPy(build_py): @@ -93,254 +54,44 @@ def make_release_tree(self, base_dir, files): _write_full_version(base_dir) -def get_dev_version(): - """Generate dev version with commit date. - Format: 2.2.devYYYYMMDD (e.g. 2.2.dev20260911) - Uses the commit date (author date) for reproducibility. - """ - base = VERSION.rstrip(".") - if not base.endswith("dev"): - return None - - try: - date_str = _git_output( - ["log", "-1", "--format=%cd", "--date=format:%Y%m%d"]) - if date_str is None: - raise RuntimeError("Git commit date is unavailable") - except Exception: - print("Warning: git not available, skipping dev package.") - return None - - return base + date_str - - -def _build_dev_package(): - """After sdist completes, repack a copy with dev version.""" - if "sdist" not in sys.argv: - return - - dev_version = get_dev_version() - if dev_version is None: - return - - from packaging.version import Version - normalized = str(Version(VERSION)) - dev_normalized = str(Version(dev_version)) - - src_name = "pypaimon-{}".format(normalized) - dev_name = "pypaimon-{}".format(dev_normalized) - - src_tar = os.path.join("dist", src_name + ".tar.gz") - if not os.path.exists(src_tar): - return - - tmp_dir = tempfile.mkdtemp() - try: - with tarfile.open(src_tar, "r:gz") as tar: - tar.extractall(tmp_dir) - - src_dir = os.path.join(tmp_dir, src_name) - dev_dir = os.path.join(tmp_dir, dev_name) - os.rename(src_dir, dev_dir) - - # Update version in PKG-INFO files - for pkg_info in [ - os.path.join(dev_dir, "PKG-INFO"), - os.path.join(dev_dir, "pypaimon.egg-info", "PKG-INFO"), - ]: - if os.path.exists(pkg_info): - with open(pkg_info, "r") as f: - content = f.read() - content = content.replace( - "Version: " + normalized, - "Version: " + dev_normalized - ) - with open(pkg_info, "w") as f: - f.write(content) - - # Update VERSION in setup.py so pip install gets the correct version - setup_py = os.path.join(dev_dir, "setup.py") - if os.path.exists(setup_py): - with open(setup_py, "r") as f: - content = f.read() - content = content.replace( - 'VERSION = "' + VERSION + '"', - 'VERSION = "' + dev_version + '"' - ) - with open(setup_py, "w") as f: - f.write(content) - - # Keep the embedded full version consistent with the dev package version. - full_version_file = os.path.join(dev_dir, "pypaimon", "_full_version") - if os.path.exists(full_version_file): - with open(full_version_file, "r") as f: - content = f.read() - version_prefix = "python-{}-".format(VERSION) - if content.startswith(version_prefix): - content = "python-{}-{}".format( - dev_version, content[len(version_prefix):]) - with open(full_version_file, "w") as f: - f.write(content) - - dev_tar = os.path.join("dist", dev_name + ".tar.gz") - with tarfile.open(dev_tar, "w:gz") as tar: - tar.add(dev_dir, arcname=dev_name) - - print("Created dev package: " + dev_tar) - finally: - shutil.rmtree(tmp_dir) - - -atexit.register(_build_dev_package) - -PACKAGES = find_packages( - include=["pypaimon*"], - exclude=["pypaimon.tests*", "pypaimon.acceptance*"], -) - - -def read_requirements(): - """Read requirements from dev/requirements.txt file.""" - requirements_path = os.path.join(os.path.dirname(__file__), 'dev', 'requirements.txt') - requirements = [] - - if os.path.exists(requirements_path): - with open(requirements_path, 'r', encoding='utf-8') as f: - for line in f: - line = line.strip() - # Skip empty lines and comments - if line and not line.startswith('#'): - requirements.append(line) - - return requirements - - -install_requires = read_requirements() - -VIDEO_DEPENDENCIES = [ - 'av>=12,<19; python_version>="3.10"', - 'Pillow; python_version>="3.10"', -] - -LEROBOT_DEPENDENCIES = [ - # datasets 4.1+ may select PyArrow 21+, while PyPaimon currently - # supports PyArrow <20. Pandas 2.2.2+ supports NumPy 2.x selected - # by LeRobot's media dependencies. - 'datasets>=4,<4.1; python_version>="3.10"', - 'pandas>=2.2.2,<3; python_version>="3.10"', - 'torch>=2.3; python_version>="3.10"', - 'lerobot>=0.4.4,<0.5; python_version>="3.10"', -] +def _legacy_metadata(): + # Python 3.6's last setuptools does not support PEP 621. Translate the same + # TOML metadata instead of keeping another dependency list or dropping 3.6. + import tomli + + with open(os.path.join(PYTHON_ROOT, "pyproject.toml"), "rb") as project_file: + config = tomli.load(project_file) + project = config["project"] + setuptools_config = config["tool"]["setuptools"] + with open(os.path.join(PYTHON_ROOT, project["readme"]), encoding="utf-8") as readme: + long_description = readme.read() + author = project["authors"][0] + return dict( + name=project["name"], + description=project["description"], + long_description=long_description, + long_description_content_type="text/markdown", + license=project["license"]["text"], + author=author["name"], + author_email=author["email"], + url=project["urls"]["Homepage"], + project_urls=project["urls"], + python_requires=project["requires-python"], + classifiers=project["classifiers"], + install_requires=project["dependencies"], + extras_require=project["optional-dependencies"], + entry_points={"console_scripts": [ + "{}={}".format(name, entry) for name, entry in project["scripts"].items() + ]}, + packages=find_packages(**setuptools_config["packages"]["find"]), + include_package_data=setuptools_config["include-package-data"], + package_data=setuptools_config["package-data"], + license_files=setuptools_config["license-files"], + ) -long_description = "See Apache Paimon Python API \ -[Doc](https://paimon.apache.org/docs/master/pypaimon/python-api/) for usage." setup( - name="pypaimon", version=VERSION, - packages=PACKAGES, - include_package_data=True, - package_data={ - "pypaimon": [ - "_full_version", - "benchmark/act/default_experiment.json", - ], - }, cmdclass={"build_py": PaimonBuildPy, "sdist": PaimonSdist}, - install_requires=install_requires, - entry_points={ - 'console_scripts': [ - 'paimon=pypaimon.cli:main', - ], - }, - extras_require={ - 'video': VIDEO_DEPENDENCIES, - 'hdf5': [ - # HDF5 loading is explicitly guarded and documented as Python 3.8+. - 'h5py>=3,<4; python_version>="3.8"', - ], - 'rosbag': [ - # rosbags is pure Python and does not require a ROS installation. - 'rosbags>=0.11.5,<0.12; python_version>="3.10"', - ], - 'lerobot': LEROBOT_DEPENDENCIES, - 'ray': [ - 'ray>=2.10,<3; python_version>="3.8"', - ], - 'torch': [ - 'torch', - 'Pillow', - ], - 'act': LEROBOT_DEPENDENCIES + [ - 'Pillow; python_version>="3.10"', - ], - 'daft': [ - 'daft>=0.7.6; python_version>="3.10"', - ], - 'oss': [ - 'oss2>=2.18,<3', - 'ossfs>=2021.8; python_version<"3.8"', - 'ossfs>=2023; python_version>="3.8"' - ], - 'jindo': [ - 'oss2>=2.18,<3', - 'pyjindosdk>=6.10.4', - ], - 'lance': [ - 'pylance>=0.20,<1; python_version>="3.9"', - 'pylance>=0.10,<1; python_version>="3.8" and python_version<"3.9"' - ], - 'vortex': [ - 'vortex-data==0.70.0; python_version>="3.11"', - ], - 'mosaic': [ - 'paimon-mosaic>=0.1.0; python_version>="3.9"', - ], - 'lumina': [ - 'lumina-data>=0.1.0' - ], - 'vindex': [ - 'paimon-vindex==0.5.0; python_version>="3.9"', - ], - 'full-text': [ - 'paimon-ftindex==0.1.0; python_version>="3.8"', - ], - 'theta-sketch': [ - 'datasketches>=4,<5; python_version<"3.9"', - 'datasketches>=5,<6; python_version>="3.9"', - ], - 'hll-sketch': [ - 'datasketches>=4,<5; python_version<"3.9"', - 'datasketches>=5,<6; python_version>="3.9"', - ], - 'datafusion': [ - 'datafusion>=54,<55; python_version>="3.10"', - ], - 'sql': [ - 'pypaimon-rust>=0.3.0; python_version>="3.10"', - 'datafusion>=54,<55; python_version>="3.10"', - ], - 'hdfs': [ - 'hdfs-native>=0.13,<1; python_version >= "3.10" and platform_system != "Windows"', - ], - }, - description="Apache Paimon Python API", - long_description=long_description, - long_description_content_type="text/markdown", - author="Apache Software Foundation", - author_email="dev@paimon.apache.org", - url="https://paimon.apache.org", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - ], - python_requires=">=3.6", + **(_legacy_metadata() if sys.version_info < (3, 7) else {}) )