From f68090d1d925b41729cc0ad3a06945c853f87ed6 Mon Sep 17 00:00:00 2001 From: Manos Date: Tue, 28 Jul 2026 19:35:06 +0200 Subject: [PATCH 1/6] Bump version to 1.0.9 Syncs the version string across pyproject.toml, setup.py, and daisy/__init__.py (previously drifted: 1.0.8 / 1.0.8 / 1.0.0). Merging this to main will trigger the pypi-release workflow to publish 1.0.9, since 1.0.8 is already live on PyPI. Co-Authored-By: Claude Sonnet 5 --- daisy/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/daisy/__init__.py b/daisy/__init__.py index e3cb0fd..7002cc8 100644 --- a/daisy/__init__.py +++ b/daisy/__init__.py @@ -17,7 +17,7 @@ (distance-r) search via searchIndex(query, SearchConfig(type=QueryType.RANGE, r=r)). """ -__version__ = "1.0.0" +__version__ = "1.0.9" __author__ = "" try: diff --git a/pyproject.toml b/pyproject.toml index 8a95f63..02edcf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "daisy-exact-search" -version = "1.0.8" +version = "1.0.9" description = "DaiSy: A Library for Scalable Data Series Similarity Search" readme = "README.md" requires-python = ">=3.10" diff --git a/setup.py b/setup.py index ee7bd93..c13d385 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ from setuptools.command.build_ext import build_ext as _build_ext # Version -__version__ = "1.0.8" +__version__ = "1.0.9" # Detect platform IS_MACOS = sys.platform == "darwin" From 0d74300f25cae5816c1ff24c34f1e76e25b7e4d1 Mon Sep 17 00:00:00 2001 From: sophisid Date: Wed, 29 Jul 2026 11:38:21 +0200 Subject: [PATCH 2/6] Add a release dry-run that rehearses the PyPI publish on every PR pypi-release.yml only builds when the version changes, so a broken build is invisible until the very push that is meant to publish it. ci.yml does not cover the gap: it builds through CMake with -DBUILD_SING=OFF, while setup.py compiles every source in the package unconditionally. release-check.yml runs the same environment and the same build steps as the release job, then stops before twine upload. It installs the built wheel into a clean venv and searches an index with it, because daisy/__init__.py downgrades a missing extension to a warning, so a wheel whose extension does not load still imports cleanly. check_release_artifacts.py covers the rejections that a green build does not rule out: a version that disagrees between setup.py and pyproject.toml, an sdist missing sources, and a wheel tagged linux_x86_64, which PyPI refuses. It runs standalone against any dist/ directory, so a release can be rehearsed locally too. --- .github/workflows/release-check.yml | 104 ++++++++++++ scripts/check_release_artifacts.py | 238 ++++++++++++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 .github/workflows/release-check.yml create mode 100644 scripts/check_release_artifacts.py diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml new file mode 100644 index 0000000..2fc72c8 --- /dev/null +++ b/.github/workflows/release-check.yml @@ -0,0 +1,104 @@ +name: Release dry-run + +# Rehearses the PyPI release on every pull request, so a release that would be +# rejected is caught here instead of on a push to main. It mirrors the +# environment and the build steps of pypi-release.yml exactly, but stops short +# of `twine upload`. +# +# Note that this covers ground ci.yml does not: ci.yml builds via CMake with +# -DBUILD_SING=OFF, while setup.py always compiles every source in the package. +# A file CMake skips can therefore break the release build while CI stays green. +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: {} + +jobs: + dry-run: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + # Kept in step with pypi-release.yml: the release publishes a single + # cp312 wheel, so that is the interpreter the rehearsal has to use. + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake libomp-dev + + - name: Install Python build dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_DaiSy.txt + + # The step that would have caught the current breakage: `python -m build` + # compiles the full extension, including sources CMake leaves out. + - name: Build distributions + run: python -m build + + - name: Verify distribution metadata + run: twine check --strict dist/* + + - name: Verify artifacts are publishable + run: python scripts/check_release_artifacts.py --dist-dir dist --check-pypi + + # Installing the built wheel rather than the source tree is what makes + # this a packaging test: it catches a package that compiles but ships + # without its Python files or its compiled extension. + - name: Install the built wheel into a clean environment + run: | + python -m venv /tmp/wheel-test + /tmp/wheel-test/bin/pip install --upgrade pip + /tmp/wheel-test/bin/pip install dist/*.whl + + - name: Smoke-test the installed package + run: | + cd /tmp + /tmp/wheel-test/bin/python - <<'PY' + import numpy as np + + import daisy + # daisy/__init__.py downgrades a missing extension to a warning, so + # import the extension explicitly or a broken wheel looks healthy. + from daisy import _core + + print("daisy imported from", daisy.__file__) + print("extension imported from", _core.__file__) + + rng = np.random.default_rng(0) + database = rng.random((256, 64), dtype=np.float32) + # searchIndex takes a 2D batch of queries, one row per query. + query = database[7:8].copy() + + search = daisy.BruteForceSearch(daisy.DistanceType.L2_SQUARED) + search.buildIndex(database) + indices, distances = search.searchIndex(query, 5) + + print("top-5 indices:", indices[0]) + print("top-5 distances:", distances[0]) + + assert indices[0][0] == 7, "nearest neighbour of a database row must be itself, got %r" % (indices[0][0],) + assert distances[0][0] < 1e-4, "self-distance should be ~0, got %r" % (distances[0][0],) + print("smoke test passed") + PY + + - name: Upload distributions for inspection + if: always() + uses: actions/upload-artifact@v4 + with: + name: dist-${{ github.sha }} + path: dist/ + if-no-files-found: warn diff --git a/scripts/check_release_artifacts.py b/scripts/check_release_artifacts.py new file mode 100644 index 0000000..b92c089 --- /dev/null +++ b/scripts/check_release_artifacts.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Validate the distributions in dist/ before they are uploaded to PyPI. + +PyPI rejects an upload for reasons that a successful `python -m build` does not +catch (bad platform tag, version already taken, metadata mismatch). Twine only +uploads after building, so a rejection wastes the whole release run and, worse, +can leave a half-published release when some files upload and others do not. + +This script runs the checks locally so a broken release is caught in CI instead. + +Usage: + python scripts/check_release_artifacts.py [--dist-dir dist] [--check-pypi] + +Exit code 0 means the artifacts look publishable. +""" + +import argparse +import json +import re +import sys +import tarfile +import urllib.error +import urllib.request +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +PYPI_NAME = "daisy-exact-search" + +# Platform tags PyPI accepts for a binary wheel. A wheel built by a plain +# `python -m build` on Linux is tagged `linux_x86_64`, which PyPI refuses: +# it carries no glibc floor, so pip cannot tell whether it will run. +ACCEPTED_PLATFORM_TAGS = ("any", "manylinux", "musllinux", "macosx", "win") + +errors = [] +warnings = [] + + +def fail(msg): + errors.append(msg) + print("FAIL: " + msg) + + +def warn(msg): + warnings.append(msg) + print("WARN: " + msg) + + +def ok(msg): + print("ok: " + msg) + + +def read_declared_versions(): + """Return the version declared in each place the project records one.""" + versions = {} + + pyproject = (PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8") + match = re.search(r'^version\s*=\s*"([^"]+)"', pyproject, re.MULTILINE) + if match: + versions["pyproject.toml"] = match.group(1) + + setup_py = (PROJECT_ROOT / "setup.py").read_text(encoding="utf-8") + match = re.search(r'^__version__\s*=\s*"([^"]+)"', setup_py, re.MULTILINE) + if match: + versions["setup.py"] = match.group(1) + + init_py = (PROJECT_ROOT / "daisy" / "__init__.py").read_text(encoding="utf-8") + match = re.search(r'^__version__\s*=\s*"([^"]+)"', init_py, re.MULTILINE) + if match: + versions["daisy/__init__.py"] = match.group(1) + + return versions + + +def check_versions(): + """setup.py and pyproject.toml both feed the build, so they must agree.""" + versions = read_declared_versions() + + for source in ("pyproject.toml", "setup.py"): + if source not in versions: + fail("could not find a version declaration in " + source) + + build_versions = {k: v for k, v in versions.items() if k != "daisy/__init__.py"} + if len(set(build_versions.values())) > 1: + fail( + "version mismatch between build files: " + + ", ".join(k + "=" + v for k, v in build_versions.items()) + ) + else: + ok("build files agree on version " + next(iter(build_versions.values()))) + + # Not a publish blocker, but users read daisy.__version__ at runtime. + runtime_version = versions.get("daisy/__init__.py") + if runtime_version and runtime_version not in build_versions.values(): + warn( + "daisy/__init__.py declares __version__ = " + + runtime_version + + " but the package is built as " + + next(iter(build_versions.values())) + ) + + return next(iter(build_versions.values())) + + +def check_artifacts_present(dist_dir, version): + sdists = sorted(dist_dir.glob("*.tar.gz")) + wheels = sorted(dist_dir.glob("*.whl")) + + if not sdists: + fail("no sdist (*.tar.gz) in " + str(dist_dir)) + elif len(sdists) > 1: + fail("more than one sdist in " + str(dist_dir) + "; clean it before building") + else: + ok("found sdist " + sdists[0].name) + + if not wheels: + fail("no wheel (*.whl) in " + str(dist_dir)) + else: + ok("found " + str(len(wheels)) + " wheel(s): " + ", ".join(w.name for w in wheels)) + + for artifact in sdists + wheels: + # Wheel and sdist filenames normalise '-' to '_' in the project name. + if version.replace("-", "_") not in artifact.name: + fail(artifact.name + " does not carry the declared version " + version) + + return sdists, wheels + + +def check_wheel_tags(wheels): + """Reject wheels PyPI will refuse, before we spend a release run finding out.""" + for wheel in wheels: + parts = wheel.stem.split("-") + if len(parts) < 5: + fail(wheel.name + " is not a valid wheel filename") + continue + + platform_tag = parts[-1] + if platform_tag.startswith(ACCEPTED_PLATFORM_TAGS): + ok(wheel.name + " has a PyPI-acceptable platform tag: " + platform_tag) + else: + fail( + wheel.name + + " has platform tag '" + + platform_tag + + "', which PyPI rejects. Run the wheel through `auditwheel repair` " + "(or build it with cibuildwheel) to get a manylinux tag." + ) + + +def check_sdist_contents(sdist): + """The sdist is what users on unsupported platforms compile from.""" + required = [ + "pyproject.toml", + "setup.py", + "README.md", + "daisy/__init__.py", + "pybinds/setup.cpp", + ] + + with tarfile.open(sdist, "r:gz") as tar: + # Every path is prefixed with the '-/' root directory. + names = {name.split("/", 1)[-1] for name in tar.getnames()} + + missing = [path for path in required if path not in names] + if missing: + fail(sdist.name + " is missing: " + ", ".join(missing)) + else: + ok(sdist.name + " contains all required files") + + # setup.py compiles these unconditionally; if MANIFEST.in drops one, the + # sdist builds here but fails for anyone installing from source. + sources = [n for n in names if n.startswith(("lib/", "commons/")) and n.endswith((".cpp", ".hpp"))] + if not sources: + fail(sdist.name + " contains no C++ sources; check MANIFEST.in") + else: + ok(sdist.name + " contains " + str(len(sources)) + " C++ source/header files") + + +def check_pypi(version): + """Warn if this version is already on PyPI; re-uploading is a hard error.""" + url = "https://pypi.org/pypi/" + PYPI_NAME + "/json" + try: + with urllib.request.urlopen(url, timeout=30) as response: + data = json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + ok(PYPI_NAME + " is not on PyPI yet; any version can be published") + return + warn("could not query PyPI (HTTP " + str(exc.code) + "); skipping duplicate check") + return + except Exception as exc: + warn("could not query PyPI (" + str(exc) + "); skipping duplicate check") + return + + published = set(data.get("releases", {})) + if version in published: + warn( + "version " + + version + + " is already published; the release job will skip the upload. " + "Bump the version to publish." + ) + else: + ok("version " + version + " is not on PyPI yet") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dist-dir", default="dist", help="directory holding the built distributions") + parser.add_argument("--check-pypi", action="store_true", help="also query PyPI for the version") + args = parser.parse_args() + + dist_dir = Path(args.dist_dir).resolve() + if not dist_dir.is_dir(): + print("FAIL: " + str(dist_dir) + " does not exist; run `python -m build` first") + return 1 + + print("Checking release artifacts in " + str(dist_dir)) + print() + + version = check_versions() + sdists, wheels = check_artifacts_present(dist_dir, version) + check_wheel_tags(wheels) + for sdist in sdists: + check_sdist_contents(sdist) + if args.check_pypi: + check_pypi(version) + + print() + if errors: + print(str(len(errors)) + " check(s) failed; these artifacts are not safe to upload.") + return 1 + + print("All checks passed" + (" (" + str(len(warnings)) + " warning(s))" if warnings else "") + ".") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 37b4f037811b5eccffa588c71489536aa2de4b0a Mon Sep 17 00:00:00 2001 From: sophisid Date: Wed, 29 Jul 2026 12:01:41 +0200 Subject: [PATCH 3/6] Keep the release dry-run's build output as a downloadable artifact Reading a failed job's log back needs an authenticated GitHub token, which makes a build failure awkward to diagnose from outside the browser. Tee the build output to build.log, upload it alongside dist/, and put the first error lines in the job summary so the cause is visible without scrolling the raw log. --- .github/workflows/release-check.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml index 2fc72c8..f49d9f3 100644 --- a/.github/workflows/release-check.yml +++ b/.github/workflows/release-check.yml @@ -46,8 +46,24 @@ jobs: # The step that would have caught the current breakage: `python -m build` # compiles the full extension, including sources CMake leaves out. + # The output is teed to a file so a compiler error stays downloadable as + # an artifact; the raw job log needs a GitHub token to read back. - name: Build distributions - run: python -m build + run: | + set -o pipefail + python -m build 2>&1 | tee build.log + + # gcc prints one error and then many follow-on lines, so surface the + # first errors in the job summary instead of making people scroll. + - name: Summarise build errors + if: failure() + run: | + { + echo "### First errors from \`python -m build\`" + echo '```' + grep -E -m 20 '(^|:)[0-9]+:[0-9]+: error:|^error:|^ERROR' build.log || echo "no error lines matched; see the build-log artifact" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" - name: Verify distribution metadata run: twine check --strict dist/* @@ -95,10 +111,12 @@ jobs: print("smoke test passed") PY - - name: Upload distributions for inspection + - name: Upload distributions and build log for inspection if: always() uses: actions/upload-artifact@v4 with: - name: dist-${{ github.sha }} - path: dist/ + name: release-check-${{ github.sha }} + path: | + dist/ + build.log if-no-files-found: warn From c531b7139ba6b9721c06a19747cb2683d1ad565a Mon Sep 17 00:00:00 2001 From: sophisid Date: Wed, 29 Jul 2026 12:13:20 +0200 Subject: [PATCH 4/6] Let the DTW SING search compile in a CPU-only build exact_DTW_SING_new called LBDshortstreamGPUinsidedynamicratev2dtw unconditionally while its declaration sits behind #if SING_CUDA_ENABLED, so the PyPI build broke. Stub it like LBDfloatstreamGPU already is. --- lib/algos/Sing.cpp | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/lib/algos/Sing.cpp b/lib/algos/Sing.cpp index 8d532f6..5b80059 100644 --- a/lib/algos/Sing.cpp +++ b/lib/algos/Sing.cpp @@ -36,6 +36,11 @@ namespace daisy #else void LBDfloatstreamGPU(sax_type *gsaxarray, float *positionmap, ts_type *paa, float *gqts, float bsf, unsigned long size, float *gpositionmap, int paa_segments, float mindist_sqrt); + void LBDshortstreamGPUinsidedynamicratev2dtw(sax_type *gsaxarray, short *positionmap, + ts_type *paaU, ts_type *paaL, float *gqtsU, float *gqtsL, + float bsf, unsigned long size, short *gpositionmap, + int paa_segments, float mindist_sqrt, unsigned long *gpuoffset, + float shortrate, int chunknumber, bool *activechunk); #endif first_buffer_layer2 *initialize_simrec(int initial_buffer_size, int number_of_buffers, int max_total_buffers_size, isax_index *index); @@ -1363,7 +1368,34 @@ namespace daisy (void)gpositionmap; (void)paa_segments; (void)mindist_sqrt; - + + } + + // The DTW search stages this on the GPU between two barriers, so the CPU + // build has to leave gpuoffset alone: the worker threads are already + // running and hold a pointer to it. + void LBDshortstreamGPUinsidedynamicratev2dtw(sax_type *gsaxarray, short *positionmap, + ts_type *paaU, ts_type *paaL, float *gqtsU, float *gqtsL, + float bsf, unsigned long size, short *gpositionmap, + int paa_segments, float mindist_sqrt, unsigned long *gpuoffset, + float shortrate, int chunknumber, bool *activechunk) + { + (void)gsaxarray; + (void)positionmap; + (void)paaU; + (void)paaL; + (void)gqtsU; + (void)gqtsL; + (void)bsf; + (void)size; + (void)gpositionmap; + (void)paa_segments; + (void)mindist_sqrt; + (void)gpuoffset; + (void)shortrate; + (void)chunknumber; + (void)activechunk; + } #endif From a9c8a0a515aa8dc14ac9ad5a128dedf89d5a9581 Mon Sep 17 00:00:00 2001 From: sophisid Date: Wed, 29 Jul 2026 12:27:00 +0200 Subject: [PATCH 5/6] Build a manylinux wheel instead of one PyPI rejects python -m build tags Linux wheels linux_x86_64, so add an auditwheel repair step to both workflows. auditwheel checks glibc, not CPU features, so -march=native would have turned a rejected wheel into a SIGILL on any narrower CPU; DAISY_PORTABLE_BUILD targets AVX2 instead. SIMD.hpp tested __AVX__ while the code it guards uses AVX2 intrinsics. --- .github/workflows/pypi-release.yml | 23 ++++++++++++++++++++++- .github/workflows/release-check.yml | 20 ++++++++++++++++++++ lib/utils/SIMD.hpp | 6 ++++-- setup.py | 19 ++++++++++++++++++- 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index e2d300f..80eb11d 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -67,11 +67,32 @@ jobs: - name: Build distributions if: steps.version_check.outputs.should_publish == 'true' + env: + DAISY_PORTABLE_BUILD: "1" run: python -m build + # Keep these steps in step with release-check.yml, or the dry-run stops + # rehearsing what actually gets uploaded. + - name: Repair the Linux wheel + if: steps.version_check.outputs.should_publish == 'true' + run: | + set -euo pipefail + pip install auditwheel patchelf + mkdir -p wheelhouse + for wheel in dist/*.whl; do + auditwheel repair "$wheel" --wheel-dir wheelhouse + done + rm dist/*.whl + mv wheelhouse/*.whl dist/ + ls -l dist/ + - name: Verify distributions if: steps.version_check.outputs.should_publish == 'true' - run: twine check dist/* + run: twine check --strict dist/* + + - name: Verify artifacts are publishable + if: steps.version_check.outputs.should_publish == 'true' + run: python scripts/check_release_artifacts.py --dist-dir dist - name: Publish to PyPI if: steps.version_check.outputs.should_publish == 'true' diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml index f49d9f3..0458e86 100644 --- a/.github/workflows/release-check.yml +++ b/.github/workflows/release-check.yml @@ -49,10 +49,30 @@ jobs: # The output is teed to a file so a compiler error stays downloadable as # an artifact; the raw job log needs a GitHub token to read back. - name: Build distributions + env: + DAISY_PORTABLE_BUILD: "1" run: | set -o pipefail python -m build 2>&1 | tee build.log + # `python -m build` tags a Linux wheel linux_x86_64, which PyPI refuses. + # auditwheel rewrites the tag to the glibc the wheel actually needs and + # vendors in the shared libraries it links, libgomp among them. + - name: Repair the Linux wheel + run: | + set -euo pipefail + # auditwheel shells out to patchelf and wants >= 0.14; the PyPI + # package ships a current binary, so we do not depend on whatever + # the runner image happens to carry. + pip install auditwheel patchelf + mkdir -p wheelhouse + for wheel in dist/*.whl; do + auditwheel repair "$wheel" --wheel-dir wheelhouse + done + rm dist/*.whl + mv wheelhouse/*.whl dist/ + ls -l dist/ + # gcc prints one error and then many follow-on lines, so surface the # first errors in the job summary instead of making people scroll. - name: Summarise build errors diff --git a/lib/utils/SIMD.hpp b/lib/utils/SIMD.hpp index 9b4b183..b3a7f7d 100644 --- a/lib/utils/SIMD.hpp +++ b/lib/utils/SIMD.hpp @@ -13,8 +13,10 @@ #include #include -// Detect AVX support -#if defined(__AVX__) || defined(_M_AVX) +// Detect AVX2 support. The guarded code uses AVX2 intrinsics (_mm256_srlv_epi32, +// _mm256_cvtepu8_epi16 and friends), so testing for plain __AVX__ turned a CPU +// with AVX but no AVX2 into a compile error instead of the scalar fallback. +#if defined(__AVX2__) #define DAISY_SIMD_AVAILABLE 1 #include #else diff --git a/setup.py b/setup.py index c13d385..5fd5508 100644 --- a/setup.py +++ b/setup.py @@ -180,8 +180,25 @@ def build_extensions(self): link_args.append("-fopenmp") # SIMD optimizations (not reliable on ARM64 Macs) + # + # -march=native tunes for the machine doing the build, which is what you + # want when compiling from source but not when building a wheel for other + # people: SIMD.hpp picks its code path at compile time, so a wheel built on + # a runner with a wider instruction set dies with SIGILL elsewhere. Set + # DAISY_PORTABLE_BUILD=1 when producing a redistributable wheel; -mavx + # stays, so DAISY_SIMD_AVAILABLE is still 1. + PORTABLE_BUILD = os.environ.get("DAISY_PORTABLE_BUILD", "0").lower() in ("1", "true", "yes") + if not (IS_MACOS and platform.machine() in ("arm64", "aarch64")): - compile_args.extend(["-mavx", "-march=native"]) + compile_args.append("-mavx") + if PORTABLE_BUILD: + # The guarded code needs AVX2, not just AVX, so a portable wheel has + # to name AVX2 explicitly once -march=native is gone. That puts the + # floor at Haswell (2013). + print("[DaiSy] Portable build requested - targeting AVX2 instead of -march=native") + compile_args.append("-mavx2") + else: + compile_args.append("-march=native") # Get the project root directory as absolute path project_root = str(Path(__file__).parent.absolute()) From c6080455004ff0d1149075ce8927cf8e6dc9afcc Mon Sep 17 00:00:00 2001 From: sophisid Date: Wed, 29 Jul 2026 12:43:09 +0200 Subject: [PATCH 6/6] Ship an extension that actually loads pybinds bound Coconut, Fresh, DumpyOS and Hercules but setup.py never compiled their sources, so the wheel failed on the first undefined typeinfo. Add them plus ds_tree and VectorDataLoader, ship *.h in the sdist, and drop the unregistered abstract base from the Hercules and DumpyOS bindings, which made module init fail once they linked. --- .github/workflows/release-check.yml | 29 +++++++++++++++++++++++++++-- MANIFEST.in | 6 +++--- pybinds/setup.cpp | 7 +++++-- setup.py | 13 +++++++++++++ 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml index 0458e86..3f48924 100644 --- a/.github/workflows/release-check.yml +++ b/.github/workflows/release-check.yml @@ -100,6 +100,33 @@ jobs: /tmp/wheel-test/bin/pip install --upgrade pip /tmp/wheel-test/bin/pip install dist/*.whl + # daisy/__init__.py turns a failed extension load into a warning, and the + # next import of daisy._core then reports "already registered" rather than + # the original cause. Exec the .so on its own, in its own process, so a + # broken extension reports the error it actually raised. + - name: Check the extension initialises + run: | + cd /tmp + /tmp/wheel-test/bin/python - <<'PY' + import glob + import importlib.util + import os + + # find_spec locates the package without running its __init__.py. + spec = importlib.util.find_spec("daisy") + assert spec is not None and spec.origin, "daisy is not importable at all" + + matches = glob.glob(os.path.join(os.path.dirname(spec.origin), "_core*.so")) + assert matches, "the wheel installed no _core extension" + + core_spec = importlib.util.spec_from_file_location("_core", matches[0]) + module = importlib.util.module_from_spec(core_spec) + core_spec.loader.exec_module(module) + + print("extension initialised:", matches[0]) + print("exported:", sorted(n for n in dir(module) if n[0].isupper())) + PY + - name: Smoke-test the installed package run: | cd /tmp @@ -107,8 +134,6 @@ jobs: import numpy as np import daisy - # daisy/__init__.py downgrades a missing extension to a warning, so - # import the extension explicitly or a broken wheel looks healthy. from daisy import _core print("daisy imported from", daisy.__file__) diff --git a/MANIFEST.in b/MANIFEST.in index 53fe9fc..8df8769 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,9 @@ include README.md include requirements_DaiSy.txt recursive-include daisy *.py -recursive-include lib *.hpp *.cpp -recursive-include commons *.hpp *.cpp -recursive-include pybinds *.cpp +recursive-include lib *.h *.hpp *.cpp +recursive-include commons *.h *.hpp *.cpp +recursive-include pybinds *.h *.hpp *.cpp recursive-exclude build * recursive-exclude DaiSy_env * recursive-exclude faiss_env * diff --git a/pybinds/setup.cpp b/pybinds/setup.cpp index 7a4193c..c7ad9b6 100644 --- a/pybinds/setup.cpp +++ b/pybinds/setup.cpp @@ -721,7 +721,10 @@ PYBIND11_MODULE(_core, m) .def_readwrite("flush_buffer_size", &daisy::HerculesConfig::flush_buffer_size) .def_readwrite("index_dir", &daisy::HerculesConfig::index_dir); - pybind11::class_(m, "Hercules", "Hercules hierarchical time series similarity index") + // Bound without SimilaritySearchAlgorithm: the base is abstract and never + // registered, and naming it here makes module init fail with "referenced + // unknown base type". Every other algorithm is bound the same way. + pybind11::class_(m, "Hercules", "Hercules hierarchical time series similarity index") .def(pybind11::init(), "Create a new Hercules with the given distance metric") .def(pybind11::init(), "Create a new Hercules with the given distance metric and configuration") .def("setNumThreads", &daisy::Hercules::setNumThreads, "Set the number of query threads") @@ -770,7 +773,7 @@ PYBIND11_MODULE(_core, m) .def_readwrite("fill_lower", &daisy::DumpyOSConfig::fill_lower) .def_readwrite("fill_upper", &daisy::DumpyOSConfig::fill_upper); - pybind11::class_(m, "DumpyOS", "DumpyOS iSAX-based multi-ary adaptive time series similarity index") + pybind11::class_(m, "DumpyOS", "DumpyOS iSAX-based multi-ary adaptive time series similarity index") .def(pybind11::init(), "Create a new DumpyOS with the given distance metric") .def(pybind11::init(), "Create a new DumpyOS with the given distance metric and configuration") .def("setNumThreads", &daisy::DumpyOS::setNumThreads, "Set the number of threads") diff --git a/setup.py b/setup.py index 5fd5508..26a63a5 100644 --- a/setup.py +++ b/setup.py @@ -111,6 +111,7 @@ def build_extensions(self): "commons/common.cpp", "commons/dataloaders.cpp", "commons/paramSetup.cpp", + "commons/VectorDataLoader.cpp", # lib - distance computers "lib/distance_computers/DistanceComputer.cpp", # lib - isax @@ -118,10 +119,22 @@ def build_extensions(self): "lib/isax/iSAXIndex.cpp", "lib/isax/iSAXPqueue.cpp", "lib/isax/iSAXSearch.cpp", + # lib - ds_tree (Hercules builds on it) + "lib/ds_tree/ds_tree_index.cpp", + "lib/ds_tree/ds_tree_search.cpp", # lib - utils "lib/utils/TimerManager.cpp", # lib - algos + # Every class pybinds/setup.cpp binds unconditionally needs its + # translation unit here, or the extension builds and then fails to load + # on the first undefined typeinfo. Sofa and Sing are the exceptions: + # their bindings sit behind SOFA_FFTW_ENABLED and SING_CUDA_ENABLED, + # neither of which this build defines as non-zero. "lib/algos/Bruteforce.cpp", + "lib/algos/Coconut.cpp", + "lib/algos/DumpyOS.cpp", + "lib/algos/Fresh.cpp", + "lib/algos/Hercules.cpp", "lib/algos/LbBruteforce.cpp", "lib/algos/Messi.cpp", "lib/algos/ParIS.cpp",