diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index e2d300f..80eb11d 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -67,11 +67,32 @@ jobs: - name: Build distributions if: steps.version_check.outputs.should_publish == 'true' + env: + DAISY_PORTABLE_BUILD: "1" run: python -m build + # Keep these steps in step with release-check.yml, or the dry-run stops + # rehearsing what actually gets uploaded. + - name: Repair the Linux wheel + if: steps.version_check.outputs.should_publish == 'true' + run: | + set -euo pipefail + pip install auditwheel patchelf + mkdir -p wheelhouse + for wheel in dist/*.whl; do + auditwheel repair "$wheel" --wheel-dir wheelhouse + done + rm dist/*.whl + mv wheelhouse/*.whl dist/ + ls -l dist/ + - name: Verify distributions if: steps.version_check.outputs.should_publish == 'true' - run: twine check dist/* + run: twine check --strict dist/* + + - name: Verify artifacts are publishable + if: steps.version_check.outputs.should_publish == 'true' + run: python scripts/check_release_artifacts.py --dist-dir dist - name: Publish to PyPI if: steps.version_check.outputs.should_publish == 'true' diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml new file mode 100644 index 0000000..3f48924 --- /dev/null +++ b/.github/workflows/release-check.yml @@ -0,0 +1,167 @@ +name: Release dry-run + +# Rehearses the PyPI release on every pull request, so a release that would be +# rejected is caught here instead of on a push to main. It mirrors the +# environment and the build steps of pypi-release.yml exactly, but stops short +# of `twine upload`. +# +# Note that this covers ground ci.yml does not: ci.yml builds via CMake with +# -DBUILD_SING=OFF, while setup.py always compiles every source in the package. +# A file CMake skips can therefore break the release build while CI stays green. +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: {} + +jobs: + dry-run: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + # Kept in step with pypi-release.yml: the release publishes a single + # cp312 wheel, so that is the interpreter the rehearsal has to use. + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake libomp-dev + + - name: Install Python build dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_DaiSy.txt + + # The step that would have caught the current breakage: `python -m build` + # compiles the full extension, including sources CMake leaves out. + # The output is teed to a file so a compiler error stays downloadable as + # an artifact; the raw job log needs a GitHub token to read back. + - name: Build distributions + env: + DAISY_PORTABLE_BUILD: "1" + run: | + set -o pipefail + python -m build 2>&1 | tee build.log + + # `python -m build` tags a Linux wheel linux_x86_64, which PyPI refuses. + # auditwheel rewrites the tag to the glibc the wheel actually needs and + # vendors in the shared libraries it links, libgomp among them. + - name: Repair the Linux wheel + run: | + set -euo pipefail + # auditwheel shells out to patchelf and wants >= 0.14; the PyPI + # package ships a current binary, so we do not depend on whatever + # the runner image happens to carry. + pip install auditwheel patchelf + mkdir -p wheelhouse + for wheel in dist/*.whl; do + auditwheel repair "$wheel" --wheel-dir wheelhouse + done + rm dist/*.whl + mv wheelhouse/*.whl dist/ + ls -l dist/ + + # gcc prints one error and then many follow-on lines, so surface the + # first errors in the job summary instead of making people scroll. + - name: Summarise build errors + if: failure() + run: | + { + echo "### First errors from \`python -m build\`" + echo '```' + grep -E -m 20 '(^|:)[0-9]+:[0-9]+: error:|^error:|^ERROR' build.log || echo "no error lines matched; see the build-log artifact" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Verify distribution metadata + run: twine check --strict dist/* + + - name: Verify artifacts are publishable + run: python scripts/check_release_artifacts.py --dist-dir dist --check-pypi + + # Installing the built wheel rather than the source tree is what makes + # this a packaging test: it catches a package that compiles but ships + # without its Python files or its compiled extension. + - name: Install the built wheel into a clean environment + run: | + python -m venv /tmp/wheel-test + /tmp/wheel-test/bin/pip install --upgrade pip + /tmp/wheel-test/bin/pip install dist/*.whl + + # daisy/__init__.py turns a failed extension load into a warning, and the + # next import of daisy._core then reports "already registered" rather than + # the original cause. Exec the .so on its own, in its own process, so a + # broken extension reports the error it actually raised. + - name: Check the extension initialises + run: | + cd /tmp + /tmp/wheel-test/bin/python - <<'PY' + import glob + import importlib.util + import os + + # find_spec locates the package without running its __init__.py. + spec = importlib.util.find_spec("daisy") + assert spec is not None and spec.origin, "daisy is not importable at all" + + matches = glob.glob(os.path.join(os.path.dirname(spec.origin), "_core*.so")) + assert matches, "the wheel installed no _core extension" + + core_spec = importlib.util.spec_from_file_location("_core", matches[0]) + module = importlib.util.module_from_spec(core_spec) + core_spec.loader.exec_module(module) + + print("extension initialised:", matches[0]) + print("exported:", sorted(n for n in dir(module) if n[0].isupper())) + PY + + - name: Smoke-test the installed package + run: | + cd /tmp + /tmp/wheel-test/bin/python - <<'PY' + import numpy as np + + import daisy + from daisy import _core + + print("daisy imported from", daisy.__file__) + print("extension imported from", _core.__file__) + + rng = np.random.default_rng(0) + database = rng.random((256, 64), dtype=np.float32) + # searchIndex takes a 2D batch of queries, one row per query. + query = database[7:8].copy() + + search = daisy.BruteForceSearch(daisy.DistanceType.L2_SQUARED) + search.buildIndex(database) + indices, distances = search.searchIndex(query, 5) + + print("top-5 indices:", indices[0]) + print("top-5 distances:", distances[0]) + + assert indices[0][0] == 7, "nearest neighbour of a database row must be itself, got %r" % (indices[0][0],) + assert distances[0][0] < 1e-4, "self-distance should be ~0, got %r" % (distances[0][0],) + print("smoke test passed") + PY + + - name: Upload distributions and build log for inspection + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-check-${{ github.sha }} + path: | + dist/ + build.log + if-no-files-found: warn diff --git a/MANIFEST.in b/MANIFEST.in index 53fe9fc..8df8769 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,9 @@ include README.md include requirements_DaiSy.txt recursive-include daisy *.py -recursive-include lib *.hpp *.cpp -recursive-include commons *.hpp *.cpp -recursive-include pybinds *.cpp +recursive-include lib *.h *.hpp *.cpp +recursive-include commons *.h *.hpp *.cpp +recursive-include pybinds *.h *.hpp *.cpp recursive-exclude build * recursive-exclude DaiSy_env * recursive-exclude faiss_env * diff --git a/daisy/__init__.py b/daisy/__init__.py index e3cb0fd..7002cc8 100644 --- a/daisy/__init__.py +++ b/daisy/__init__.py @@ -17,7 +17,7 @@ (distance-r) search via searchIndex(query, SearchConfig(type=QueryType.RANGE, r=r)). """ -__version__ = "1.0.0" +__version__ = "1.0.9" __author__ = "" try: diff --git a/lib/algos/Sing.cpp b/lib/algos/Sing.cpp index 8d532f6..5b80059 100644 --- a/lib/algos/Sing.cpp +++ b/lib/algos/Sing.cpp @@ -36,6 +36,11 @@ namespace daisy #else void LBDfloatstreamGPU(sax_type *gsaxarray, float *positionmap, ts_type *paa, float *gqts, float bsf, unsigned long size, float *gpositionmap, int paa_segments, float mindist_sqrt); + void LBDshortstreamGPUinsidedynamicratev2dtw(sax_type *gsaxarray, short *positionmap, + ts_type *paaU, ts_type *paaL, float *gqtsU, float *gqtsL, + float bsf, unsigned long size, short *gpositionmap, + int paa_segments, float mindist_sqrt, unsigned long *gpuoffset, + float shortrate, int chunknumber, bool *activechunk); #endif first_buffer_layer2 *initialize_simrec(int initial_buffer_size, int number_of_buffers, int max_total_buffers_size, isax_index *index); @@ -1363,7 +1368,34 @@ namespace daisy (void)gpositionmap; (void)paa_segments; (void)mindist_sqrt; - + + } + + // The DTW search stages this on the GPU between two barriers, so the CPU + // build has to leave gpuoffset alone: the worker threads are already + // running and hold a pointer to it. + void LBDshortstreamGPUinsidedynamicratev2dtw(sax_type *gsaxarray, short *positionmap, + ts_type *paaU, ts_type *paaL, float *gqtsU, float *gqtsL, + float bsf, unsigned long size, short *gpositionmap, + int paa_segments, float mindist_sqrt, unsigned long *gpuoffset, + float shortrate, int chunknumber, bool *activechunk) + { + (void)gsaxarray; + (void)positionmap; + (void)paaU; + (void)paaL; + (void)gqtsU; + (void)gqtsL; + (void)bsf; + (void)size; + (void)gpositionmap; + (void)paa_segments; + (void)mindist_sqrt; + (void)gpuoffset; + (void)shortrate; + (void)chunknumber; + (void)activechunk; + } #endif diff --git a/lib/utils/SIMD.hpp b/lib/utils/SIMD.hpp index 9b4b183..b3a7f7d 100644 --- a/lib/utils/SIMD.hpp +++ b/lib/utils/SIMD.hpp @@ -13,8 +13,10 @@ #include #include -// Detect AVX support -#if defined(__AVX__) || defined(_M_AVX) +// Detect AVX2 support. The guarded code uses AVX2 intrinsics (_mm256_srlv_epi32, +// _mm256_cvtepu8_epi16 and friends), so testing for plain __AVX__ turned a CPU +// with AVX but no AVX2 into a compile error instead of the scalar fallback. +#if defined(__AVX2__) #define DAISY_SIMD_AVAILABLE 1 #include #else diff --git a/pybinds/setup.cpp b/pybinds/setup.cpp index 7a4193c..c7ad9b6 100644 --- a/pybinds/setup.cpp +++ b/pybinds/setup.cpp @@ -721,7 +721,10 @@ PYBIND11_MODULE(_core, m) .def_readwrite("flush_buffer_size", &daisy::HerculesConfig::flush_buffer_size) .def_readwrite("index_dir", &daisy::HerculesConfig::index_dir); - pybind11::class_(m, "Hercules", "Hercules hierarchical time series similarity index") + // Bound without SimilaritySearchAlgorithm: the base is abstract and never + // registered, and naming it here makes module init fail with "referenced + // unknown base type". Every other algorithm is bound the same way. + pybind11::class_(m, "Hercules", "Hercules hierarchical time series similarity index") .def(pybind11::init(), "Create a new Hercules with the given distance metric") .def(pybind11::init(), "Create a new Hercules with the given distance metric and configuration") .def("setNumThreads", &daisy::Hercules::setNumThreads, "Set the number of query threads") @@ -770,7 +773,7 @@ PYBIND11_MODULE(_core, m) .def_readwrite("fill_lower", &daisy::DumpyOSConfig::fill_lower) .def_readwrite("fill_upper", &daisy::DumpyOSConfig::fill_upper); - pybind11::class_(m, "DumpyOS", "DumpyOS iSAX-based multi-ary adaptive time series similarity index") + pybind11::class_(m, "DumpyOS", "DumpyOS iSAX-based multi-ary adaptive time series similarity index") .def(pybind11::init(), "Create a new DumpyOS with the given distance metric") .def(pybind11::init(), "Create a new DumpyOS with the given distance metric and configuration") .def("setNumThreads", &daisy::DumpyOS::setNumThreads, "Set the number of threads") diff --git a/pyproject.toml b/pyproject.toml index 8a95f63..02edcf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "daisy-exact-search" -version = "1.0.8" +version = "1.0.9" description = "DaiSy: A Library for Scalable Data Series Similarity Search" readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/check_release_artifacts.py b/scripts/check_release_artifacts.py new file mode 100644 index 0000000..b92c089 --- /dev/null +++ b/scripts/check_release_artifacts.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Validate the distributions in dist/ before they are uploaded to PyPI. + +PyPI rejects an upload for reasons that a successful `python -m build` does not +catch (bad platform tag, version already taken, metadata mismatch). Twine only +uploads after building, so a rejection wastes the whole release run and, worse, +can leave a half-published release when some files upload and others do not. + +This script runs the checks locally so a broken release is caught in CI instead. + +Usage: + python scripts/check_release_artifacts.py [--dist-dir dist] [--check-pypi] + +Exit code 0 means the artifacts look publishable. +""" + +import argparse +import json +import re +import sys +import tarfile +import urllib.error +import urllib.request +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +PYPI_NAME = "daisy-exact-search" + +# Platform tags PyPI accepts for a binary wheel. A wheel built by a plain +# `python -m build` on Linux is tagged `linux_x86_64`, which PyPI refuses: +# it carries no glibc floor, so pip cannot tell whether it will run. +ACCEPTED_PLATFORM_TAGS = ("any", "manylinux", "musllinux", "macosx", "win") + +errors = [] +warnings = [] + + +def fail(msg): + errors.append(msg) + print("FAIL: " + msg) + + +def warn(msg): + warnings.append(msg) + print("WARN: " + msg) + + +def ok(msg): + print("ok: " + msg) + + +def read_declared_versions(): + """Return the version declared in each place the project records one.""" + versions = {} + + pyproject = (PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8") + match = re.search(r'^version\s*=\s*"([^"]+)"', pyproject, re.MULTILINE) + if match: + versions["pyproject.toml"] = match.group(1) + + setup_py = (PROJECT_ROOT / "setup.py").read_text(encoding="utf-8") + match = re.search(r'^__version__\s*=\s*"([^"]+)"', setup_py, re.MULTILINE) + if match: + versions["setup.py"] = match.group(1) + + init_py = (PROJECT_ROOT / "daisy" / "__init__.py").read_text(encoding="utf-8") + match = re.search(r'^__version__\s*=\s*"([^"]+)"', init_py, re.MULTILINE) + if match: + versions["daisy/__init__.py"] = match.group(1) + + return versions + + +def check_versions(): + """setup.py and pyproject.toml both feed the build, so they must agree.""" + versions = read_declared_versions() + + for source in ("pyproject.toml", "setup.py"): + if source not in versions: + fail("could not find a version declaration in " + source) + + build_versions = {k: v for k, v in versions.items() if k != "daisy/__init__.py"} + if len(set(build_versions.values())) > 1: + fail( + "version mismatch between build files: " + + ", ".join(k + "=" + v for k, v in build_versions.items()) + ) + else: + ok("build files agree on version " + next(iter(build_versions.values()))) + + # Not a publish blocker, but users read daisy.__version__ at runtime. + runtime_version = versions.get("daisy/__init__.py") + if runtime_version and runtime_version not in build_versions.values(): + warn( + "daisy/__init__.py declares __version__ = " + + runtime_version + + " but the package is built as " + + next(iter(build_versions.values())) + ) + + return next(iter(build_versions.values())) + + +def check_artifacts_present(dist_dir, version): + sdists = sorted(dist_dir.glob("*.tar.gz")) + wheels = sorted(dist_dir.glob("*.whl")) + + if not sdists: + fail("no sdist (*.tar.gz) in " + str(dist_dir)) + elif len(sdists) > 1: + fail("more than one sdist in " + str(dist_dir) + "; clean it before building") + else: + ok("found sdist " + sdists[0].name) + + if not wheels: + fail("no wheel (*.whl) in " + str(dist_dir)) + else: + ok("found " + str(len(wheels)) + " wheel(s): " + ", ".join(w.name for w in wheels)) + + for artifact in sdists + wheels: + # Wheel and sdist filenames normalise '-' to '_' in the project name. + if version.replace("-", "_") not in artifact.name: + fail(artifact.name + " does not carry the declared version " + version) + + return sdists, wheels + + +def check_wheel_tags(wheels): + """Reject wheels PyPI will refuse, before we spend a release run finding out.""" + for wheel in wheels: + parts = wheel.stem.split("-") + if len(parts) < 5: + fail(wheel.name + " is not a valid wheel filename") + continue + + platform_tag = parts[-1] + if platform_tag.startswith(ACCEPTED_PLATFORM_TAGS): + ok(wheel.name + " has a PyPI-acceptable platform tag: " + platform_tag) + else: + fail( + wheel.name + + " has platform tag '" + + platform_tag + + "', which PyPI rejects. Run the wheel through `auditwheel repair` " + "(or build it with cibuildwheel) to get a manylinux tag." + ) + + +def check_sdist_contents(sdist): + """The sdist is what users on unsupported platforms compile from.""" + required = [ + "pyproject.toml", + "setup.py", + "README.md", + "daisy/__init__.py", + "pybinds/setup.cpp", + ] + + with tarfile.open(sdist, "r:gz") as tar: + # Every path is prefixed with the '-/' root directory. + names = {name.split("/", 1)[-1] for name in tar.getnames()} + + missing = [path for path in required if path not in names] + if missing: + fail(sdist.name + " is missing: " + ", ".join(missing)) + else: + ok(sdist.name + " contains all required files") + + # setup.py compiles these unconditionally; if MANIFEST.in drops one, the + # sdist builds here but fails for anyone installing from source. + sources = [n for n in names if n.startswith(("lib/", "commons/")) and n.endswith((".cpp", ".hpp"))] + if not sources: + fail(sdist.name + " contains no C++ sources; check MANIFEST.in") + else: + ok(sdist.name + " contains " + str(len(sources)) + " C++ source/header files") + + +def check_pypi(version): + """Warn if this version is already on PyPI; re-uploading is a hard error.""" + url = "https://pypi.org/pypi/" + PYPI_NAME + "/json" + try: + with urllib.request.urlopen(url, timeout=30) as response: + data = json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + ok(PYPI_NAME + " is not on PyPI yet; any version can be published") + return + warn("could not query PyPI (HTTP " + str(exc.code) + "); skipping duplicate check") + return + except Exception as exc: + warn("could not query PyPI (" + str(exc) + "); skipping duplicate check") + return + + published = set(data.get("releases", {})) + if version in published: + warn( + "version " + + version + + " is already published; the release job will skip the upload. " + "Bump the version to publish." + ) + else: + ok("version " + version + " is not on PyPI yet") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dist-dir", default="dist", help="directory holding the built distributions") + parser.add_argument("--check-pypi", action="store_true", help="also query PyPI for the version") + args = parser.parse_args() + + dist_dir = Path(args.dist_dir).resolve() + if not dist_dir.is_dir(): + print("FAIL: " + str(dist_dir) + " does not exist; run `python -m build` first") + return 1 + + print("Checking release artifacts in " + str(dist_dir)) + print() + + version = check_versions() + sdists, wheels = check_artifacts_present(dist_dir, version) + check_wheel_tags(wheels) + for sdist in sdists: + check_sdist_contents(sdist) + if args.check_pypi: + check_pypi(version) + + print() + if errors: + print(str(len(errors)) + " check(s) failed; these artifacts are not safe to upload.") + return 1 + + print("All checks passed" + (" (" + str(len(warnings)) + " warning(s))" if warnings else "") + ".") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py index ee7bd93..26a63a5 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ from setuptools.command.build_ext import build_ext as _build_ext # Version -__version__ = "1.0.8" +__version__ = "1.0.9" # Detect platform IS_MACOS = sys.platform == "darwin" @@ -111,6 +111,7 @@ def build_extensions(self): "commons/common.cpp", "commons/dataloaders.cpp", "commons/paramSetup.cpp", + "commons/VectorDataLoader.cpp", # lib - distance computers "lib/distance_computers/DistanceComputer.cpp", # lib - isax @@ -118,10 +119,22 @@ def build_extensions(self): "lib/isax/iSAXIndex.cpp", "lib/isax/iSAXPqueue.cpp", "lib/isax/iSAXSearch.cpp", + # lib - ds_tree (Hercules builds on it) + "lib/ds_tree/ds_tree_index.cpp", + "lib/ds_tree/ds_tree_search.cpp", # lib - utils "lib/utils/TimerManager.cpp", # lib - algos + # Every class pybinds/setup.cpp binds unconditionally needs its + # translation unit here, or the extension builds and then fails to load + # on the first undefined typeinfo. Sofa and Sing are the exceptions: + # their bindings sit behind SOFA_FFTW_ENABLED and SING_CUDA_ENABLED, + # neither of which this build defines as non-zero. "lib/algos/Bruteforce.cpp", + "lib/algos/Coconut.cpp", + "lib/algos/DumpyOS.cpp", + "lib/algos/Fresh.cpp", + "lib/algos/Hercules.cpp", "lib/algos/LbBruteforce.cpp", "lib/algos/Messi.cpp", "lib/algos/ParIS.cpp", @@ -180,8 +193,25 @@ def build_extensions(self): link_args.append("-fopenmp") # SIMD optimizations (not reliable on ARM64 Macs) + # + # -march=native tunes for the machine doing the build, which is what you + # want when compiling from source but not when building a wheel for other + # people: SIMD.hpp picks its code path at compile time, so a wheel built on + # a runner with a wider instruction set dies with SIGILL elsewhere. Set + # DAISY_PORTABLE_BUILD=1 when producing a redistributable wheel; -mavx + # stays, so DAISY_SIMD_AVAILABLE is still 1. + PORTABLE_BUILD = os.environ.get("DAISY_PORTABLE_BUILD", "0").lower() in ("1", "true", "yes") + if not (IS_MACOS and platform.machine() in ("arm64", "aarch64")): - compile_args.extend(["-mavx", "-march=native"]) + compile_args.append("-mavx") + if PORTABLE_BUILD: + # The guarded code needs AVX2, not just AVX, so a portable wheel has + # to name AVX2 explicitly once -march=native is gone. That puts the + # floor at Haswell (2013). + print("[DaiSy] Portable build requested - targeting AVX2 instead of -march=native") + compile_args.append("-mavx2") + else: + compile_args.append("-march=native") # Get the project root directory as absolute path project_root = str(Path(__file__).parent.absolute())