diff --git a/.github/workflows/blosc1-compat.yaml b/.github/workflows/blosc1-compat.yaml new file mode 100644 index 00000000..e348e23a --- /dev/null +++ b/.github/workflows/blosc1-compat.yaml @@ -0,0 +1,97 @@ +name: Blosc1 compatibility + +# Property-based and exhaustive tests (tests/test_blosc1_compat.py) that chunks written by +# numcodecs' Blosc decode with C-Blosc 1.x, and the reverse, using python-blosc (which +# bundles C-Blosc 1.x) as the reference. +# +# This workflow is self-contained and optional. To turn it off, any of: +# * set the repository variable BLOSC1_COMPAT_CI to 'false' +# (Settings > Secrets and variables > Actions > Variables), without a code change; +# * delete this file; nothing else refers to it. tests/test_blosc1_compat.py skips itself +# when python-blosc or hypothesis is missing, so the other workflows are unaffected; +# * run pytest elsewhere with -m "not blosc1_compat". + +on: + push: + branches: [main] + paths: + - '.github/workflows/blosc1-compat.yaml' + - 'tests/test_blosc1_compat.py' + - 'src/numcodecs/blosc.pyx' + - 'src/numcodecs/_blosc_shim.h' + - 'src/numcodecs/meson.build' + - 'subprojects/**' + - 'packages/c-blosc2-blosc1-compat/**' + - 'c-blosc' + pull_request: + paths: + - '.github/workflows/blosc1-compat.yaml' + - 'tests/test_blosc1_compat.py' + - 'src/numcodecs/blosc.pyx' + - 'src/numcodecs/_blosc_shim.h' + - 'src/numcodecs/meson.build' + - 'subprojects/**' + - 'packages/c-blosc2-blosc1-compat/**' + - 'c-blosc' + schedule: + - cron: '0 5 * * 1' # weekly, with many more examples than a pull request + workflow_dispatch: + inputs: + examples: + description: 'Hypothesis examples per property' + default: '1000' + +permissions: + contents: read + +jobs: + compat: + # the repository variable is the off switch + if: vars.BLOSC1_COMPAT_CI != 'false' + name: ${{ matrix.backend }} backend, ${{ matrix.platform }} + runs-on: ${{ matrix.platform }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + # the c-blosc 1.x backend is the control: the tests must pass on it too, which + # shows that they test the format and not an implementation detail + - {backend: blosc1, platform: ubuntu-latest} + - {backend: blosc2, platform: ubuntu-latest} + - {backend: blosc2, platform: ubuntu-22.04-arm} + - {backend: blosc2, platform: macos-14} + - {backend: blosc2, platform: windows-latest} + defaults: + run: + shell: bash -e {0} + env: + BLOSC1_COMPAT_EXAMPLES: ${{ inputs.examples || (github.event_name == 'schedule' && '5000') || '300' }} + steps: + - uses: actions/checkout@v6 + with: + # pyproject.toml's license-files lists the c-blosc submodule's licenses, and the + # blosc1 backend builds it + submodules: recursive + fetch-depth: 0 # required for version resolution + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + cache: 'pip' + - name: Install numcodecs with the ${{ matrix.backend }} backend + run: python -m pip install -v ".[test]" -Csetup-args=-Dblosc_backend=${{ matrix.backend }} + - name: Install the C-Blosc 1.x reference + # python-blosc has wheels for most platforms and builds from source (with cmake) + # for the rest + run: python -m pip install blosc hypothesis cmake + - name: Versions + run: | + python - <<'EOF' + import blosc, hypothesis, numcodecs.blosc as nb + print("numcodecs backend:", nb.LIBRARY, nb.VERSION_STRING, nb.list_compressors()) + print("reference:", "python-blosc", blosc.__version__, "C-Blosc", blosc.VERSION_STRING) + print("hypothesis:", hypothesis.__version__) + assert nb.LIBRARY == ("c-blosc2" if "${{ matrix.backend }}" == "blosc2" else "c-blosc") + EOF + - name: Run the compatibility tests + run: pytest tests/test_blosc1_compat.py -m blosc1_compat --no-cov -v -p no:cacheprovider diff --git a/.github/workflows/blosc2-compat.yaml b/.github/workflows/blosc2-compat.yaml new file mode 100644 index 00000000..582411f8 --- /dev/null +++ b/.github/workflows/blosc2-compat.yaml @@ -0,0 +1,85 @@ +name: c-blosc2 Blosc1 compatibility + +on: + push: + branches: [main] + paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h'] + pull_request: + paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h'] + workflow_dispatch: + +jobs: + wrap-in-sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: python packages/c-blosc2-blosc1-compat/tools/sync_wrap.py --check + + forward-compat: + # c-blosc2 chunks must be readable by C-Blosc 1.x + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: pip install meson ninja cmake + - name: Build C-Blosc 1.x + run: | + git clone --depth 1 --branch v1.21.6 https://github.com/Blosc/c-blosc.git /tmp/c-blosc + cmake -S /tmp/c-blosc -B /tmp/b1 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release + cmake --build /tmp/b1 -j + - name: Build c-blosc2 through the wrap + run: | + mkdir /tmp/probe && cd /tmp/probe + ln -s "$GITHUB_WORKSPACE/subprojects" subprojects + printf "project('probe','c')\nb = dependency('blosc2')\nexecutable('probe', files('p.c'), dependencies: b)\n" > meson.build + printf '#include \nint main(void){return 0;}\n' > p.c + meson setup b && meson compile -C b + - name: Run the compatibility check + run: | + B=/tmp/probe/b + # c-blosc2 uses a system zstd/lz4/zlib when it finds one, otherwise the wrap's + # static build; link whichever is there + LIBS="$(find $B/subprojects -name libblosc2.a | head -n 1)" + for l in zstd lz4 z; do + a="$(find $B/subprojects -name lib$l.a | head -n 1)" + LIBS="$LIBS ${a:--l$l}" + done + python packages/c-blosc2-blosc1-compat/tests/run.py \ + --blosc1-include /tmp/c-blosc/blosc --blosc1-lib /tmp/b1/blosc \ + --blosc2-include "/tmp/probe/subprojects/c-blosc2/include" \ + --blosc2-ldflags "$LIBS" + + test-blosc2-backend: + # the numcodecs test suite with Blosc built on c-blosc2 instead of c-blosc 1.x + runs-on: ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + - {platform: ubuntu-latest, python-version: "3.13"} + - {platform: ubuntu-latest, python-version: "3.14t"} + - {platform: ubuntu-22.04-arm, python-version: "3.13"} + - {platform: macos-14, python-version: "3.13"} + - {platform: windows-latest, python-version: "3.13"} + defaults: + run: + shell: bash -e {0} + steps: + - uses: actions/checkout@v6 + with: + # pyproject.toml's license-files still lists the c-blosc submodule's licenses + submodules: recursive + fetch-depth: 0 # required for version resolution + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install numcodecs with the c-blosc2 backend + run: python -m pip install -v ".[test,test_extras]" -Csetup-args=-Dblosc_backend=blosc2 + - name: Check the backend + run: python -c "import numcodecs.blosc as b; print(b.LIBRARY, b.VERSION_STRING, b.list_compressors()); assert b.LIBRARY == 'c-blosc2'" + - name: Run tests + run: pytest -v diff --git a/.gitignore b/.gitignore index 14facfe7..2fbc0cc9 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,9 @@ ENV/ .pixi/* *.egg-info pixi.lock -uv.lock \ No newline at end of file +uv.lock +# Wrapped subprojects: only the .wrap files and packagefiles are tracked +/subprojects/*/ +!/subprojects/packagefiles/ +/subprojects/packagecache/ +/subprojects/.wraplock diff --git a/docs/release.md b/docs/release.md index ee770933..2dc29904 100644 --- a/docs/release.md +++ b/docs/release.md @@ -18,6 +18,21 @@ ## 0.17.1 +### Enhancements + +* Add an optional c-blosc2 backend for the `Blosc` codec, selected at build time with + `-Dblosc_backend=blosc2` (default is still c-blosc 1.x). c-blosc2 is built from a meson + wrap (`subprojects/c-blosc2.wrap`) of a c-blosc2 branch that can write Blosc1-format + chunks that C-Blosc 1.x can decode (stock c-blosc2 output is not readable by C-Blosc 1.x). + zstd, lz4 and zlib are resolved separately from Blosc (system libraries or WrapDB wraps, + `-Dsystem_*`) and can be left out of Blosc with `-Dblosc_zstd=disabled` and + `-Dblosc_zlib=disabled`. `numcodecs.blosc.LIBRARY` reports the library in use; c-blosc2 + does not provide `snappy`. +* Validate Blosc frames before decompression, allowing valid zero-length frames to round-trip + while rejecting truncated frames, by {user}`Igor Stadnyk `, {issue}`831` (#854). +* `Blosc` can encode arrays whose dtype has an itemsize of 0, by + {user}`Raashish Aggarwal `, {issue}`516` (#870). + ### Security * `Pickle.decode` now refuses to unpickle data unless the process opts in, because diff --git a/meson.options b/meson.options index f055af87..e353f976 100644 --- a/meson.options +++ b/meson.options @@ -1,5 +1,11 @@ +option('blosc_backend', type: 'combo', choices: ['blosc1', 'blosc2'], value: 'blosc1', + description: 'C library behind the Blosc codec. Both write Blosc1-format chunks.') option('system_blosc', type: 'feature', value: 'disabled', - description: 'Use system-installed Blosc library') + description: 'Use system-installed Blosc library (c-blosc 1.x for blosc_backend=blosc1, c-blosc2 for blosc2)') +option('blosc_zstd', type: 'feature', value: 'enabled', + description: 'blosc_backend=blosc2 only: build Zstandard support into Blosc') +option('blosc_zlib', type: 'feature', value: 'enabled', + description: 'blosc_backend=blosc2 only: build zlib support into Blosc') option('system_lz4', type: 'feature', value: 'disabled', description: 'Use system-installed LZ4 library') option('system_zlib', type: 'feature', value: 'disabled', diff --git a/packages/c-blosc2-blosc1-compat/README.md b/packages/c-blosc2-blosc1-compat/README.md new file mode 100644 index 00000000..4e99adad --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/README.md @@ -0,0 +1,93 @@ +# c-blosc2-blosc1-compat + +A build overlay for [c-blosc2](https://github.com/Blosc/c-blosc2) that makes it usable as the +backend of numcodecs' `Blosc` codec: a meson build (upstream only has CMake) for a c-blosc2 +branch that can **write chunks that C-Blosc 1.x can decode**. Stock c-blosc2 cannot, even +through its `blosc1_*` API or with `BLOSC_BLOSC1_COMPAT` (its README says so, and +`tests/run.py` confirms it). + +This directory is self-contained (it does not import anything from numcodecs) so it can be +split out with `git subtree split` if it ever needs to live in its own repository. + +## Contents + +| Path | What | +|---|---| +| `UPSTREAM` | the c-blosc2 branch and commit in use; `subprojects/c-blosc2.wrap` must match | +| `meson/` | `meson.build` and `meson.options` for c-blosc2 | +| `tests/` | the forward-compatibility check, see below | +| `tools/` | `sync_wrap.py` | +| `TODO.md` | what to do once the c-blosc2 work is released | + +## The c-blosc2 branch + +`subprojects/c-blosc2.wrap` clones the `blosc1-format-writer` branch of +[mkitti/c-blosc2](https://github.com/mkitti/c-blosc2), pinned to a commit. The branch is +upstream c-blosc2 plus the change proposed in +[Blosc/c-blosc2#815](https://github.com/Blosc/c-blosc2/pull/815): +`blosc2_ctx_set_blosc1_format()`, which makes a compression context write format version 2 +(Blosc1) chunks. Format version 2 differs from what c-blosc2 writes in three ways that +matter to a reader: + +1. the format version byte and the 16-byte header; +2. bitshuffle: version 2 shuffles a block only if its element count is a multiple of 8, and + stores it unshuffled otherwise; +3. block splitting: a C-Blosc 1.x reader recomputes it from the typesize and blocksize. + +Only plain shuffle/bitshuffle with blosclz, lz4, lz4hc, zlib and zstd are supported in this +mode. The output decodes with C-Blosc 1.x but is not byte-identical to what it writes. + +The reader fix for bit-shuffled version 2 blocks with leftover bytes +([#812](https://github.com/Blosc/c-blosc2/issues/812), +[#813](https://github.com/Blosc/c-blosc2/pull/813)) is already in upstream c-blosc2 and so +in the branch. + +## Wiring into the build + +Meson only looks for `patch_directory` in `subprojects/packagefiles`, so +`subprojects/packagefiles/c-blosc2/` holds a copy of `meson/`. After editing here run +`python tools/sync_wrap.py`; CI runs it with `--check`. + +## Testing + +`tests/run.py` builds small C programs against C-Blosc 1.x and c-blosc2 and checks 735 +cases in three directions (see its docstring for the command line): c-blosc2 global-context +to C-Blosc 1.x, c-blosc2 context API to C-Blosc 1.x, and C-Blosc 1.x to c-blosc2. All must +decode. The c-blosc2 branch has its own unit tests (`tests/test_blosc1_format.c`). + +## Compatibility CI + +`tests/test_blosc1_compat.py` (repository `tests/`, not this directory) checks that chunks +written by numcodecs decode with C-Blosc 1.x and the reverse, with python-blosc, which +bundles C-Blosc 1.x, as the reference. It combines property-based tests (hypothesis) over +compressors, levels, shuffles, typesizes up to 300, buffer sizes around the edges that +matter (32 bytes, multiples of 8 and 128 elements, bytes left over after the last item), +block sizes and data kinds, with an exhaustive grid. Both numcodecs code paths are checked: +the one through blosc's global context and the one through a context of its own. The +compressed bytes are not expected to equal what C-Blosc 1.x writes (the libraries pick +different default block sizes); the decoded bytes are. + +`.github/workflows/blosc1-compat.yaml` runs it on the c-blosc2 backend on Linux (x86_64 and +arm64), macOS and Windows, and on the default c-blosc 1.x backend as a control. It runs 300 +examples per property on a pull request and 5000 weekly. It is independent of everything +else and can be turned off by + +- setting the repository variable `BLOSC1_COMPAT_CI` to `false` (no code change), +- deleting the workflow file (the test module skips itself when python-blosc or hypothesis + is not installed, so no other workflow changes), or +- `pytest -m "not blosc1_compat"` anywhere else. + +To run it locally: `pip install blosc hypothesis`, then +`pytest tests/test_blosc1_compat.py` (`BLOSC1_COMPAT_EXAMPLES` sets the examples). + +## Updating c-blosc2 + +Edit `UPSTREAM` and the `revision` (and `url`, if the branch moves) in +`subprojects/c-blosc2.wrap` together; a test checks that they agree. Then run +`python tools/sync_wrap.py` and the tests. If upstream changes its source file list, update +`meson/meson.build`. + +## Upstreaming + +The wrap depends on a personal fork until Blosc/c-blosc2#815 is merged and released; see +`TODO.md`. diff --git a/packages/c-blosc2-blosc1-compat/TODO.md b/packages/c-blosc2-blosc1-compat/TODO.md new file mode 100644 index 00000000..80cf696d --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/TODO.md @@ -0,0 +1,23 @@ +# TODO + +## Move the wrap off the fork once Blosc/c-blosc2#815 is released + +`subprojects/c-blosc2.wrap` clones the `blosc1-format-writer` branch of +https://github.com/mkitti/c-blosc2 (see `UPSTREAM`). The change is proposed upstream in +https://github.com/Blosc/c-blosc2/pull/815 (it builds on #812 / #813, already merged). + +Check periodically whether #815 has been merged and which c-blosc2 release contains it. +When there is such a release: + +1. point the wrap at it: `url = https://github.com/Blosc/c-blosc2.git`, `revision = ` (or switch back to a `[wrap-file]` with `source_url` and `source_hash`); +2. update `UPSTREAM` to match, and check that `meson/meson.build` still lists the right + source files; +3. run `python tools/sync_wrap.py` and `tests/run.py`; +4. update `docs/release.md` and `README.md`. + +If upstream asks for changes (for example to the `BLOSC_BLOSC1_COMPAT` behavior) or rejects +it, update the branch, `UPSTREAM` and the wrap's `revision` to match. + +Do not release numcodecs with the wrap pointing at the fork: a personal branch is not a +stable build dependency. diff --git a/packages/c-blosc2-blosc1-compat/UPSTREAM b/packages/c-blosc2-blosc1-compat/UPSTREAM new file mode 100644 index 00000000..15aac53e --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/UPSTREAM @@ -0,0 +1,5 @@ +repository = https://github.com/mkitti/c-blosc2 +branch = blosc1-format-writer +commit = c919bfe6834cd05ca853a4b63340b693bed9892f +version = 3.3.6.dev +pull_request = https://github.com/Blosc/c-blosc2/pull/815 diff --git a/packages/c-blosc2-blosc1-compat/meson/meson.build b/packages/c-blosc2-blosc1-compat/meson/meson.build new file mode 100644 index 00000000..ceb48c8a --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/meson/meson.build @@ -0,0 +1,130 @@ +# Meson build for c-blosc2, which upstream only builds with CMake. +# Only the core library plus the built-in codec/filter/tuner plugins are built +# (no ZFP, tests, benchmarks or examples). Used via subprojects/c-blosc2.wrap. +project( + 'c-blosc2', 'c', + version: '3.3.6.dev', + meson_version: '>=1.6.0', + default_options: ['c_std=gnu11', 'warning_level=0'], +) + +cc = meson.get_compiler('c') +host_system = host_machine.system() +cpu = host_machine.cpu_family() + +blosc2_args = ['-DBUILD_STATIC'] +if host_system != 'windows' + blosc2_args += ['-D_GNU_SOURCE'] +else + blosc2_args += ['-D_CRT_SECURE_NO_WARNINGS'] +endif + +deps = [dependency('threads')] +if host_system != 'windows' + deps += cc.find_library('m', required: false) + deps += cc.find_library('dl', required: false) +endif + +# --- compression libraries (resolved by the parent project or WrapDB) --- +lz4_dep = dependency('liblz4', fallback: ['lz4', 'liblz4_dep'], + default_options: ['default_library=static']) +deps += lz4_dep +blosc2_args += ['-DHAVE_LZ4'] + +zstd_dep = dependency('libzstd', required: get_option('zstd'), + fallback: ['zstd', 'libzstd_dep'], + default_options: ['default_library=static']) +if zstd_dep.found() + deps += zstd_dep + blosc2_args += ['-DHAVE_ZSTD'] +endif + +zlib_dep = dependency('zlib', required: get_option('zlib'), + fallback: ['zlib', 'zlib_dep'], + default_options: ['default_library=static']) +if zlib_dep.found() + deps += zlib_dep + blosc2_args += ['-DHAVE_ZLIB'] +endif + +sources = files( + 'blosc/blosc2.c', 'blosc/blosclz.c', 'blosc/fastcopy.c', 'blosc/schunk.c', + 'blosc/frame.c', 'blosc/stune.c', 'blosc/delta.c', 'blosc/shuffle.c', + 'blosc/shuffle-generic.c', 'blosc/bitshuffle-generic.c', + 'blosc/trunc-prec.c', 'blosc/timestamp.c', 'blosc/sframe.c', + 'blosc/directories.c', 'blosc/blosc2-stdio.c', 'blosc/b2nd.c', + 'blosc/b2nd_utils.c', + # plugins: registries and the plugins they register (ZFP excluded) + 'plugins/codecs/codecs-registry.c', + 'plugins/codecs/ndlz/ndlz.c', 'plugins/codecs/ndlz/ndlz4x4.c', + 'plugins/codecs/ndlz/ndlz8x8.c', 'plugins/codecs/ndlz/xxhash.c', + 'plugins/filters/filters-registry.c', + 'plugins/filters/bytedelta/bytedelta.c', + 'plugins/filters/int_trunc/int_trunc.c', + 'plugins/filters/ndcell/ndcell.c', 'plugins/filters/ndmean/ndmean.c', + 'plugins/tuners/tuners-registry.c', +) +if host_system == 'windows' + sources += files('blosc/win32/threading.c') +endif + +# --- SIMD kernels: detected by compiler capability (cross-compile safe) --- +# Each kernel file gets its ISA flag; shuffle.c only learns that it may +# dispatch to them at run time (SHUFFLE_*_ENABLED) and is built portably. +shuffle_defs = [] +simd_libs = [] +msvc = cc.get_argument_syntax() == 'msvc' + +if cpu in ['x86', 'x86_64'] + if not get_option('sse2').disabled() and cc.has_argument(msvc ? '/arch:SSE2' : '-msse2') + simd_libs += static_library('blosc2_sse2', + files('blosc/shuffle-sse2.c', 'blosc/bitshuffle-sse2.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + (msvc ? [] : ['-msse2']), + dependencies: deps) + shuffle_defs += '-DSHUFFLE_SSE2_ENABLED' + endif + if not get_option('avx2').disabled() and cc.has_argument(msvc ? '/arch:AVX2' : '-mavx2') + simd_libs += static_library('blosc2_avx2', + files('blosc/shuffle-avx2.c', 'blosc/bitshuffle-avx2.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + (msvc ? ['/arch:AVX2'] : ['-mavx2']), + dependencies: deps) + shuffle_defs += '-DSHUFFLE_AVX2_ENABLED' + endif + if not get_option('avx512').disabled() and not msvc and cc.has_argument('-mavx512bw') + simd_libs += static_library('blosc2_avx512', + files('blosc/bitshuffle-avx512.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + ['-mavx512f', '-mavx512bw'], + dependencies: deps) + shuffle_defs += '-DSHUFFLE_AVX512_ENABLED' + endif +endif + +if cpu == 'aarch64' and not get_option('neon').disabled() + simd_libs += static_library('blosc2_neon', + files('blosc/shuffle-neon.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + ['-flax-vector-conversions'], + dependencies: deps) + shuffle_defs += '-DSHUFFLE_NEON_ENABLED' +endif + +blosc2_inc = include_directories('include') + +blosc2_lib = static_library('blosc2', sources, + include_directories: [blosc2_inc, include_directories('blosc')], + c_args: blosc2_args + shuffle_defs, + dependencies: deps, + link_whole: simd_libs, + pic: true, +) + +blosc2_dep = declare_dependency( + link_with: blosc2_lib, + include_directories: blosc2_inc, + dependencies: deps, + compile_args: ['-DBUILD_STATIC'], +) +meson.override_dependency('blosc2', blosc2_dep) diff --git a/packages/c-blosc2-blosc1-compat/meson/meson.options b/packages/c-blosc2-blosc1-compat/meson/meson.options new file mode 100644 index 00000000..45623a18 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/meson/meson.options @@ -0,0 +1,12 @@ +option('zstd', type: 'feature', value: 'enabled', + description: 'Build Zstandard support into Blosc2') +option('zlib', type: 'feature', value: 'enabled', + description: 'Build zlib support into Blosc2') +option('sse2', type: 'feature', value: 'auto', + description: 'Enable SSE2 shuffle kernels') +option('avx2', type: 'feature', value: 'auto', + description: 'Enable AVX2 shuffle kernels') +option('avx512', type: 'feature', value: 'auto', + description: 'Enable AVX512 bitshuffle kernels') +option('neon', type: 'feature', value: 'auto', + description: 'Enable NEON shuffle kernels') diff --git a/packages/c-blosc2-blosc1-compat/tests/c/common.h b/packages/c-blosc2-blosc1-compat/tests/c/common.h new file mode 100644 index 00000000..c59dc23c --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/common.h @@ -0,0 +1,12 @@ +#include +#include +#include +#include +static const char *cnames[] = {"blosclz","lz4","lz4hc","zlib","zstd"}; +static const int shuffles[] = {0,1,2}; +static const int typesizes[] = {1,2,3,4,7,8,16}; +static const size_t sizes[] = {1, 33, 127, 1001, 4096, 65537, 100003}; +static void fill(uint8_t *p, size_t n, int ts, unsigned seed) { + unsigned x = seed; + for (size_t i = 0; i < n; i++) { if ((i % ts) == 0) x = x*1103515245u + 12345u; p[i] = (uint8_t)((i/ts)*3 + ((i%ts)? (x>>16)&0x7 : 0)); } +} diff --git a/packages/c-blosc2-blosc1-compat/tests/c/cons.c b/packages/c-blosc2-blosc1-compat/tests/c/cons.c new file mode 100644 index 00000000..b7316bfe --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/cons.c @@ -0,0 +1,22 @@ +// consumer: c-blosc1 decompress; verifies against regenerated data +#include +#include "common.h" +int main(int argc, char **argv) { + const char *mode = argv[1]; + blosc_init(); blosc_set_nthreads(1); + int ok = 0, bad = 0; + for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { + size_t nb = sizes[z]; int ts = typesizes[t]; + + char fn[256]; snprintf(fn, sizeof fn, "out/%s_%s_s%d_t%d_n%zu.bin", mode, cnames[c], shuffles[s], ts, nb); + FILE *f = fopen(fn, "rb"); if (!f) { printf("missing %s\n", fn); continue; } + uint8_t *cbuf = malloc(nb + 1024); size_t cb = fread(cbuf, 1, nb + 1024, f); fclose(f); + uint8_t *want = malloc(nb), *got = calloc(1, nb); fill(want, nb, ts, 1234); + int r = blosc_decompress(cbuf, got, nb); + if (r == (int)nb && !memcmp(want, got, nb)) ok++; + else { if (bad < 100000) printf("BAD %s ret=%d ver=%d\n", fn, r, cbuf[0]); bad++; } + free(cbuf); free(want); free(got); + } + printf("%s: c-blosc1 decoded ok=%d bad=%d\n", mode, ok, bad); + return bad != 0; +} diff --git a/packages/c-blosc2-blosc1-compat/tests/c/cons2.c b/packages/c-blosc2-blosc1-compat/tests/c/cons2.c new file mode 100644 index 00000000..6318d0b7 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/cons2.c @@ -0,0 +1,22 @@ +// consumer: c-blosc1 decompress; verifies against regenerated data +#include +#include "common.h" +int main(int argc, char **argv) { + const char *mode = argv[1]; + blosc2_init(); blosc2_set_nthreads(1); + int ok = 0, bad = 0; + for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { + size_t nb = sizes[z]; int ts = typesizes[t]; + + char fn[256]; snprintf(fn, sizeof fn, "out/%s_%s_s%d_t%d_n%zu.bin", mode, cnames[c], shuffles[s], ts, nb); + FILE *f = fopen(fn, "rb"); if (!f) { printf("missing %s\n", fn); continue; } + uint8_t *cbuf = malloc(nb + 1024); size_t cb = fread(cbuf, 1, nb + 1024, f); fclose(f); + uint8_t *want = malloc(nb), *got = calloc(1, nb); fill(want, nb, ts, 1234); + int r = blosc1_decompress(cbuf, got, nb); + if (r == (int)nb && !memcmp(want, got, nb)) ok++; + else { if (bad < 100000) printf("BAD %s ret=%d ver=%d\n", fn, r, cbuf[0]); bad++; } + free(cbuf); free(want); free(got); + } + printf("%s: c-blosc1 decoded ok=%d bad=%d\n", mode, ok, bad); + return bad != 0; +} diff --git a/packages/c-blosc2-blosc1-compat/tests/c/prod.c b/packages/c-blosc2-blosc1-compat/tests/c/prod.c new file mode 100644 index 00000000..621129b0 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/prod.c @@ -0,0 +1,36 @@ +// producer: c-blosc2, mode: "global" (blosc1_compress, compat env), "global_noenv", "ctx" +#include +#include "common.h" +int main(int argc, char **argv) { + const char *mode = argv[1]; + if (!strcmp(mode, "global")) setenv("BLOSC_BLOSC1_COMPAT", "TRUE", 1); + blosc2_init(); blosc2_set_nthreads(1); if (getenv("SPLITMODE")) blosc1_set_splitmode(atoi(getenv("SPLITMODE"))); + int n = 0; + for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { + size_t nb = sizes[z]; int ts = typesizes[t]; + + uint8_t *src = malloc(nb), *dst = malloc(nb + BLOSC2_MAX_OVERHEAD); + fill(src, nb, ts, 1234); + int cb; + if (!strcmp(mode, "ctx")) { + blosc2_cparams cp = BLOSC2_CPARAMS_DEFAULTS; + cp.compcode = blosc2_compname_to_compcode(cnames[c]); cp.clevel = 5; cp.typesize = ts; cp.nthreads = 1; + memset(cp.filters, 0, sizeof cp.filters); + if (shuffles[s]==1) cp.filters[BLOSC2_MAX_FILTERS-1] = BLOSC_SHUFFLE; + if (shuffles[s]==2) cp.filters[BLOSC2_MAX_FILTERS-1] = BLOSC_BITSHUFFLE; + blosc2_context *ctx = blosc2_create_cctx(cp); + blosc2_ctx_set_blosc1_format(ctx, true); + cb = blosc2_compress_ctx(ctx, src, (int32_t)nb, dst, (int32_t)(nb + BLOSC2_MAX_OVERHEAD)); + blosc2_free_ctx(ctx); + } else { + blosc1_set_compressor(cnames[c]); + cb = blosc1_compress(5, shuffles[s], ts, nb, src, dst, nb + BLOSC2_MAX_OVERHEAD); + } + if (cb <= 0) { fprintf(stderr, "compress fail %s s%d t%d n%zu: %d\n", cnames[c], s, t, nb, cb); return 2; } + char fn[256]; snprintf(fn, sizeof fn, "out/%s_%s_s%d_t%d_n%zu.bin", mode, cnames[c], s, ts, nb); + FILE *f = fopen(fn, "wb"); fwrite(dst, 1, cb, f); fclose(f); n++; + free(src); free(dst); + } + printf("%s: wrote %d buffers; first byte(version) of last = see check\n", mode, n); + return 0; +} diff --git a/packages/c-blosc2-blosc1-compat/tests/c/prod1.c b/packages/c-blosc2-blosc1-compat/tests/c/prod1.c new file mode 100644 index 00000000..2bb853eb --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/prod1.c @@ -0,0 +1,11 @@ +#include +#include "common.h" +int main(void){ blosc_init(); blosc_set_nthreads(1); + for (int c=0;c<5;c++) for(int s=0;s<3;s++) for(int t=0;t<7;t++) for(int z=0;z<7;z++){ + size_t nb=sizes[z]; int ts=typesizes[t]; + uint8_t *src=malloc(nb),*dst=malloc(nb+BLOSC_MAX_OVERHEAD); fill(src,nb,ts,1234); + blosc_set_compressor(cnames[c]); int cb=blosc_compress(5,shuffles[s],ts,nb,src,dst,nb+BLOSC_MAX_OVERHEAD); + if(cb<=0){fprintf(stderr,"fail\n");return 2;} + char fn[256]; snprintf(fn,sizeof fn,"out/b1_%s_s%d_t%d_n%zu.bin",cnames[c],shuffles[s],ts,nb); + FILE*f=fopen(fn,"wb"); fwrite(dst,1,cb,f); fclose(f);} + return 0;} diff --git a/packages/c-blosc2-blosc1-compat/tests/run.py b/packages/c-blosc2-blosc1-compat/tests/run.py new file mode 100755 index 00000000..219d587d --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/run.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Check that chunks written by c-blosc2 in Blosc1 format can be read by C-Blosc 1.x. + +Compiles the programs in tests/c against both libraries and runs, over every +compressor in {blosclz, lz4, lz4hc, zlib, zstd} x shuffle {none, byte, bit} x typesize +x buffer size (including sizes that are not a multiple of the typesize): + + global, ctx c-blosc2 writes Blosc1 chunks through blosc1_compress / blosc2_compress_ctx, + C-Blosc 1.x decodes them. + b1 C-Blosc 1.x writes, c-blosc2 decodes (backward compatibility). + +Example: + run.py --blosc1-include c-blosc/blosc --blosc1-lib build-b1/blosc \\ + --blosc2-include c-blosc2/include --blosc2-ldflags "libblosc2.a libzstd.a liblz4.a -lz" +""" + +import argparse +import os +import shlex +import subprocess +import sys +import tempfile +from pathlib import Path + +C = Path(__file__).resolve().parent / "c" + + +def sh(cmd, **kw): + subprocess.run(cmd, check=True, **kw) + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--blosc1-include", required=True, help="directory with blosc.h (C-Blosc 1.x)") + ap.add_argument( + "--blosc1-lib", required=True, help="directory with the C-Blosc 1.x shared library" + ) + ap.add_argument( + "--blosc2-include", + required=True, + help="directory with blosc2.h (c-blosc2 with Blosc1 format support)", + ) + ap.add_argument( + "--blosc2-ldflags", required=True, help="libraries to link c-blosc2 and its codecs" + ) + ap.add_argument("--cc", default=os.environ.get("CC", "cc")) + a = ap.parse_args() + + lib1 = Path(a.blosc1_lib).resolve() + ld1 = [f"-L{lib1}", "-lblosc", f"-Wl,-rpath,{lib1}"] + ld2 = [*shlex.split(a.blosc2_ldflags), "-lpthread", "-lm"] + + def inc(d): + return "-I" + str(Path(d).resolve()) + + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + (tmp / "out").mkdir() + cc = [a.cc, "-O1", f"-I{C}"] + sh([*cc, inc(a.blosc2_include), str(C / "prod.c"), "-o", "prod", *ld2], cwd=tmp) + sh([*cc, inc(a.blosc1_include), str(C / "cons.c"), "-o", "cons", *ld1], cwd=tmp) + sh([*cc, inc(a.blosc1_include), str(C / "prod1.c"), "-o", "prod1", *ld1], cwd=tmp) + sh([*cc, inc(a.blosc2_include), str(C / "cons2.c"), "-o", "cons2", *ld2], cwd=tmp) + failed = False + for mode in ("global", "ctx"): + sh(["./prod", mode], cwd=tmp, stdout=subprocess.DEVNULL) + failed |= subprocess.run(["./cons", mode], cwd=tmp, check=False).returncode != 0 + sh(["./prod1"], cwd=tmp) + failed |= subprocess.run(["./cons2", "b1"], cwd=tmp, check=False).returncode != 0 + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py b/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py new file mode 100755 index 00000000..f604d333 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Copy this package's build overlay into subprojects/packagefiles/c-blosc2. + +Meson wraps only look for `patch_directory` in subprojects/packagefiles, +so that directory holds copies of what lives here. Run without arguments to update the +copies, or with --check (used in CI) to fail if they are out of date. +""" + +import argparse +import filecmp +import shutil +import sys +from pathlib import Path + +PKG = Path(__file__).resolve().parents[1] +DEST = PKG.parents[1] / "subprojects" / "packagefiles" / "c-blosc2" + + +def sources(): + for f in sorted((PKG / "meson").iterdir()): + yield f, DEST / f.name + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--check", action="store_true") + check = ap.parse_args().check + expected = {dst for _, dst in sources()} + stale = [d for d in DEST.rglob("*") if d.is_file() and d not in expected] + bad = [ + dst + for src, dst in sources() + if not dst.exists() or not filecmp.cmp(src, dst, shallow=False) + ] + if check: + for f in bad + stale: + print(f"out of sync: {f}", file=sys.stderr) + return 1 if bad or stale else 0 + for src, dst in sources(): + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dst) + for f in stale: + f.unlink() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index 185ecc3a..9d042862 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,6 +148,9 @@ norecursedirs = [ ] log_cli_level = "INFO" xfail_strict = true +markers = [ + "blosc1_compat: compatibility of Blosc with C-Blosc 1.x (needs python-blosc and hypothesis; see .github/workflows/blosc1-compat.yaml)", +] filterwarnings = [ "error", "ignore:As the c extension couldn't be imported:RuntimeWarning", # Ignore warning about pure python google_crc32c (on Python 3.14) @@ -253,3 +256,9 @@ conflicts = [ { group = "test-zarr-main" } ] ] + +[tool.meson-python.args] +# Wrapped subprojects (c-blosc2, zstd, lz4, zlib) are static build dependencies: +# do not install their artifacts, but ship their sources in the sdist. +install = ['--skip-subprojects'] +dist = ['--include-subprojects'] diff --git a/src/numcodecs/_blosc_shim.h b/src/numcodecs/_blosc_shim.h new file mode 100644 index 00000000..49162db8 --- /dev/null +++ b/src/numcodecs/_blosc_shim.h @@ -0,0 +1,146 @@ +/* Common C API for the two Blosc backends that blosc.pyx can be built against. + * + * - c-blosc 1.x (default): thin aliases for the blosc_* API. + * - c-blosc2 (-DNUMCODECS_BLOSC2): the blosc1_*, blosc2_* API. Chunks are always written + * in the Blosc1 format (format version 2, 16-byte header) so that they stay readable + * by C-Blosc 1.x; this needs the blosc1-format patch applied to c-blosc2 (see + * subprojects/packagefiles/c-blosc2/blosc1-format.patch). c-blosc2 reads both formats. + */ +#ifndef NUMCODECS_BLOSC_SHIM_H +#define NUMCODECS_BLOSC_SHIM_H + +#include +#include + +#ifdef NUMCODECS_BLOSC2 + +#include + +#define NC_BLOSC_LIBRARY "c-blosc2" +#define NC_BLOSC_VERSION_STRING BLOSC2_VERSION_STRING +#define NC_BLOSC_VERSION_DATE BLOSC2_VERSION_DATE +/* Blosc1 chunk header size; what is written. Buffers are sized with NC_BLOSC_DEST_OVERHEAD. */ +#define NC_BLOSC_MAX_OVERHEAD BLOSC_MIN_HEADER_LENGTH +#define NC_BLOSC_DEST_OVERHEAD BLOSC2_MAX_OVERHEAD +#define NC_BLOSC_MAX_BUFFERSIZE BLOSC2_MAX_BUFFERSIZE +#define NC_BLOSC_MAX_THREADS 256 +#define NC_BLOSC_MAX_TYPESIZE BLOSC_MAX_TYPESIZE +#define NC_BLOSC_MIN_HEADER_LENGTH BLOSC_MIN_HEADER_LENGTH + +#define nc_blosc_init blosc2_init +#define nc_blosc_destroy blosc2_destroy +#define nc_blosc_get_nthreads blosc2_get_nthreads +#define nc_blosc_set_nthreads blosc2_set_nthreads +#define nc_blosc_set_compressor blosc1_set_compressor +#define nc_blosc_set_blocksize blosc1_set_blocksize +#define nc_blosc_list_compressors blosc2_list_compressors +#define nc_blosc_compress blosc1_compress +#define nc_blosc_decompress blosc1_decompress +#define nc_blosc_cbuffer_sizes blosc1_cbuffer_sizes +#define nc_blosc_cbuffer_validate blosc1_cbuffer_validate +#define nc_blosc_cbuffer_complib blosc2_cbuffer_complib +#define nc_blosc_cbuffer_metainfo blosc1_cbuffer_metainfo + +/* The global context writes Blosc1 chunks when this environment variable is set. It is + * read on each compression call, so set it before the first one. */ +static inline void nc_blosc_setup(void) { +#ifdef _WIN32 + _putenv_s("BLOSC_BLOSC1_COMPAT", "1"); +#else + setenv("BLOSC_BLOSC1_COMPAT", "1", 1); +#endif +} + +/* Equivalent of blosc_compress_ctx: no global state, so no locking is needed. */ +static inline int nc_blosc_compress_ctx( + int clevel, int doshuffle, size_t typesize, size_t nbytes, const void* src, + void* dest, size_t destsize, const char* compressor, size_t blocksize, + int numinternalthreads) { + int compcode = blosc2_compname_to_compcode(compressor); + if (compcode < 0) { + return -1; + } + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.compcode = (uint8_t)compcode; + cparams.clevel = (uint8_t)clevel; + cparams.typesize = (int32_t)typesize; + cparams.blocksize = (int32_t)blocksize; + cparams.nthreads = (int16_t)numinternalthreads; + memset(cparams.filters, 0, sizeof(cparams.filters)); + memset(cparams.filters_meta, 0, sizeof(cparams.filters_meta)); + if (doshuffle == BLOSC_SHUFFLE) { + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + } + else if (doshuffle == BLOSC_BITSHUFFLE) { + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_BITSHUFFLE; + } + blosc2_context* ctx = blosc2_create_cctx(cparams); + if (ctx == NULL) { + return -1; + } + int rc = blosc2_ctx_set_blosc1_format(ctx, true); + if (rc < 0) { + blosc2_free_ctx(ctx); + return rc; + } + rc = blosc2_compress_ctx(ctx, src, (int32_t)nbytes, dest, (int32_t)destsize); + blosc2_free_ctx(ctx); + return rc; +} + +/* Equivalent of blosc_decompress_ctx; `srcsize` is the number of valid source bytes. */ +static inline int nc_blosc_decompress_ctx( + const void* src, size_t srcsize, void* dest, size_t destsize, + int numinternalthreads) { + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = (int16_t)numinternalthreads; + blosc2_context* ctx = blosc2_create_dctx(dparams); + if (ctx == NULL) { + return -1; + } + int rc = blosc2_decompress_ctx(ctx, src, (int32_t)srcsize, dest, (int32_t)destsize); + blosc2_free_ctx(ctx); + return rc; +} + +#else /* c-blosc 1.x */ + +#include + +#define NC_BLOSC_LIBRARY "c-blosc" +#define NC_BLOSC_VERSION_STRING BLOSC_VERSION_STRING +#define NC_BLOSC_VERSION_DATE BLOSC_VERSION_DATE +#define NC_BLOSC_MAX_OVERHEAD BLOSC_MAX_OVERHEAD +#define NC_BLOSC_DEST_OVERHEAD BLOSC_MAX_OVERHEAD +#define NC_BLOSC_MAX_BUFFERSIZE BLOSC_MAX_BUFFERSIZE +#define NC_BLOSC_MAX_THREADS BLOSC_MAX_THREADS +#define NC_BLOSC_MAX_TYPESIZE BLOSC_MAX_TYPESIZE +#define NC_BLOSC_MIN_HEADER_LENGTH BLOSC_MIN_HEADER_LENGTH + +#define nc_blosc_init blosc_init +#define nc_blosc_destroy blosc_destroy +#define nc_blosc_get_nthreads blosc_get_nthreads +#define nc_blosc_set_nthreads blosc_set_nthreads +#define nc_blosc_set_compressor blosc_set_compressor +#define nc_blosc_set_blocksize blosc_set_blocksize +#define nc_blosc_list_compressors blosc_list_compressors +#define nc_blosc_compress blosc_compress +#define nc_blosc_decompress blosc_decompress +#define nc_blosc_cbuffer_sizes blosc_cbuffer_sizes +#define nc_blosc_cbuffer_validate blosc_cbuffer_validate +#define nc_blosc_cbuffer_complib blosc_cbuffer_complib +#define nc_blosc_cbuffer_metainfo blosc_cbuffer_metainfo +#define nc_blosc_compress_ctx blosc_compress_ctx + +static inline void nc_blosc_setup(void) {} + +static inline int nc_blosc_decompress_ctx( + const void* src, size_t srcsize, void* dest, size_t destsize, + int numinternalthreads) { + (void)srcsize; + return blosc_decompress_ctx(src, dest, destsize, numinternalthreads); +} + +#endif + +#endif /* NUMCODECS_BLOSC_SHIM_H */ diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index 8c52dd95..20099c49 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -18,50 +18,57 @@ from .compat import ensure_contiguous_ndarray from .abc import Codec -cdef extern from "blosc.h": +cdef extern from "_blosc_shim.h": cdef enum: - BLOSC_MAX_OVERHEAD, - BLOSC_VERSION_STRING, - BLOSC_VERSION_DATE, + NC_BLOSC_MAX_OVERHEAD, + NC_BLOSC_DEST_OVERHEAD, + NC_BLOSC_MIN_HEADER_LENGTH, + NC_BLOSC_VERSION_STRING, + NC_BLOSC_VERSION_DATE, + NC_BLOSC_MAX_BUFFERSIZE, + NC_BLOSC_MAX_THREADS, + NC_BLOSC_MAX_TYPESIZE, BLOSC_NOSHUFFLE, BLOSC_SHUFFLE, BLOSC_BITSHUFFLE, - BLOSC_MAX_BUFFERSIZE, - BLOSC_MAX_THREADS, - BLOSC_MAX_TYPESIZE, BLOSC_DOSHUFFLE, BLOSC_DOBITSHUFFLE, BLOSC_MEMCPYED - void blosc_init() - void blosc_destroy() - int blosc_get_nthreads() - int blosc_set_nthreads(int nthreads) - int blosc_set_compressor(const char *compname) - void blosc_set_blocksize(size_t blocksize) - char* blosc_list_compressors() - int blosc_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes, - void* src, void* dest, size_t destsize) nogil - int blosc_decompress(void *src, void *dest, size_t destsize) nogil - int blosc_getitem(void* src, int start, int nitems, void* dest) - int blosc_compress_ctx(int clevel, int doshuffle, size_t typesize, size_t nbytes, - const void* src, void* dest, size_t destsize, - const char* compressor, size_t blocksize, - int numinternalthreads) nogil - int blosc_decompress_ctx(const void* src, void* dest, size_t destsize, - int numinternalthreads) nogil - void blosc_cbuffer_sizes(const void* cbuffer, size_t* nbytes, size_t* cbytes, - size_t* blocksize) - char* blosc_cbuffer_complib(const void* cbuffer) - void blosc_cbuffer_metainfo(const void* cbuffer, size_t* typesize, int* flags) - - -MAX_OVERHEAD = BLOSC_MAX_OVERHEAD -MAX_BUFFERSIZE = BLOSC_MAX_BUFFERSIZE -MAX_THREADS = BLOSC_MAX_THREADS -MAX_TYPESIZE = BLOSC_MAX_TYPESIZE -VERSION_STRING = BLOSC_VERSION_STRING -VERSION_DATE = BLOSC_VERSION_DATE + const char* NC_BLOSC_LIBRARY + + void nc_blosc_setup() + void nc_blosc_init() + void nc_blosc_destroy() + int nc_blosc_get_nthreads() + int nc_blosc_set_nthreads(int nthreads) + int nc_blosc_set_compressor(const char *compname) + void nc_blosc_set_blocksize(size_t blocksize) + char* nc_blosc_list_compressors() + int nc_blosc_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes, + void* src, void* dest, size_t destsize) nogil + int nc_blosc_decompress(void *src, void *dest, size_t destsize) nogil + int nc_blosc_compress_ctx(int clevel, int doshuffle, size_t typesize, size_t nbytes, + const void* src, void* dest, size_t destsize, + const char* compressor, size_t blocksize, + int numinternalthreads) nogil + int nc_blosc_decompress_ctx(const void* src, size_t srcsize, void* dest, + size_t destsize, int numinternalthreads) nogil + int nc_blosc_cbuffer_validate(const void* cbuffer, size_t cbytes, size_t* nbytes) + void nc_blosc_cbuffer_sizes(const void* cbuffer, size_t* nbytes, size_t* cbytes, + size_t* blocksize) + const char* nc_blosc_cbuffer_complib(const void* cbuffer) + void nc_blosc_cbuffer_metainfo(const void* cbuffer, size_t* typesize, int* flags) + + +MAX_OVERHEAD = NC_BLOSC_MAX_OVERHEAD +MAX_BUFFERSIZE = NC_BLOSC_MAX_BUFFERSIZE +MAX_THREADS = NC_BLOSC_MAX_THREADS +MAX_TYPESIZE = NC_BLOSC_MAX_TYPESIZE +VERSION_STRING = NC_BLOSC_VERSION_STRING +VERSION_DATE = NC_BLOSC_VERSION_DATE +# name of the C library this module is built against: "c-blosc" or "c-blosc2" +LIBRARY = ( NC_BLOSC_LIBRARY).decode() VERSION_STRING = VERSION_STRING.decode() VERSION_DATE = VERSION_DATE.decode() __version__ = VERSION_STRING @@ -74,7 +81,8 @@ AUTOSHUFFLE = -1 AUTOBLOCKS = 0 # computed once at import; blosc_list_compressors() fills a static buffer without locking -_COMPRESSORS = tuple(blosc_list_compressors().decode('ascii').split(',')) +nc_blosc_setup() +_COMPRESSORS = tuple(nc_blosc_list_compressors().decode('ascii').split(',')) # serializes access to blosc's global context _MUTEX = threading.Lock() @@ -95,13 +103,13 @@ _importer_pid = os.getpid() def _init(): """Initialize the Blosc library environment.""" with _MUTEX: - blosc_init() + nc_blosc_init() def _destroy(): """Destroy the Blosc library environment.""" with _MUTEX: - blosc_destroy() + nc_blosc_destroy() def list_compressors(): @@ -113,7 +121,7 @@ def get_nthreads(): """Get the number of threads that Blosc uses internally for compression and decompression.""" with _MUTEX: - return blosc_get_nthreads() + return nc_blosc_get_nthreads() def set_nthreads(int nthreads): @@ -121,7 +129,7 @@ def set_nthreads(int nthreads): decompression.""" # blosc_set_nthreads re-creates the global context with _MUTEX: - return blosc_set_nthreads(nthreads) + return nc_blosc_set_nthreads(nthreads) def _cbuffer_sizes(source): @@ -146,7 +154,7 @@ def _cbuffer_sizes(source): source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - blosc_cbuffer_sizes(source_pb.buf, &nbytes, &cbytes, &blocksize) + nc_blosc_cbuffer_sizes(source_pb.buf, &nbytes, &cbytes, &blocksize) return nbytes, cbytes, blocksize @@ -162,7 +170,7 @@ def cbuffer_complib(source): source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - complib = blosc_cbuffer_complib(source_pb.buf) + complib = nc_blosc_cbuffer_complib(source_pb.buf) complib = complib.decode('ascii') @@ -192,7 +200,7 @@ def _cbuffer_metainfo(source): source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - blosc_cbuffer_metainfo(source_pb.buf, &typesize, &flags) + nc_blosc_cbuffer_metainfo(source_pb.buf, &typesize, &flags) # decompose flags if flags & BLOSC_DOSHUFFLE: @@ -267,6 +275,20 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, itemsize = typesize else: itemsize = source_pb.itemsize + if itemsize == 0: + # blosc does not accept a typesize of 0, which zero-size dtypes such as a + # structured dtype with a zero-length field give; the buffer is empty in + # that case, so any typesize works + itemsize = 1 + + # A requested blocksize must not exceed the buffer and must be a multiple of the + # typesize. c-blosc 1.x enforces this itself; c-blosc2 does not and then writes + # chunks it cannot read back. + if blocksize > 0: + if blocksize > nbytes and nbytes > 0: + blocksize = nbytes + if blocksize > itemsize: + blocksize -= blocksize % itemsize # determine shuffle if shuffle == AUTOSHUFFLE: @@ -281,7 +303,7 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, try: # setup destination - dest = PyBytes_FromStringAndSize(NULL, nbytes + BLOSC_MAX_OVERHEAD) + dest = PyBytes_FromStringAndSize(NULL, nbytes + NC_BLOSC_DEST_OVERHEAD) dest_ptr = PyBytes_AS_STRING(dest) # perform compression @@ -294,24 +316,24 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, with _MUTEX: # set compressor - compressor_set = blosc_set_compressor(cname) + compressor_set = nc_blosc_set_compressor(cname) if compressor_set < 0: # shouldn't happen if we checked against list of compressors # already, but just in case _err_bad_cname(cname_str) # set blocksize - blosc_set_blocksize(blocksize) + nc_blosc_set_blocksize(blocksize) # perform compression with nogil: - cbytes = blosc_compress(clevel, shuffle, itemsize, nbytes, source_ptr, - dest_ptr, nbytes + BLOSC_MAX_OVERHEAD) + cbytes = nc_blosc_compress(clevel, shuffle, itemsize, nbytes, source_ptr, + dest_ptr, nbytes + NC_BLOSC_DEST_OVERHEAD) else: with nogil: - cbytes = blosc_compress_ctx(clevel, shuffle, itemsize, nbytes, source_ptr, - dest_ptr, nbytes + BLOSC_MAX_OVERHEAD, + cbytes = nc_blosc_compress_ctx(clevel, shuffle, itemsize, nbytes, source_ptr, + dest_ptr, nbytes + NC_BLOSC_DEST_OVERHEAD, cname, blocksize, 1) finally: @@ -334,7 +356,8 @@ def decompress(source, dest=None): ---------- source : bytes-like Compressed data, including blosc header. Can be any object supporting the buffer - protocol. + protocol. Bytes after the first complete Blosc frame are ignored for backward + compatibility. dest : array-like, optional Object to decompress into. @@ -361,8 +384,16 @@ def decompress(source, dest=None): # get source pointer source_ptr = source_pb.buf - # determine buffer size - blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize) + # Read the declared frame size only after proving the complete header is present. + # Validate exactly that frame so trailing bytes remain backward compatible. + if source_pb.len < NC_BLOSC_MIN_HEADER_LENGTH: + raise RuntimeError('invalid blosc frame: buffer is too small') + nc_blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize) + if cbytes > source_pb.len: + raise RuntimeError('invalid blosc frame: buffer is truncated') + ret = nc_blosc_cbuffer_validate(source_ptr, cbytes, &nbytes) + if ret != 0: + raise RuntimeError('invalid blosc frame: header validation failed') # setup destination buffer if dest is None: @@ -384,21 +415,27 @@ def decompress(source, dest=None): raise ValueError('destination buffer too small; expected at least %s, ' 'got %s' % (nbytes, dest_nbytes)) + # Preserve the previous loud failure for a non-empty destination paired + # with an empty frame instead of returning an untouched output buffer. + if nbytes == 0 and dest_nbytes != 0: + raise RuntimeError('cannot decompress an empty blosc frame into a ' + 'non-empty destination buffer') + # perform decompression if _get_use_threads(): # allow blosc to use threads internally with _MUTEX: with nogil: - ret = blosc_decompress(source_ptr, dest_ptr, nbytes) + ret = nc_blosc_decompress(source_ptr, dest_ptr, nbytes) else: with nogil: - ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1) + ret = nc_blosc_decompress_ctx(source_ptr, cbytes, dest_ptr, nbytes, 1) finally: pass # handle errors - if ret <= 0: + if ret < 0 or ret != nbytes: raise RuntimeError('error during blosc decompression: %d' % ret) return dest diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index 41a3a7a2..f71bd3d1 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -38,172 +38,219 @@ if host_machine.cpu_family() == 'x86_64' endif endif -# --- Vendored zstd --- -zstd_dep = dependency('libzstd', required: get_option('system_zstd')) - -if not zstd_dep.found() - zstd_sources = files( - # common/ - '../../c-blosc/internal-complibs/zstd-1.5.6/common/debug.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/fse_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/pool.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/threading.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/xxhash.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/zstd_common.c', - # compress/ - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/fse_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/hist.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/huf_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_literals.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_sequences.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_superblock.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_double_fast.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_fast.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_lazy.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_ldm.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_opt.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstdmt_compress.c', - # decompress/ - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_ddict.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress_block.c', - # dictBuilder/ - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/cover.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/divsufsort.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/fastcover.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/zdict.c', - ) +use_blosc2 = get_option('blosc_backend') == 'blosc2' - # Assembly files — meson handles .S natively - if host_machine.cpu_family() == 'x86_64' - zstd_sources += files( - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S', - ) +if use_blosc2 + # --- Compression libraries from the system or from WrapDB subprojects --- + # Each library is resolved here, once, so that the standalone zstd/lz4 codecs and the + # c-blosc2 subproject share the same build. The subproject is configured first so + # that c-blosc2's own dependency() lookups find it. + sub_opts = ['default_library=static'] + + # zlib first: zstd looks it up as an optional dependency + if get_option('system_zlib').disabled() + subproject('zlib', default_options: sub_opts) + else + dependency('zlib', required: get_option('system_zlib')) endif - zstd_inc = include_directories( - '../../c-blosc/internal-complibs/zstd-1.5.6', - '../../c-blosc/internal-complibs/zstd-1.5.6/common', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', - ) + if get_option('system_lz4').disabled() + lz4_dep = subproject('lz4', default_options: sub_opts).get_variable('liblz4_dep') + else + lz4_dep = dependency('liblz4', required: get_option('system_lz4')) + endif - zstd_lib = static_library('zstd_vendored', zstd_sources, - include_directories: zstd_inc, - c_args: c_args, - ) - zstd_dep = declare_dependency( - link_with: zstd_lib, - include_directories: zstd_inc, - ) -endif + if get_option('system_zstd').disabled() + zstd_dep = subproject('zstd', default_options: sub_opts).get_variable('libzstd_dep') + else + zstd_dep = dependency('libzstd', required: get_option('system_zstd')) + endif -# --- Vendored lz4 --- -lz4_dep = dependency('liblz4', required: get_option('system_lz4')) + blosc_sub_opts = [ + 'zstd=' + (get_option('blosc_zstd').disabled() ? 'disabled' : 'enabled'), + 'zlib=' + (get_option('blosc_zlib').disabled() ? 'disabled' : 'enabled'), + 'sse2=' + (have_sse2 ? 'enabled' : 'disabled'), + 'avx2=' + (have_avx2 ? 'enabled' : 'disabled'), + ] + if get_option('system_blosc').disabled() + blosc_dep = subproject( + 'c-blosc2', + default_options: sub_opts + blosc_sub_opts, + ).get_variable('blosc2_dep') + else + blosc_dep = dependency('blosc2', version: '>=3.0', required: get_option('system_blosc')) + endif + blosc_c_args = ['-DNUMCODECS_BLOSC2'] +else + # --- Vendored zstd --- + zstd_dep = dependency('libzstd', required: get_option('system_zstd')) -if not lz4_dep.found() - lz4_sources = files( - '../../c-blosc/internal-complibs/lz4-1.10.0/lz4.c', - '../../c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', - ) - lz4_inc = include_directories('../../c-blosc/internal-complibs/lz4-1.10.0') + if not zstd_dep.found() + zstd_sources = files( + # common/ + '../../c-blosc/internal-complibs/zstd-1.5.6/common/debug.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/fse_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/pool.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/threading.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/xxhash.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/zstd_common.c', + # compress/ + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/fse_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/hist.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/huf_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_literals.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_sequences.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_superblock.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_double_fast.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_fast.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_lazy.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_ldm.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_opt.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstdmt_compress.c', + # decompress/ + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_ddict.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress_block.c', + # dictBuilder/ + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/cover.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/divsufsort.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/fastcover.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/zdict.c', + ) - lz4_lib = static_library('lz4_vendored', lz4_sources, - include_directories: lz4_inc, - c_args: c_args, - ) - lz4_dep = declare_dependency( - link_with: lz4_lib, - include_directories: lz4_inc, - ) -endif + # Assembly files — meson handles .S natively + if host_machine.cpu_family() == 'x86_64' + zstd_sources += files( + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S', + ) + endif -# --- Vendored blosc --- -blosc_dep = dependency('blosc', required: get_option('system_blosc')) - -if not blosc_dep.found() - # --- Vendored zlib --- - zlib_dep = dependency('zlib', required: get_option('system_zlib')) - - if not zlib_dep.found() - zlib_sources = files( - '../../c-blosc/internal-complibs/zlib-1.3.1/adler32.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/compress.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/crc32.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/deflate.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzclose.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzlib.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzread.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzwrite.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/infback.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inffast.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inflate.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/trees.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/zutil.c', + zstd_inc = include_directories( + '../../c-blosc/internal-complibs/zstd-1.5.6', + '../../c-blosc/internal-complibs/zstd-1.5.6/common', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', ) - zlib_inc = include_directories('../../c-blosc/internal-complibs/zlib-1.3.1') - zlib_lib = static_library('zlib_vendored', zlib_sources, - include_directories: zlib_inc, + zstd_lib = static_library('zstd_vendored', zstd_sources, + include_directories: zstd_inc, c_args: c_args, ) - zlib_dep = declare_dependency( - link_with: zlib_lib, - include_directories: zlib_inc, + zstd_dep = declare_dependency( + link_with: zstd_lib, + include_directories: zstd_inc, ) endif - blosc_sources = files( - '../../c-blosc/blosc/blosc.c', - '../../c-blosc/blosc/blosclz.c', - '../../c-blosc/blosc/fastcopy.c', - '../../c-blosc/blosc/shuffle.c', - '../../c-blosc/blosc/shuffle-generic.c', - '../../c-blosc/blosc/bitshuffle-generic.c', - ) + # --- Vendored lz4 --- + lz4_dep = dependency('liblz4', required: get_option('system_lz4')) - blosc_c_args = c_args + ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] + if not lz4_dep.found() + lz4_sources = files( + '../../c-blosc/internal-complibs/lz4-1.10.0/lz4.c', + '../../c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', + ) + lz4_inc = include_directories('../../c-blosc/internal-complibs/lz4-1.10.0') - if have_sse2 - blosc_sources += files( - '../../c-blosc/blosc/shuffle-sse2.c', - '../../c-blosc/blosc/bitshuffle-sse2.c', + lz4_lib = static_library('lz4_vendored', lz4_sources, + include_directories: lz4_inc, + c_args: c_args, + ) + lz4_dep = declare_dependency( + link_with: lz4_lib, + include_directories: lz4_inc, ) - blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] - if host_machine.system() == 'windows' - blosc_c_args += ['-D__SSE2__'] - endif endif - if have_avx2 - blosc_sources += files( - '../../c-blosc/blosc/shuffle-avx2.c', - '../../c-blosc/blosc/bitshuffle-avx2.c', + # --- Vendored blosc --- + blosc_dep = dependency('blosc', required: get_option('system_blosc')) + + if not blosc_dep.found() + # --- Vendored zlib --- + zlib_dep = dependency('zlib', required: get_option('system_zlib')) + + if not zlib_dep.found() + zlib_sources = files( + '../../c-blosc/internal-complibs/zlib-1.3.1/adler32.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/compress.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/crc32.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/deflate.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzclose.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzlib.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzread.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzwrite.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/infback.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inffast.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inflate.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/trees.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/zutil.c', + ) + zlib_inc = include_directories('../../c-blosc/internal-complibs/zlib-1.3.1') + + zlib_lib = static_library('zlib_vendored', zlib_sources, + include_directories: zlib_inc, + c_args: c_args, + ) + zlib_dep = declare_dependency( + link_with: zlib_lib, + include_directories: zlib_inc, + ) + endif + + blosc_sources = files( + '../../c-blosc/blosc/blosc.c', + '../../c-blosc/blosc/blosclz.c', + '../../c-blosc/blosc/fastcopy.c', + '../../c-blosc/blosc/shuffle.c', + '../../c-blosc/blosc/shuffle-generic.c', + '../../c-blosc/blosc/bitshuffle-generic.c', ) - blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] - if host_machine.system() == 'windows' - blosc_c_args += ['-D__AVX2__'] + + blosc_c_args = c_args + ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] + + if have_sse2 + blosc_sources += files( + '../../c-blosc/blosc/shuffle-sse2.c', + '../../c-blosc/blosc/bitshuffle-sse2.c', + ) + blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__SSE2__'] + endif endif - endif - blosc_inc = include_directories('../../c-blosc/blosc') + if have_avx2 + blosc_sources += files( + '../../c-blosc/blosc/shuffle-avx2.c', + '../../c-blosc/blosc/bitshuffle-avx2.c', + ) + blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__AVX2__'] + endif + endif - blosc_lib = static_library('blosc_vendored', blosc_sources, - include_directories: blosc_inc, - dependencies: [lz4_dep, zlib_dep, zstd_dep], - c_args: blosc_c_args, - ) - blosc_dep = declare_dependency( - link_with: blosc_lib, - include_directories: blosc_inc, - ) + blosc_inc = include_directories('../../c-blosc/blosc') + + blosc_lib = static_library('blosc_vendored', blosc_sources, + include_directories: blosc_inc, + dependencies: [lz4_dep, zlib_dep, zstd_dep], + c_args: blosc_c_args, + ) + blosc_dep = declare_dependency( + link_with: blosc_lib, + include_directories: blosc_inc, + ) + endif + + blosc_c_args = [] endif # --- Cython extension modules --- @@ -212,7 +259,8 @@ endif py.extension_module('blosc', 'blosc.pyx', dependencies: [blosc_dep], - c_args: c_args, + include_directories: include_directories('.'), + c_args: c_args + blosc_c_args, link_args: link_args, install: true, subdir: 'numcodecs', diff --git a/subprojects/c-blosc2.wrap b/subprojects/c-blosc2.wrap new file mode 100644 index 00000000..a484923b --- /dev/null +++ b/subprojects/c-blosc2.wrap @@ -0,0 +1,9 @@ +[wrap-git] +directory = c-blosc2 +url = https://github.com/mkitti/c-blosc2.git +# Branch blosc1-format-writer of the fork (Blosc/c-blosc2#815); pinned to its head commit. +revision = c919bfe6834cd05ca853a4b63340b693bed9892f +patch_directory = c-blosc2 + +[provide] +dependency_names = blosc2 diff --git a/subprojects/lz4.wrap b/subprojects/lz4.wrap new file mode 100644 index 00000000..7732b9a2 --- /dev/null +++ b/subprojects/lz4.wrap @@ -0,0 +1,13 @@ +[wrap-file] +directory = lz4-1.10.0 +source_url = https://github.com/lz4/lz4/archive/v1.10.0.tar.gz +source_filename = lz4-1.10.0.tgz +source_hash = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b +patch_filename = lz4_1.10.0-2_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/lz4_1.10.0-2/get_patch +patch_hash = ce568b4e1a7a593bb233f3ec97af88ceb850524cef802fe3f916b21d8a79b6b6 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/lz4_1.10.0-2/lz4-1.10.0.tgz +wrapdb_version = 1.10.0-2 + +[provide] +liblz4 = liblz4_dep diff --git a/subprojects/packagefiles/c-blosc2/meson.build b/subprojects/packagefiles/c-blosc2/meson.build new file mode 100644 index 00000000..ceb48c8a --- /dev/null +++ b/subprojects/packagefiles/c-blosc2/meson.build @@ -0,0 +1,130 @@ +# Meson build for c-blosc2, which upstream only builds with CMake. +# Only the core library plus the built-in codec/filter/tuner plugins are built +# (no ZFP, tests, benchmarks or examples). Used via subprojects/c-blosc2.wrap. +project( + 'c-blosc2', 'c', + version: '3.3.6.dev', + meson_version: '>=1.6.0', + default_options: ['c_std=gnu11', 'warning_level=0'], +) + +cc = meson.get_compiler('c') +host_system = host_machine.system() +cpu = host_machine.cpu_family() + +blosc2_args = ['-DBUILD_STATIC'] +if host_system != 'windows' + blosc2_args += ['-D_GNU_SOURCE'] +else + blosc2_args += ['-D_CRT_SECURE_NO_WARNINGS'] +endif + +deps = [dependency('threads')] +if host_system != 'windows' + deps += cc.find_library('m', required: false) + deps += cc.find_library('dl', required: false) +endif + +# --- compression libraries (resolved by the parent project or WrapDB) --- +lz4_dep = dependency('liblz4', fallback: ['lz4', 'liblz4_dep'], + default_options: ['default_library=static']) +deps += lz4_dep +blosc2_args += ['-DHAVE_LZ4'] + +zstd_dep = dependency('libzstd', required: get_option('zstd'), + fallback: ['zstd', 'libzstd_dep'], + default_options: ['default_library=static']) +if zstd_dep.found() + deps += zstd_dep + blosc2_args += ['-DHAVE_ZSTD'] +endif + +zlib_dep = dependency('zlib', required: get_option('zlib'), + fallback: ['zlib', 'zlib_dep'], + default_options: ['default_library=static']) +if zlib_dep.found() + deps += zlib_dep + blosc2_args += ['-DHAVE_ZLIB'] +endif + +sources = files( + 'blosc/blosc2.c', 'blosc/blosclz.c', 'blosc/fastcopy.c', 'blosc/schunk.c', + 'blosc/frame.c', 'blosc/stune.c', 'blosc/delta.c', 'blosc/shuffle.c', + 'blosc/shuffle-generic.c', 'blosc/bitshuffle-generic.c', + 'blosc/trunc-prec.c', 'blosc/timestamp.c', 'blosc/sframe.c', + 'blosc/directories.c', 'blosc/blosc2-stdio.c', 'blosc/b2nd.c', + 'blosc/b2nd_utils.c', + # plugins: registries and the plugins they register (ZFP excluded) + 'plugins/codecs/codecs-registry.c', + 'plugins/codecs/ndlz/ndlz.c', 'plugins/codecs/ndlz/ndlz4x4.c', + 'plugins/codecs/ndlz/ndlz8x8.c', 'plugins/codecs/ndlz/xxhash.c', + 'plugins/filters/filters-registry.c', + 'plugins/filters/bytedelta/bytedelta.c', + 'plugins/filters/int_trunc/int_trunc.c', + 'plugins/filters/ndcell/ndcell.c', 'plugins/filters/ndmean/ndmean.c', + 'plugins/tuners/tuners-registry.c', +) +if host_system == 'windows' + sources += files('blosc/win32/threading.c') +endif + +# --- SIMD kernels: detected by compiler capability (cross-compile safe) --- +# Each kernel file gets its ISA flag; shuffle.c only learns that it may +# dispatch to them at run time (SHUFFLE_*_ENABLED) and is built portably. +shuffle_defs = [] +simd_libs = [] +msvc = cc.get_argument_syntax() == 'msvc' + +if cpu in ['x86', 'x86_64'] + if not get_option('sse2').disabled() and cc.has_argument(msvc ? '/arch:SSE2' : '-msse2') + simd_libs += static_library('blosc2_sse2', + files('blosc/shuffle-sse2.c', 'blosc/bitshuffle-sse2.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + (msvc ? [] : ['-msse2']), + dependencies: deps) + shuffle_defs += '-DSHUFFLE_SSE2_ENABLED' + endif + if not get_option('avx2').disabled() and cc.has_argument(msvc ? '/arch:AVX2' : '-mavx2') + simd_libs += static_library('blosc2_avx2', + files('blosc/shuffle-avx2.c', 'blosc/bitshuffle-avx2.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + (msvc ? ['/arch:AVX2'] : ['-mavx2']), + dependencies: deps) + shuffle_defs += '-DSHUFFLE_AVX2_ENABLED' + endif + if not get_option('avx512').disabled() and not msvc and cc.has_argument('-mavx512bw') + simd_libs += static_library('blosc2_avx512', + files('blosc/bitshuffle-avx512.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + ['-mavx512f', '-mavx512bw'], + dependencies: deps) + shuffle_defs += '-DSHUFFLE_AVX512_ENABLED' + endif +endif + +if cpu == 'aarch64' and not get_option('neon').disabled() + simd_libs += static_library('blosc2_neon', + files('blosc/shuffle-neon.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + ['-flax-vector-conversions'], + dependencies: deps) + shuffle_defs += '-DSHUFFLE_NEON_ENABLED' +endif + +blosc2_inc = include_directories('include') + +blosc2_lib = static_library('blosc2', sources, + include_directories: [blosc2_inc, include_directories('blosc')], + c_args: blosc2_args + shuffle_defs, + dependencies: deps, + link_whole: simd_libs, + pic: true, +) + +blosc2_dep = declare_dependency( + link_with: blosc2_lib, + include_directories: blosc2_inc, + dependencies: deps, + compile_args: ['-DBUILD_STATIC'], +) +meson.override_dependency('blosc2', blosc2_dep) diff --git a/subprojects/packagefiles/c-blosc2/meson.options b/subprojects/packagefiles/c-blosc2/meson.options new file mode 100644 index 00000000..45623a18 --- /dev/null +++ b/subprojects/packagefiles/c-blosc2/meson.options @@ -0,0 +1,12 @@ +option('zstd', type: 'feature', value: 'enabled', + description: 'Build Zstandard support into Blosc2') +option('zlib', type: 'feature', value: 'enabled', + description: 'Build zlib support into Blosc2') +option('sse2', type: 'feature', value: 'auto', + description: 'Enable SSE2 shuffle kernels') +option('avx2', type: 'feature', value: 'auto', + description: 'Enable AVX2 shuffle kernels') +option('avx512', type: 'feature', value: 'auto', + description: 'Enable AVX512 bitshuffle kernels') +option('neon', type: 'feature', value: 'auto', + description: 'Enable NEON shuffle kernels') diff --git a/subprojects/zlib.wrap b/subprojects/zlib.wrap new file mode 100644 index 00000000..0626401a --- /dev/null +++ b/subprojects/zlib.wrap @@ -0,0 +1,14 @@ +[wrap-file] +directory = zlib-1.3.2 +source_url = https://zlib.net/zlib-1.3.2.tar.xz +source_fallback_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.2-1/get_source/zlib-1.3.2.tar.xz +source_filename = zlib-1.3.2.tar.xz +source_hash = d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3 +patch_filename = zlib_1.3.2-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.2-1/get_patch +patch_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zlib_1.3.2-1/zlib_1.3.2-1_patch.zip +patch_hash = 5ae7a2e92f823df118cfb8c1b23d94e3117864392b3446581d669049b2fba6dd +wrapdb_version = 1.3.2-1 + +[provide] +dependency_names = zlib diff --git a/subprojects/zstd.wrap b/subprojects/zstd.wrap new file mode 100644 index 00000000..fc6e4fcb --- /dev/null +++ b/subprojects/zstd.wrap @@ -0,0 +1,13 @@ +[wrap-file] +directory = zstd-1.5.7 +source_url = https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz +source_filename = zstd-1.5.7.tar.gz +source_hash = eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 +patch_filename = zstd_1.5.7-3_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zstd_1.5.7-3/get_patch +patch_hash = 75ccb7b5bd26ae38b9d2db40ea4915504137835c15857ea833e70af1fe6f8686 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zstd_1.5.7-3/zstd-1.5.7.tar.gz +wrapdb_version = 1.5.7-3 + +[provide] +libzstd = libzstd_dep diff --git a/tests/test_blosc.py b/tests/test_blosc.py index b920f03e..18147786 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -1,3 +1,4 @@ +import struct import threading import time from multiprocessing import Pool @@ -38,6 +39,8 @@ Blosc(shuffle=Blosc.SHUFFLE, blocksize=2**8), Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE, blocksize=2**8), ] +# compressors can be left out of a build (e.g. -Dblosc_zstd=disabled with c-blosc2) +codecs = [c if c is None or c.cname in blosc.list_compressors() else None for c in codecs] # mix of dtypes: integer, float, bool, string @@ -57,6 +60,7 @@ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'), np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'), np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'), + np.empty(0, dtype='u1'), ] @@ -77,6 +81,35 @@ def test_encode_decode(array, codec): check_encode_decode(array, codec) +def test_empty_encode_decode(use_threads): + blosc.use_threads = use_threads + try: + check_encode_decode(np.empty(0, dtype='u1'), Blosc()) + finally: + blosc.use_threads = None + + +def test_empty_decode_rejects_nonempty_destination(): + codec = Blosc() + encoded = codec.encode(b'') + with pytest.raises(RuntimeError, match='non-empty destination buffer'): + codec.decode(encoded, out=bytearray(1)) + + +def test_decompress_allows_trailing_bytes(): + codec = Blosc() + original = b'some data to compress' + encoded = codec.encode(original) + assert codec.decode(encoded + b'padding') == original + + +def test_decompress_rejects_truncated_frame(): + codec = Blosc() + encoded = codec.encode(b'some data to compress') + with pytest.raises(RuntimeError, match='buffer is truncated'): + codec.decode(encoded[:-1]) + + def test_config(): codec = Blosc(cname='zstd', clevel=3, shuffle=1) check_config(codec) @@ -165,7 +198,11 @@ def test_compress_metainfo(dtype, use_threads): enc = blosc.compress(arr, cname.encode(), 1, shuffle) typesize, did_shuffle, _ = blosc._cbuffer_metainfo(enc) assert typesize == arr.dtype.itemsize - assert did_shuffle == shuffle + if typesize == 1 and shuffle == Blosc.SHUFFLE and blosc.LIBRARY == 'c-blosc2': + # c-blosc2 omits the byte shuffle flag when it is a no-op + assert did_shuffle in (Blosc.NOSHUFFLE, shuffle) + else: + assert did_shuffle == shuffle def test_compress_autoshuffle(use_threads): @@ -203,7 +240,8 @@ def test_backwards_compatibility(): def _encode_worker(data): - compressor = Blosc(cname='zlib', clevel=9, shuffle=Blosc.SHUFFLE) + cname = 'zlib' if 'zlib' in blosc.list_compressors() else 'lz4' + compressor = Blosc(cname=cname, clevel=9, shuffle=Blosc.SHUFFLE) return compressor.encode(data) @@ -326,7 +364,86 @@ def test_typesize_less_than_1(): compressor.encode(arr.tobytes()) +@pytest.mark.parametrize( + "shuffle", [Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE, Blosc.AUTOSHUFFLE] +) +def test_zero_itemsize(shuffle): + # a structured dtype with a zero-length field has itemsize 0, which blosc + # rejects as a typesize (and older versions crashed on, #516) + arr = np.zeros((5,), dtype=[("foo", int, (0,))]) + assert arr.dtype.itemsize == 0 + enc = Blosc(shuffle=shuffle).encode(arr) + nbytes, _, _ = blosc._cbuffer_sizes(enc) + assert nbytes == 0 + + def test_config_no_typesize(): codec = Blosc(shuffle=Blosc.SHUFFLE, typesize=5) config = codec.get_config() assert "typesize" not in config + + +@pytest.mark.parametrize('shuffle', [Blosc.SHUFFLE, Blosc.BITSHUFFLE]) +@pytest.mark.parametrize('blocksize', [0, 2**8, 4097]) +@pytest.mark.parametrize('nbytes', [1, 31, 33, 127, 1000, 65537]) +@pytest.mark.parametrize('typesize', [1, 2, 3, 4, 7, 8, 12, 16, 17, 32]) +def test_shuffle_roundtrip_typesizes(typesize, nbytes, blocksize, shuffle): + # Exercises the SIMD (un)shuffle kernels, including the scalar tail used when a + # block is not a multiple of the vector width or the typesize. + src = np.random.default_rng(0).integers(0, 64, nbytes, dtype='u1').tobytes() + codec = Blosc(cname='lz4', clevel=5, shuffle=shuffle, blocksize=blocksize, typesize=typesize) + assert bytes(codec.decode(codec.encode(src))) == src + + +@pytest.mark.parametrize('cname', blosc.list_compressors()) +@pytest.mark.parametrize('shuffle', [Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE]) +@pytest.mark.parametrize('typesize', [1, 2, 3, 8]) +def test_output_is_blosc1_format(cname, shuffle, typesize): + # Chunks must stay readable by C-Blosc 1.x: format version 2 in the first byte + src = np.arange(4096, dtype='u1').tobytes() + enc = Blosc(cname=cname, shuffle=shuffle, typesize=typesize).encode(src) + assert enc[0] == 2 + + +def _blosc1_header(nbytes, cbytes, typesize=1, blocksize=1): + # version 2, lz4, no flags, then nbytes, blocksize, cbytes (little endian) + return struct.pack('> np.uint64(8 * j)) & np.uint64(0xFF) + elif kind == "periodic": + period = rng.integers(1, 9) + pattern = rng.integers(0, 256, size=(period, itemsize), dtype=np.uint8) + items[:] = pattern[np.arange(nelements) % period] + elif kind == "sparse": + mask = rng.random(items.shape) < 0.02 + items[mask] = rng.integers(0, 256, size=int(mask.sum()), dtype=np.uint8) + tail = rng.integers(0, 256, size=leftover, dtype=np.uint8) + return np.concatenate([items.reshape(-1), tail]).tobytes() + + +@st.composite +def payloads(draw): + itemsize = draw(st.sampled_from(EDGE_ITEMSIZES) | st.integers(1, 300)) + nelements = draw(st.sampled_from(EDGE_ELEMENTS) | st.integers(0, 4096)) + # the bytes that do not make up a whole item are copied through unfiltered + leftover = draw(st.sampled_from([0, 0, 1, itemsize - 1, itemsize // 2])) + leftover = max(0, min(leftover, itemsize - 1)) + kind = draw(st.sampled_from(KINDS)) + seed = draw(st.integers(0, 2**32 - 1)) + return itemsize, make_payload(kind, seed, itemsize, nelements, leftover) + + +compress_params = st.fixed_dictionaries( + { + "cname": st.sampled_from(CNAMES), + "clevel": st.integers(0, 9), + "shuffle": st.sampled_from(SHUFFLES), + "blocksize": st.sampled_from(EDGE_BLOCKSIZES) | st.integers(0, 2**20), + } +) + + +def v1_typesize(itemsize): + return itemsize if itemsize <= MAX_TYPESIZE else 1 + + +def v1_compress(data, itemsize, cname, clevel, shuffle, blocksize): + """Compress with C-Blosc 1.x.""" + blosc1.set_nthreads(1) + blosc1.set_blocksize(blocksize) + try: + return blosc1.compress( + data, typesize=v1_typesize(itemsize), clevel=clevel, shuffle=shuffle, cname=cname + ) + finally: + blosc1.set_blocksize(0) + + +@contextlib.contextmanager +def numcodecs_path(path): + """Make numcodecs compress and decompress through one of its two code paths. + + numcodecs uses blosc's global context when it may use threads, which is the default in + the main thread, and a context of its own otherwise (``use_threads = False``, other + threads, forked processes). With the c-blosc2 backend the two differ in how they ask for + Blosc1 chunks, so both have to be checked. ``use_threads`` is a global, so this is a + context manager and not a (function-scoped) fixture, which hypothesis does not allow. + """ + saved = nc_blosc.use_threads + nc_blosc.use_threads = path == "global" + try: + yield + finally: + nc_blosc.use_threads = saved + + +PATHS = pytest.mark.parametrize("path", ["global", "context"]) + + +def nc_compress(data, itemsize, cname, clevel, shuffle, blocksize): + """Compress with numcodecs, the way zarr does: bytes and an explicit typesize.""" + return nc_blosc.Blosc( + cname=cname, clevel=clevel, shuffle=shuffle, blocksize=blocksize, typesize=itemsize + ).encode(np.frombuffer(data, dtype=np.uint8)) + + +def check_header(chunk, data, itemsize): + """A chunk is a sound Blosc1 chunk of ``data``, whoever wrote it.""" + assert chunk[0] == 2, "format version is not the Blosc1 one" + nbytes, cbytes, blocksize = blosc1.get_cbuffer_sizes(chunk) + assert nbytes == len(data) + assert cbytes == len(chunk) + if data: + assert blocksize <= len(data) + assert chunk[3] == v1_typesize(itemsize) + + +@PATHS +@SETTINGS +@given(payload=payloads(), params=compress_params) +def test_decodes_in_c_blosc1(path, payload, params): + """Forward: what numcodecs writes, C-Blosc 1.x reads.""" + itemsize, data = payload + with numcodecs_path(path): + chunk = nc_compress(data, itemsize, **params) + check_header(chunk, data, itemsize) + assert blosc1.decompress(chunk) == data + + +@PATHS +@SETTINGS +@given(payload=payloads(), params=compress_params) +def test_decodes_from_c_blosc1(path, payload, params): + """Backward: what C-Blosc 1.x writes, numcodecs reads.""" + itemsize, data = payload + chunk = v1_compress(data, itemsize, **params) + with numcodecs_path(path): + assert bytes(nc_blosc.decompress(chunk)) == data + + +@PATHS +@SETTINGS +@given(payload=payloads(), params=compress_params) +def test_numcodecs_roundtrip(path, payload, params): + itemsize, data = payload + with numcodecs_path(path): + chunk = nc_compress(data, itemsize, **params) + assert bytes(nc_blosc.decompress(chunk)) == data + # the codec without a typesize takes it from the array, here bytes + codec = nc_blosc.Blosc(**params) + assert bytes(codec.decode(codec.encode(np.frombuffer(data, dtype=np.uint8)))) == data + + +@PATHS +@pytest.mark.parametrize( + ("itemsize", "dtype"), [(1, "u1"), (2, "u2"), (4, "f4"), (8, "f8"), (16, "c16")] +) +@SETTINGS +@given( + params=compress_params, + nelements=st.sampled_from(EDGE_ELEMENTS) | st.integers(0, 2048), + seed=st.integers(0, 2**32 - 1), +) +def test_typed_arrays(path, itemsize, dtype, params, nelements, seed): + """Arrays give numcodecs their typesize themselves, without a typesize argument.""" + data = make_payload("random" if seed % 2 else "ramp", seed, itemsize, nelements, 0) + array = np.frombuffer(data, dtype=dtype) + with numcodecs_path(path): + chunk = nc_blosc.compress( + array, + params["cname"].encode(), + params["clevel"], + params["shuffle"], + # a blocksize is not an array property, so ask for the automatic one here + 0, + ) + assert bytes(nc_blosc.decompress(chunk)) == data + check_header(chunk, data, itemsize) + assert blosc1.decompress(chunk) == data + + +# A deterministic grid for the same, so that a failure names its parameters and runs the +# same everywhere: every compressor x shuffle x typesize x size, in both directions +GRID = list( + itertools.product( + CNAMES, + SHUFFLES, + [1, 2, 3, 4, 7, 8, 16, 17, 255, 300], + [0, 1, 31, 32, 33, 127, 1001, 4096, 65537, 100003], + ) +) + + +@PATHS +@pytest.mark.parametrize(("cname", "shuffle", "itemsize", "nbytes"), GRID) +def test_grid(path, cname, shuffle, itemsize, nbytes): + data = make_payload("ramp", 1234, itemsize, nbytes // itemsize, nbytes % itemsize) + assert len(data) == nbytes + params = {"cname": cname, "clevel": 5, "shuffle": shuffle, "blocksize": 0} + + with numcodecs_path(path): + chunk = nc_compress(data, itemsize, **params) + check_header(chunk, data, itemsize) + assert blosc1.decompress(chunk) == data + + chunk = v1_compress(data, itemsize, **params) + with numcodecs_path(path): + assert bytes(nc_blosc.decompress(chunk)) == data diff --git a/tests/test_blosc2_package.py b/tests/test_blosc2_package.py new file mode 100644 index 00000000..c30816b6 --- /dev/null +++ b/tests/test_blosc2_package.py @@ -0,0 +1,70 @@ +"""Checks on packages/c-blosc2-blosc1-compat, which only exist in a source checkout.""" + +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE = ROOT / "packages" / "c-blosc2-blosc1-compat" +WRAP = ROOT / "subprojects" / "c-blosc2.wrap" + +pytestmark = pytest.mark.skipif(not PACKAGE.exists(), reason="not a source checkout") + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, PACKAGE / "tools" / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _run(script, *args): + return subprocess.run( + [sys.executable, str(PACKAGE / "tools" / script), *args], + capture_output=True, + text=True, + check=False, + ) + + +def test_wrap_copies_are_in_sync(): + result = _run("sync_wrap.py", "--check") + assert result.returncode == 0, result.stderr + + +def test_wrap_matches_upstream_pin(): + pin = dict( + line.split(" = ") + for line in (PACKAGE / "UPSTREAM").read_text().splitlines() + if " = " in line + ) + wrap = WRAP.read_text() + assert f"url = {pin['repository']}.git" in wrap + assert f"revision = {pin['commit']}" in wrap + + +def test_sync_wrap_detects_a_stale_copy(tmp_path, monkeypatch): + sync_wrap = _load("sync_wrap") + package = tmp_path / "packages" / "pkg" + (package / "meson").mkdir(parents=True) + (package / "meson" / "meson.build").write_text("project('x')\n") + dest = tmp_path / "subprojects" / "packagefiles" / "c-blosc2" + monkeypatch.setattr(sync_wrap, "PKG", package) + monkeypatch.setattr(sync_wrap, "DEST", dest) + + monkeypatch.setattr(sys, "argv", ["sync_wrap.py", "--check"]) + assert sync_wrap.main() == 1 # nothing copied yet + monkeypatch.setattr(sys, "argv", ["sync_wrap.py"]) + assert sync_wrap.main() == 0 + monkeypatch.setattr(sys, "argv", ["sync_wrap.py", "--check"]) + assert sync_wrap.main() == 0 + + (dest / "meson.build").write_text("changed\n") + (dest / "stale.txt").write_text("left over\n") + assert sync_wrap.main() == 1 + monkeypatch.setattr(sys, "argv", ["sync_wrap.py"]) + assert sync_wrap.main() == 0 + assert not (dest / "stale.txt").exists()