From be18d32d5777cb3fe8bb8e9c2fa7d78a3582b3ed Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 20 Oct 2026 16:52:06 -0400 Subject: [PATCH 01/10] Add c-blosc2 meson wrap with a Blosc1-format patch series packages/c-blosc2-blosc1-compat holds a meson build for c-blosc2 (upstream is CMake only), four patches that let c-blosc2 write chunks C-Blosc 1.x can decode, and a check that decodes 735 c-blosc2 chunks with C-Blosc 1.x and the reverse. subprojects/ has the c-blosc2 wrap (patched through packagefiles, kept in sync by tools/sync_wrap.py) and WrapDB wraps for zstd, lz4 and zlib. A workflow checks the sync, runs the compatibility check, and weekly tries the series on upstream main. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/blosc2-compat.yaml | 59 ++++++++ .gitignore | 7 +- packages/c-blosc2-blosc1-compat/README.md | 71 ++++++++++ packages/c-blosc2-blosc1-compat/UPSTREAM | 4 + .../c-blosc2-blosc1-compat/meson/meson.build | 130 ++++++++++++++++++ .../meson/meson.options | 12 ++ ...001-reader-bitunshuffle-v2-leftovers.patch | 21 +++ .../patches/0002-blosc1-format-flag.patch | 95 +++++++++++++ .../0003-blosc1-format-bitshuffle.patch | 24 ++++ .../0004-blosc1-format-split-rule.patch | 33 +++++ .../c-blosc2-blosc1-compat/tests/c/common.h | 12 ++ .../c-blosc2-blosc1-compat/tests/c/cons.c | 22 +++ .../c-blosc2-blosc1-compat/tests/c/cons2.c | 22 +++ .../c-blosc2-blosc1-compat/tests/c/prod.c | 36 +++++ .../c-blosc2-blosc1-compat/tests/c/prod1.c | 11 ++ packages/c-blosc2-blosc1-compat/tests/run.py | 63 +++++++++ .../tools/make_wrap_diff_files.py | 6 + .../c-blosc2-blosc1-compat/tools/rebase.py | 42 ++++++ .../c-blosc2-blosc1-compat/tools/sync_wrap.py | 45 ++++++ subprojects/c-blosc2.wrap | 10 ++ subprojects/lz4.wrap | 13 ++ subprojects/packagefiles/c-blosc2/meson.build | 130 ++++++++++++++++++ .../packagefiles/c-blosc2/meson.options | 12 ++ ...001-reader-bitunshuffle-v2-leftovers.patch | 21 +++ .../patches/0002-blosc1-format-flag.patch | 95 +++++++++++++ .../0003-blosc1-format-bitshuffle.patch | 24 ++++ .../0004-blosc1-format-split-rule.patch | 33 +++++ subprojects/zlib.wrap | 14 ++ subprojects/zstd.wrap | 13 ++ 29 files changed, 1079 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/blosc2-compat.yaml create mode 100644 packages/c-blosc2-blosc1-compat/README.md create mode 100644 packages/c-blosc2-blosc1-compat/UPSTREAM create mode 100644 packages/c-blosc2-blosc1-compat/meson/meson.build create mode 100644 packages/c-blosc2-blosc1-compat/meson/meson.options create mode 100644 packages/c-blosc2-blosc1-compat/patches/0001-reader-bitunshuffle-v2-leftovers.patch create mode 100644 packages/c-blosc2-blosc1-compat/patches/0002-blosc1-format-flag.patch create mode 100644 packages/c-blosc2-blosc1-compat/patches/0003-blosc1-format-bitshuffle.patch create mode 100644 packages/c-blosc2-blosc1-compat/patches/0004-blosc1-format-split-rule.patch create mode 100644 packages/c-blosc2-blosc1-compat/tests/c/common.h create mode 100644 packages/c-blosc2-blosc1-compat/tests/c/cons.c create mode 100644 packages/c-blosc2-blosc1-compat/tests/c/cons2.c create mode 100644 packages/c-blosc2-blosc1-compat/tests/c/prod.c create mode 100644 packages/c-blosc2-blosc1-compat/tests/c/prod1.c create mode 100644 packages/c-blosc2-blosc1-compat/tests/run.py create mode 100644 packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py create mode 100644 packages/c-blosc2-blosc1-compat/tools/rebase.py create mode 100644 packages/c-blosc2-blosc1-compat/tools/sync_wrap.py create mode 100644 subprojects/c-blosc2.wrap create mode 100644 subprojects/lz4.wrap create mode 100644 subprojects/packagefiles/c-blosc2/meson.build create mode 100644 subprojects/packagefiles/c-blosc2/meson.options create mode 100644 subprojects/packagefiles/c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch create mode 100644 subprojects/packagefiles/c-blosc2/patches/0002-blosc1-format-flag.patch create mode 100644 subprojects/packagefiles/c-blosc2/patches/0003-blosc1-format-bitshuffle.patch create mode 100644 subprojects/packagefiles/c-blosc2/patches/0004-blosc1-format-split-rule.patch create mode 100644 subprojects/zlib.wrap create mode 100644 subprojects/zstd.wrap diff --git a/.github/workflows/blosc2-compat.yaml b/.github/workflows/blosc2-compat.yaml new file mode 100644 index 000000000..e9ea3c5ad --- /dev/null +++ b/.github/workflows/blosc2-compat.yaml @@ -0,0 +1,59 @@ +name: c-blosc2 Blosc1 compatibility + +on: + push: + branches: [main] + paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h'] + pull_request: + paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h'] + schedule: + - cron: '0 6 * * 1' # weekly: does the series still apply to upstream c-blosc2 main? + workflow_dispatch: + +jobs: + wrap-in-sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: python packages/c-blosc2-blosc1-compat/tools/sync_wrap.py --check + + forward-compat: + # c-blosc2 chunks must be readable by C-Blosc 1.x + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: pip install meson ninja cmake + - name: Build C-Blosc 1.x + run: | + git clone --depth 1 --branch v1.21.6 https://github.com/Blosc/c-blosc.git /tmp/c-blosc + cmake -S /tmp/c-blosc -B /tmp/b1 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release + cmake --build /tmp/b1 -j + - name: Build the patched c-blosc2 through the wrap + run: | + mkdir /tmp/probe && cd /tmp/probe + ln -s "$GITHUB_WORKSPACE/subprojects" subprojects + printf "project('probe','c')\nb = dependency('blosc2')\nexecutable('probe', files('p.c'), dependencies: b)\n" > meson.build + printf '#include \nint main(void){return 0;}\n' > p.c + meson setup b && meson compile -C b + - name: Run the compatibility check + run: | + B=/tmp/probe/b/subprojects + python packages/c-blosc2-blosc1-compat/tests/run.py \ + --blosc1-include /tmp/c-blosc/blosc --blosc1-lib /tmp/b1/blosc \ + --blosc2-include "$(echo /tmp/probe/subprojects/c-blosc2-*/include)" \ + --blosc2-ldflags "$(echo $B/c-blosc2-*/libblosc2.a) $(echo $B/zstd-*/build/meson/lib/libzstd.a) $(echo $B/lz4-*/build/meson/meson/lib/liblz4.a) $(echo $B/zlib-*/libz.a)" + + rebase: + # Informational: does the series still apply to upstream main? Fails only on the schedule. + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: python packages/c-blosc2-blosc1-compat/tools/rebase.py main diff --git a/.gitignore b/.gitignore index 14facfe79..2fbc0cc90 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,9 @@ ENV/ .pixi/* *.egg-info pixi.lock -uv.lock \ No newline at end of file +uv.lock +# Wrapped subprojects: only the .wrap files and packagefiles are tracked +/subprojects/*/ +!/subprojects/packagefiles/ +/subprojects/packagecache/ +/subprojects/.wraplock diff --git a/packages/c-blosc2-blosc1-compat/README.md b/packages/c-blosc2-blosc1-compat/README.md new file mode 100644 index 000000000..311400476 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/README.md @@ -0,0 +1,71 @@ +# c-blosc2-blosc1-compat + +An overlay for [c-blosc2](https://github.com/Blosc/c-blosc2) that makes it usable as the +backend of numcodecs' `Blosc` codec: a meson build, plus a small patch series that lets +c-blosc2 **write chunks that C-Blosc 1.x can decode**. Stock c-blosc2 cannot, even through +its `blosc1_*` API or with `BLOSC_BLOSC1_COMPAT` (its README says so, and +`tests/run.py` confirms it). + +This directory is self-contained (it does not import anything from numcodecs) so it can be +split out with `git subtree split` if it ever needs to live in its own repository, or be +deleted if upstream accepts the patches. + +## Contents + +| Path | What | +|---|---| +| `UPSTREAM` | the pinned c-blosc2 commit and source hash; `subprojects/c-blosc2.wrap` must match | +| `patches/` | the patch series, applied in order; one concern per patch | +| `meson/` | `meson.build` and `meson.options` for c-blosc2 (upstream only has CMake) | +| `tests/` | the forward-compatibility check, see below | +| `tools/` | `sync_wrap.py`, `rebase.py` | + +## The patches + +Format version 2 (Blosc1) differs from what c-blosc2 writes in three ways that matter to a +reader, plus one reader bug: + +1. `0001` reader: bytes left over after a bit-shuffled version-2 block were not copied. + A plain bug fix, independent of the rest. +2. `0002` an opt-in flag: write the 16-byte Blosc1 header with format version 2. Set with + `blosc2_ctx_set_blosc1_format()` or, for the global context, `BLOSC_BLOSC1_COMPAT`. + Off by default, no ABI change. +3. `0003` bitshuffle: version 2 shuffles a block only if its element count is a multiple + of 8, and stores it unshuffled otherwise. +4. `0004` block splitting: use the C-Blosc 1.x rule, which its reader recomputes. + +Only plain shuffle/bitshuffle with blosclz, lz4, lz4hc, zlib and zstd are supported in this +mode. The output decodes with C-Blosc 1.x but is not byte-identical to what it writes. + +## Wiring into the build + +Meson only looks for `patch_directory` and `diff_files` in `subprojects/packagefiles`, so +`subprojects/packagefiles/c-blosc2/` holds copies of `meson/` and `patches/`. After editing +here run `python tools/sync_wrap.py`; CI runs it with `--check`. If a patch is added or +removed, update `diff_files` in `subprojects/c-blosc2.wrap` +(`tools/make_wrap_diff_files.py` prints the line). + +## Testing + +`tests/run.py` builds small C programs against C-Blosc 1.x and the patched c-blosc2 and +checks 735 cases in three directions (see its docstring for the command line): +c-blosc2 global-context to C-Blosc 1.x, c-blosc2 context API to C-Blosc 1.x, and C-Blosc 1.x +to c-blosc2. All must decode. + +## Updating c-blosc2 + +`python tools/rebase.py ` downloads that c-blosc2, tries to apply the +series, and reports which patch stops applying. To bump: edit `UPSTREAM` and +`subprojects/c-blosc2.wrap` (directory, URL, hash), fix any patch that fails, re-export it +as a unified diff against the pristine tree (`diff -ruN a b`), run `tools/sync_wrap.py` +and the tests. The scheduled workflow `blosc2-compat-rebase` does the first step weekly. + +## Upstreaming + +Track each patch here when a PR exists, and delete the patch once it is in a release +that `UPSTREAM` pins: + +| Patch | Upstream | +|---|---| +| 0001 | not yet submitted | +| 0002–0004 | not yet submitted | diff --git a/packages/c-blosc2-blosc1-compat/UPSTREAM b/packages/c-blosc2-blosc1-compat/UPSTREAM new file mode 100644 index 000000000..98bf86eba --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/UPSTREAM @@ -0,0 +1,4 @@ +repository = https://github.com/Blosc/c-blosc2 +commit = 38258631914de7f6ca2cf55c9675d771796ab96c +version = 3.3.6.dev +sha256 = c5556a00d8fc6a6a7be17e8f54e5c2aae2d304e740ac1aeb245db0f5731950a6 diff --git a/packages/c-blosc2-blosc1-compat/meson/meson.build b/packages/c-blosc2-blosc1-compat/meson/meson.build new file mode 100644 index 000000000..ceb48c8a2 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/meson/meson.build @@ -0,0 +1,130 @@ +# Meson build for c-blosc2, which upstream only builds with CMake. +# Only the core library plus the built-in codec/filter/tuner plugins are built +# (no ZFP, tests, benchmarks or examples). Used via subprojects/c-blosc2.wrap. +project( + 'c-blosc2', 'c', + version: '3.3.6.dev', + meson_version: '>=1.6.0', + default_options: ['c_std=gnu11', 'warning_level=0'], +) + +cc = meson.get_compiler('c') +host_system = host_machine.system() +cpu = host_machine.cpu_family() + +blosc2_args = ['-DBUILD_STATIC'] +if host_system != 'windows' + blosc2_args += ['-D_GNU_SOURCE'] +else + blosc2_args += ['-D_CRT_SECURE_NO_WARNINGS'] +endif + +deps = [dependency('threads')] +if host_system != 'windows' + deps += cc.find_library('m', required: false) + deps += cc.find_library('dl', required: false) +endif + +# --- compression libraries (resolved by the parent project or WrapDB) --- +lz4_dep = dependency('liblz4', fallback: ['lz4', 'liblz4_dep'], + default_options: ['default_library=static']) +deps += lz4_dep +blosc2_args += ['-DHAVE_LZ4'] + +zstd_dep = dependency('libzstd', required: get_option('zstd'), + fallback: ['zstd', 'libzstd_dep'], + default_options: ['default_library=static']) +if zstd_dep.found() + deps += zstd_dep + blosc2_args += ['-DHAVE_ZSTD'] +endif + +zlib_dep = dependency('zlib', required: get_option('zlib'), + fallback: ['zlib', 'zlib_dep'], + default_options: ['default_library=static']) +if zlib_dep.found() + deps += zlib_dep + blosc2_args += ['-DHAVE_ZLIB'] +endif + +sources = files( + 'blosc/blosc2.c', 'blosc/blosclz.c', 'blosc/fastcopy.c', 'blosc/schunk.c', + 'blosc/frame.c', 'blosc/stune.c', 'blosc/delta.c', 'blosc/shuffle.c', + 'blosc/shuffle-generic.c', 'blosc/bitshuffle-generic.c', + 'blosc/trunc-prec.c', 'blosc/timestamp.c', 'blosc/sframe.c', + 'blosc/directories.c', 'blosc/blosc2-stdio.c', 'blosc/b2nd.c', + 'blosc/b2nd_utils.c', + # plugins: registries and the plugins they register (ZFP excluded) + 'plugins/codecs/codecs-registry.c', + 'plugins/codecs/ndlz/ndlz.c', 'plugins/codecs/ndlz/ndlz4x4.c', + 'plugins/codecs/ndlz/ndlz8x8.c', 'plugins/codecs/ndlz/xxhash.c', + 'plugins/filters/filters-registry.c', + 'plugins/filters/bytedelta/bytedelta.c', + 'plugins/filters/int_trunc/int_trunc.c', + 'plugins/filters/ndcell/ndcell.c', 'plugins/filters/ndmean/ndmean.c', + 'plugins/tuners/tuners-registry.c', +) +if host_system == 'windows' + sources += files('blosc/win32/threading.c') +endif + +# --- SIMD kernels: detected by compiler capability (cross-compile safe) --- +# Each kernel file gets its ISA flag; shuffle.c only learns that it may +# dispatch to them at run time (SHUFFLE_*_ENABLED) and is built portably. +shuffle_defs = [] +simd_libs = [] +msvc = cc.get_argument_syntax() == 'msvc' + +if cpu in ['x86', 'x86_64'] + if not get_option('sse2').disabled() and cc.has_argument(msvc ? '/arch:SSE2' : '-msse2') + simd_libs += static_library('blosc2_sse2', + files('blosc/shuffle-sse2.c', 'blosc/bitshuffle-sse2.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + (msvc ? [] : ['-msse2']), + dependencies: deps) + shuffle_defs += '-DSHUFFLE_SSE2_ENABLED' + endif + if not get_option('avx2').disabled() and cc.has_argument(msvc ? '/arch:AVX2' : '-mavx2') + simd_libs += static_library('blosc2_avx2', + files('blosc/shuffle-avx2.c', 'blosc/bitshuffle-avx2.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + (msvc ? ['/arch:AVX2'] : ['-mavx2']), + dependencies: deps) + shuffle_defs += '-DSHUFFLE_AVX2_ENABLED' + endif + if not get_option('avx512').disabled() and not msvc and cc.has_argument('-mavx512bw') + simd_libs += static_library('blosc2_avx512', + files('blosc/bitshuffle-avx512.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + ['-mavx512f', '-mavx512bw'], + dependencies: deps) + shuffle_defs += '-DSHUFFLE_AVX512_ENABLED' + endif +endif + +if cpu == 'aarch64' and not get_option('neon').disabled() + simd_libs += static_library('blosc2_neon', + files('blosc/shuffle-neon.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + ['-flax-vector-conversions'], + dependencies: deps) + shuffle_defs += '-DSHUFFLE_NEON_ENABLED' +endif + +blosc2_inc = include_directories('include') + +blosc2_lib = static_library('blosc2', sources, + include_directories: [blosc2_inc, include_directories('blosc')], + c_args: blosc2_args + shuffle_defs, + dependencies: deps, + link_whole: simd_libs, + pic: true, +) + +blosc2_dep = declare_dependency( + link_with: blosc2_lib, + include_directories: blosc2_inc, + dependencies: deps, + compile_args: ['-DBUILD_STATIC'], +) +meson.override_dependency('blosc2', blosc2_dep) diff --git a/packages/c-blosc2-blosc1-compat/meson/meson.options b/packages/c-blosc2-blosc1-compat/meson/meson.options new file mode 100644 index 000000000..45623a184 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/meson/meson.options @@ -0,0 +1,12 @@ +option('zstd', type: 'feature', value: 'enabled', + description: 'Build Zstandard support into Blosc2') +option('zlib', type: 'feature', value: 'enabled', + description: 'Build zlib support into Blosc2') +option('sse2', type: 'feature', value: 'auto', + description: 'Enable SSE2 shuffle kernels') +option('avx2', type: 'feature', value: 'auto', + description: 'Enable AVX2 shuffle kernels') +option('avx512', type: 'feature', value: 'auto', + description: 'Enable AVX512 bitshuffle kernels') +option('neon', type: 'feature', value: 'auto', + description: 'Enable NEON shuffle kernels') diff --git a/packages/c-blosc2-blosc1-compat/patches/0001-reader-bitunshuffle-v2-leftovers.patch b/packages/c-blosc2-blosc1-compat/patches/0001-reader-bitunshuffle-v2-leftovers.patch new file mode 100644 index 000000000..04671582b --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/patches/0001-reader-bitunshuffle-v2-leftovers.patch @@ -0,0 +1,21 @@ +From: numcodecs contributors +Subject: [PATCH] Copy the leftover bytes when bit-unshuffling a version 2 block + +A Blosc1 (format version 2) block whose element count is a multiple of 8 +stores the bytes that do not fill a whole element after the shuffled part. +The reader skipped them, so chunks written by C-Blosc 1.x with a typesize +that does not divide the block size were decoded with garbage at the end. +--- +diff -ruN a/blosc/shuffle.c b/blosc/shuffle.c +--- a/blosc/shuffle.c ++++ b/blosc/shuffle.c +@@ -498,6 +498,9 @@ + BLOSC_TRACE_ERROR("the impossible happened: the bitunshuffle filter failed!"); + return ret; + } ++ /* Copy the leftovers, as Blosc1 does */ ++ size_t offset = size * typesize; ++ memcpy((uint8_t *) dest + offset, (const uint8_t *) src + offset, blocksize - offset); + } + else { + memcpy(dest, src, blocksize); diff --git a/packages/c-blosc2-blosc1-compat/patches/0002-blosc1-format-flag.patch b/packages/c-blosc2-blosc1-compat/patches/0002-blosc1-format-flag.patch new file mode 100644 index 000000000..8ca87d0b7 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/patches/0002-blosc1-format-flag.patch @@ -0,0 +1,95 @@ +From: numcodecs contributors +Subject: [PATCH] Add an opt-in switch to write Blosc1 (format version 2) chunks + +A compression context gets a blosc1_format flag. When set, the chunk header is +the 16-byte Blosc1 header with format version 2. The flag is set with the new +blosc2_ctx_set_blosc1_format(), and for the global context by the existing +BLOSC_BLOSC1_COMPAT environment variable. Off by default; no change to the +default Blosc2 output. The filter and block-split changes that the version-2 +format also requires follow in the next patches. +--- +diff -ruN a/blosc/context.h b/blosc/context.h +--- a/blosc/context.h ++++ b/blosc/context.h +@@ -40,6 +40,7 @@ + uint8_t header_flags; /* Flags for header */ + uint8_t blosc2_flags; /* Flags specific for blosc2 */ + uint8_t blosc2_flags2; /* Secondary flags specific for blosc2 */ ++ bool blosc1_format; /* Write Blosc1 (format version 2) chunks, readable by C-Blosc 1.x */ + int32_t sourcesize; /* Number of bytes in source buffer */ + int32_t header_overhead; /* The number of bytes in chunk header */ + int32_t nblocks; /* Number of total blocks in buffer */ +diff -ruN a/include/blosc2.h b/include/blosc2.h +--- a/include/blosc2.h ++++ b/include/blosc2.h +@@ -2719,6 +2719,20 @@ + */ + BLOSC_EXPORT void blosc1_set_splitmode(int splitmode); + ++/** ++ * @brief Make a compression context write Blosc1 chunks. ++ * ++ * The chunks use format version 2 and the 16-byte header, so that they can also be ++ * decoded by C-Blosc 1.x. Only plain shuffle/bitshuffle filters and the blosclz, ++ * lz4, lz4hc, zlib and zstd codecs can be used. Contexts are not Blosc1 by default. ++ * ++ * @param context A compression context. ++ * @param enable Whether to write Blosc1 chunks. ++ * ++ * @return 0 on success, a negative error code otherwise. ++ */ ++BLOSC_EXPORT int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable); ++ + + /** + * @brief Get the offsets of a frame in a super-chunk. +diff -ruN a/blosc/blosc2.c b/blosc/blosc2.c +--- a/blosc/blosc2.c ++++ b/blosc/blosc2.c +@@ -1008,6 +1008,9 @@ + header->version = (context->blosc2_flags2 & BLOSC2_VL_BLOCKS) ? + BLOSC2_VERSION_FORMAT_VL_BLOCKS : + BLOSC2_VERSION_FORMAT_STABLE; ++ if (context->blosc1_format) { ++ header->version = BLOSC1_VERSION_FORMAT; ++ } + header->versionlz = compcode_to_compversion(context->compcode); + header->flags = context->header_flags; + header->typesize = (uint8_t)context->typesize; +@@ -3153,6 +3161,15 @@ + } + + ++int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable) { ++ if (context == NULL || context->do_compress != 1) { ++ return BLOSC2_ERROR_INVALID_PARAM; ++ } ++ context->blosc1_format = enable; ++ return BLOSC2_ERROR_SUCCESS; ++} ++ ++ + /* The public secure routine for compression with context. */ + int blosc2_compress_ctx(blosc2_context* context, const void* src, int32_t srcsize, + void* dest, int32_t destsize) { +@@ -3173,8 +3190,8 @@ + return error; + } + +- /* Write the extended header */ +- error = write_compression_header(context, true); ++ /* Write the extended header, or the Blosc1 header if requested */ ++ error = write_compression_header(context, !context->blosc1_format); + if (error < 0) { + return error; + } +@@ -3896,6 +3913,9 @@ + } + + blosc2_pthread_mutex_lock(&global_comp_mutex); ++ ++ /* Blosc1 compatibility mode is read before anything depends on it (e.g. block splitting) */ ++ g_global_context->blosc1_format = (getenv("BLOSC_BLOSC1_COMPAT") != NULL); + + /* Initialize a context compression */ + uint8_t* filters = calloc(1, BLOSC2_MAX_FILTERS); diff --git a/packages/c-blosc2-blosc1-compat/patches/0003-blosc1-format-bitshuffle.patch b/packages/c-blosc2-blosc1-compat/patches/0003-blosc1-format-bitshuffle.patch new file mode 100644 index 000000000..9146d0bc0 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/patches/0003-blosc1-format-bitshuffle.patch @@ -0,0 +1,24 @@ +From: numcodecs contributors +Subject: [PATCH] Follow the version 2 bitshuffle rule when writing Blosc1 chunks + +Format version 2 bit-shuffles a block only when its number of elements is a +multiple of 8, and stores it unshuffled otherwise. Blosc2 shuffles the largest +multiple-of-8 prefix instead, which C-Blosc 1.x decodes to wrong data. +--- +diff -ruN a/blosc/blosc2.c b/blosc/blosc2.c +--- a/blosc/blosc2.c ++++ b/blosc/blosc2.c +@@ -1119,7 +1122,12 @@ + blosc2_shuffle(filters_meta[i] == 0 ? typesize : filters_meta[i], bsize, _src, _dest); + break; + case BLOSC_BITSHUFFLE: +- if (blosc2_bitshuffle(typesize, bsize, _src, _dest) < 0) { ++ if (context->blosc1_format && ((bsize / typesize) % 8) != 0) { ++ // Format version 2 (Blosc1) only bit-shuffles blocks whose number of ++ // elements is a multiple of 8; other blocks are stored unshuffled. ++ memcpy(_dest, _src, (size_t)bsize); ++ } ++ else if (blosc2_bitshuffle(typesize, bsize, _src, _dest) < 0) { + return NULL; + } + break; diff --git a/packages/c-blosc2-blosc1-compat/patches/0004-blosc1-format-split-rule.patch b/packages/c-blosc2-blosc1-compat/patches/0004-blosc1-format-split-rule.patch new file mode 100644 index 000000000..09c4bcac5 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/patches/0004-blosc1-format-split-rule.patch @@ -0,0 +1,33 @@ +From: numcodecs contributors +Subject: [PATCH] Use the C-Blosc 1.x block split rule when writing Blosc1 chunks + +A C-Blosc 1.x reader recomputes whether a block is split from the typesize and +blocksize (typesize <= 16 and at least 128 elements); Blosc2 uses a smaller +minimum, so it could write split blocks the reader treats as unsplit. +--- +diff -ruN a/blosc/stune.c b/blosc/stune.c +--- a/blosc/stune.c ++++ b/blosc/stune.c +@@ -184,6 +184,22 @@ + } + + int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) { ++ if (context->blosc1_format) { ++ // C-Blosc 1.x readers infer splitting from these sizes (and the "dont split" flag), ++ // so a split is only possible if this condition holds. ++ int blosc1_splittable = (typesize <= MAX_STREAMS) && (blocksize / typesize) >= 128; ++ switch (context->splitmode) { ++ case BLOSC_ALWAYS_SPLIT: ++ return blosc1_splittable; ++ case BLOSC_NEVER_SPLIT: ++ return 0; ++ case BLOSC_AUTO_SPLIT: ++ return blosc1_splittable && context->compcode == BLOSC_BLOSCLZ; ++ default: ++ // The zstd support was introduced at the same time than the split flag ++ return blosc1_splittable && context->compcode != BLOSC_ZSTD; ++ } ++ } + switch (context->splitmode) { + case BLOSC_ALWAYS_SPLIT: + return 1; diff --git a/packages/c-blosc2-blosc1-compat/tests/c/common.h b/packages/c-blosc2-blosc1-compat/tests/c/common.h new file mode 100644 index 000000000..c59dc23c5 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/common.h @@ -0,0 +1,12 @@ +#include +#include +#include +#include +static const char *cnames[] = {"blosclz","lz4","lz4hc","zlib","zstd"}; +static const int shuffles[] = {0,1,2}; +static const int typesizes[] = {1,2,3,4,7,8,16}; +static const size_t sizes[] = {1, 33, 127, 1001, 4096, 65537, 100003}; +static void fill(uint8_t *p, size_t n, int ts, unsigned seed) { + unsigned x = seed; + for (size_t i = 0; i < n; i++) { if ((i % ts) == 0) x = x*1103515245u + 12345u; p[i] = (uint8_t)((i/ts)*3 + ((i%ts)? (x>>16)&0x7 : 0)); } +} diff --git a/packages/c-blosc2-blosc1-compat/tests/c/cons.c b/packages/c-blosc2-blosc1-compat/tests/c/cons.c new file mode 100644 index 000000000..2a535d4c5 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/cons.c @@ -0,0 +1,22 @@ +// consumer: c-blosc1 decompress; verifies against regenerated data +#include +#include "common.h" +int main(int argc, char **argv) { + const char *mode = argv[1]; + blosc_init(); blosc_set_nthreads(1); + int ok = 0, bad = 0; + for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { + size_t nb = sizes[z]; int ts = typesizes[t]; + + char fn[256]; snprintf(fn, sizeof fn, "out/%s_%s_s%d_t%d_n%zu.bin", mode, cnames[c], shuffles[s], ts, nb); + FILE *f = fopen(fn, "rb"); if (!f) { printf("missing %s\n", fn); continue; } + uint8_t *cbuf = malloc(nb + 1024); size_t cb = fread(cbuf, 1, nb + 1024, f); fclose(f); + uint8_t *want = malloc(nb), *got = calloc(1, nb); fill(want, nb, ts, 1234); + int r = blosc_decompress(cbuf, got, nb); + if (r == (int)nb && !memcmp(want, got, nb)) ok++; + else { if (bad < 100000) printf("BAD %s ret=%d ver=%d\n", fn, r, cbuf[0]); bad++; } + free(cbuf); free(want); free(got); + } + printf("%s: c-blosc1 decoded ok=%d bad=%d\n", mode, ok, bad); + return bad != 0; +} diff --git a/packages/c-blosc2-blosc1-compat/tests/c/cons2.c b/packages/c-blosc2-blosc1-compat/tests/c/cons2.c new file mode 100644 index 000000000..21cb6d884 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/cons2.c @@ -0,0 +1,22 @@ +// consumer: c-blosc1 decompress; verifies against regenerated data +#include +#include "common.h" +int main(int argc, char **argv) { + const char *mode = argv[1]; + blosc2_init(); blosc2_set_nthreads(1); + int ok = 0, bad = 0; + for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { + size_t nb = sizes[z]; int ts = typesizes[t]; + + char fn[256]; snprintf(fn, sizeof fn, "out/%s_%s_s%d_t%d_n%zu.bin", mode, cnames[c], shuffles[s], ts, nb); + FILE *f = fopen(fn, "rb"); if (!f) { printf("missing %s\n", fn); continue; } + uint8_t *cbuf = malloc(nb + 1024); size_t cb = fread(cbuf, 1, nb + 1024, f); fclose(f); + uint8_t *want = malloc(nb), *got = calloc(1, nb); fill(want, nb, ts, 1234); + int r = blosc1_decompress(cbuf, got, nb); + if (r == (int)nb && !memcmp(want, got, nb)) ok++; + else { if (bad < 100000) printf("BAD %s ret=%d ver=%d\n", fn, r, cbuf[0]); bad++; } + free(cbuf); free(want); free(got); + } + printf("%s: c-blosc1 decoded ok=%d bad=%d\n", mode, ok, bad); + return bad != 0; +} diff --git a/packages/c-blosc2-blosc1-compat/tests/c/prod.c b/packages/c-blosc2-blosc1-compat/tests/c/prod.c new file mode 100644 index 000000000..ab494a1bd --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/prod.c @@ -0,0 +1,36 @@ +// producer: c-blosc2, mode: "global" (blosc1_compress, compat env), "global_noenv", "ctx" +#include +#include "common.h" +int main(int argc, char **argv) { + const char *mode = argv[1]; + if (!strcmp(mode, "global")) setenv("BLOSC_BLOSC1_COMPAT", "TRUE", 1); + blosc2_init(); blosc2_set_nthreads(1); if (getenv("SPLITMODE")) blosc1_set_splitmode(atoi(getenv("SPLITMODE"))); + int n = 0; + for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { + size_t nb = sizes[z]; int ts = typesizes[t]; + + uint8_t *src = malloc(nb), *dst = malloc(nb + BLOSC2_MAX_OVERHEAD); + fill(src, nb, ts, 1234); + int cb; + if (!strcmp(mode, "ctx")) { + blosc2_cparams cp = BLOSC2_CPARAMS_DEFAULTS; + cp.compcode = blosc2_compname_to_compcode(cnames[c]); cp.clevel = 5; cp.typesize = ts; cp.nthreads = 1; + memset(cp.filters, 0, sizeof cp.filters); + if (shuffles[s]==1) cp.filters[BLOSC2_MAX_FILTERS-1] = BLOSC_SHUFFLE; + if (shuffles[s]==2) cp.filters[BLOSC2_MAX_FILTERS-1] = BLOSC_BITSHUFFLE; + blosc2_context *ctx = blosc2_create_cctx(cp); + blosc2_ctx_set_blosc1_format(ctx, true); + cb = blosc2_compress_ctx(ctx, src, (int32_t)nb, dst, (int32_t)(nb + BLOSC2_MAX_OVERHEAD)); + blosc2_free_ctx(ctx); + } else { + blosc1_set_compressor(cnames[c]); + cb = blosc1_compress(5, shuffles[s], ts, nb, src, dst, nb + BLOSC2_MAX_OVERHEAD); + } + if (cb <= 0) { fprintf(stderr, "compress fail %s s%d t%d n%zu: %d\n", cnames[c], s, t, nb, cb); return 2; } + char fn[256]; snprintf(fn, sizeof fn, "out/%s_%s_s%d_t%d_n%zu.bin", mode, cnames[c], s, ts, nb); + FILE *f = fopen(fn, "wb"); fwrite(dst, 1, cb, f); fclose(f); n++; + free(src); free(dst); + } + printf("%s: wrote %d buffers; first byte(version) of last = see check\n", mode, n); + return 0; +} diff --git a/packages/c-blosc2-blosc1-compat/tests/c/prod1.c b/packages/c-blosc2-blosc1-compat/tests/c/prod1.c new file mode 100644 index 000000000..0b78cd0d4 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/c/prod1.c @@ -0,0 +1,11 @@ +#include +#include "common.h" +int main(void){ blosc_init(); blosc_set_nthreads(1); + for (int c=0;c<5;c++) for(int s=0;s<3;s++) for(int t=0;t<7;t++) for(int z=0;z<7;z++){ + size_t nb=sizes[z]; int ts=typesizes[t]; + uint8_t *src=malloc(nb),*dst=malloc(nb+BLOSC_MAX_OVERHEAD); fill(src,nb,ts,1234); + blosc_set_compressor(cnames[c]); int cb=blosc_compress(5,shuffles[s],ts,nb,src,dst,nb+BLOSC_MAX_OVERHEAD); + if(cb<=0){fprintf(stderr,"fail\n");return 2;} + char fn[256]; snprintf(fn,sizeof fn,"out/b1_%s_s%d_t%d_n%zu.bin",cnames[c],shuffles[s],ts,nb); + FILE*f=fopen(fn,"wb"); fwrite(dst,1,cb,f); fclose(f);} + return 0;} diff --git a/packages/c-blosc2-blosc1-compat/tests/run.py b/packages/c-blosc2-blosc1-compat/tests/run.py new file mode 100644 index 000000000..cd99919e8 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tests/run.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Check that chunks written by the patched c-blosc2 can be read by C-Blosc 1.x. + +Compiles the programs in tests/c against both libraries and runs, over every +compressor in {blosclz, lz4, lz4hc, zlib, zstd} x shuffle {none, byte, bit} x typesize +x buffer size (including sizes that are not a multiple of the typesize): + + global, ctx c-blosc2 writes Blosc1 chunks through blosc1_compress / blosc2_compress_ctx, + C-Blosc 1.x decodes them. + b1 C-Blosc 1.x writes, c-blosc2 decodes (backward compatibility). + +Example: + run.py --blosc1-include c-blosc/blosc --blosc1-lib build-b1/blosc \\ + --blosc2-include c-blosc2/include --blosc2-ldflags "libblosc2.a libzstd.a liblz4.a -lz" +""" +import argparse +import os +import shlex +import subprocess +import sys +import tempfile +from pathlib import Path + +C = Path(__file__).resolve().parent / "c" + + +def sh(cmd, **kw): + subprocess.run(cmd, check=True, **kw) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--blosc1-include", required=True, help="directory with blosc.h (C-Blosc 1.x)") + ap.add_argument("--blosc1-lib", required=True, help="directory with the C-Blosc 1.x shared library") + ap.add_argument("--blosc2-include", required=True, help="directory with blosc2.h (patched c-blosc2)") + ap.add_argument("--blosc2-ldflags", required=True, help="libraries to link c-blosc2 and its codecs") + ap.add_argument("--cc", default=os.environ.get("CC", "cc")) + a = ap.parse_args() + + lib1 = Path(a.blosc1_lib).resolve() + ld1 = ["-L" + str(lib1), "-lblosc", "-Wl,-rpath," + str(lib1)] + ld2 = shlex.split(a.blosc2_ldflags) + ["-lpthread", "-lm"] + inc = lambda d: "-I" + str(Path(d).resolve()) # noqa: E731 + + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + (tmp / "out").mkdir() + cc = [a.cc, "-O1", "-I" + str(C)] + sh(cc + [inc(a.blosc2_include), str(C / "prod.c"), "-o", "prod"] + ld2, cwd=tmp) + sh(cc + [inc(a.blosc1_include), str(C / "cons.c"), "-o", "cons"] + ld1, cwd=tmp) + sh(cc + [inc(a.blosc1_include), str(C / "prod1.c"), "-o", "prod1"] + ld1, cwd=tmp) + sh(cc + [inc(a.blosc2_include), str(C / "cons2.c"), "-o", "cons2"] + ld2, cwd=tmp) + failed = False + for mode in ("global", "ctx"): + sh(["./prod", mode], cwd=tmp, stdout=subprocess.DEVNULL) + failed |= subprocess.run(["./cons", mode], cwd=tmp).returncode != 0 + sh(["./prod1"], cwd=tmp) + failed |= subprocess.run(["./cons2", "b1"], cwd=tmp).returncode != 0 + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py b/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py new file mode 100644 index 000000000..d762a92fb --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +"""Print the `diff_files` line for subprojects/c-blosc2.wrap from the patch series.""" +from pathlib import Path + +pkg = Path(__file__).resolve().parents[1] +print("diff_files = " + ", ".join(f"c-blosc2/patches/{p.name}" for p in sorted((pkg / "patches").glob("*.patch")))) diff --git a/packages/c-blosc2-blosc1-compat/tools/rebase.py b/packages/c-blosc2-blosc1-compat/tools/rebase.py new file mode 100644 index 000000000..bc5104ada --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tools/rebase.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Check that the patch series applies to a c-blosc2 commit or tag. + +Usage: rebase.py [REF] (default: the commit in UPSTREAM) +""" +import subprocess +import sys +import tarfile +import tempfile +import urllib.request +from pathlib import Path + +PKG = Path(__file__).resolve().parents[1] + + +def main(): + ref = sys.argv[1] if len(sys.argv) > 1 else dict( + line.split(" = ") for line in (PKG / "UPSTREAM").read_text().split("\n") if " = " in line + )["commit"] + url = f"https://github.com/Blosc/c-blosc2/archive/{ref}.tar.gz" + with tempfile.TemporaryDirectory() as tmp: + tar = Path(tmp) / "src.tar.gz" + urllib.request.urlretrieve(url, tar) + with tarfile.open(tar) as t: + t.extractall(tmp, **({"filter": "data"} if hasattr(tarfile, "data_filter") else {})) + (src,) = [d for d in Path(tmp).iterdir() if d.is_dir()] + status = 0 + for patch in sorted((PKG / "patches").glob("*.patch")): + r = subprocess.run( + ["patch", "-p1", "--no-backup-if-mismatch", "-i", str(patch)], + cwd=src, capture_output=True, text=True, + ) + print(f"{'ok ' if r.returncode == 0 else 'FAIL'} {patch.name}") + if r.returncode: + print(r.stdout + r.stderr) + status = 1 + break + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py b/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py new file mode 100644 index 000000000..2c67d94d5 --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Copy this package's build overlay and patches into subprojects/packagefiles/c-blosc2. + +Meson wraps only look for `patch_directory` and `diff_files` in subprojects/packagefiles, +so that directory holds copies of what lives here. Run without arguments to update the +copies, or with --check (used in CI) to fail if they are out of date. +""" +import argparse +import filecmp +import shutil +import sys +from pathlib import Path + +PKG = Path(__file__).resolve().parents[1] +DEST = PKG.parents[1] / "subprojects" / "packagefiles" / "c-blosc2" + + +def sources(): + for f in sorted((PKG / "meson").iterdir()): + yield f, DEST / f.name + for f in sorted((PKG / "patches").glob("*.patch")): + yield f, DEST / "patches" / f.name + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--check", action="store_true") + check = ap.parse_args().check + expected = {dst for _, dst in sources()} + stale = [d for d in DEST.rglob("*") if d.is_file() and d not in expected] + bad = [dst for src, dst in sources() if not dst.exists() or not filecmp.cmp(src, dst, shallow=False)] + if check: + for f in bad + stale: + print(f"out of sync: {f}", file=sys.stderr) + return 1 if bad or stale else 0 + for src, dst in sources(): + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dst) + for f in stale: + f.unlink() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/subprojects/c-blosc2.wrap b/subprojects/c-blosc2.wrap new file mode 100644 index 000000000..6654e73fe --- /dev/null +++ b/subprojects/c-blosc2.wrap @@ -0,0 +1,10 @@ +[wrap-file] +directory = c-blosc2-38258631914de7f6ca2cf55c9675d771796ab96c +source_url = https://github.com/Blosc/c-blosc2/archive/38258631914de7f6ca2cf55c9675d771796ab96c.tar.gz +source_filename = c-blosc2-38258631914de7f6ca2cf55c9675d771796ab96c.tar.gz +source_hash = c5556a00d8fc6a6a7be17e8f54e5c2aae2d304e740ac1aeb245db0f5731950a6 +patch_directory = c-blosc2 +diff_files = c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch, c-blosc2/patches/0002-blosc1-format-flag.patch, c-blosc2/patches/0003-blosc1-format-bitshuffle.patch, c-blosc2/patches/0004-blosc1-format-split-rule.patch + +[provide] +dependency_names = blosc2 diff --git a/subprojects/lz4.wrap b/subprojects/lz4.wrap new file mode 100644 index 000000000..7732b9a2b --- /dev/null +++ b/subprojects/lz4.wrap @@ -0,0 +1,13 @@ +[wrap-file] +directory = lz4-1.10.0 +source_url = https://github.com/lz4/lz4/archive/v1.10.0.tar.gz +source_filename = lz4-1.10.0.tgz +source_hash = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b +patch_filename = lz4_1.10.0-2_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/lz4_1.10.0-2/get_patch +patch_hash = ce568b4e1a7a593bb233f3ec97af88ceb850524cef802fe3f916b21d8a79b6b6 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/lz4_1.10.0-2/lz4-1.10.0.tgz +wrapdb_version = 1.10.0-2 + +[provide] +liblz4 = liblz4_dep diff --git a/subprojects/packagefiles/c-blosc2/meson.build b/subprojects/packagefiles/c-blosc2/meson.build new file mode 100644 index 000000000..ceb48c8a2 --- /dev/null +++ b/subprojects/packagefiles/c-blosc2/meson.build @@ -0,0 +1,130 @@ +# Meson build for c-blosc2, which upstream only builds with CMake. +# Only the core library plus the built-in codec/filter/tuner plugins are built +# (no ZFP, tests, benchmarks or examples). Used via subprojects/c-blosc2.wrap. +project( + 'c-blosc2', 'c', + version: '3.3.6.dev', + meson_version: '>=1.6.0', + default_options: ['c_std=gnu11', 'warning_level=0'], +) + +cc = meson.get_compiler('c') +host_system = host_machine.system() +cpu = host_machine.cpu_family() + +blosc2_args = ['-DBUILD_STATIC'] +if host_system != 'windows' + blosc2_args += ['-D_GNU_SOURCE'] +else + blosc2_args += ['-D_CRT_SECURE_NO_WARNINGS'] +endif + +deps = [dependency('threads')] +if host_system != 'windows' + deps += cc.find_library('m', required: false) + deps += cc.find_library('dl', required: false) +endif + +# --- compression libraries (resolved by the parent project or WrapDB) --- +lz4_dep = dependency('liblz4', fallback: ['lz4', 'liblz4_dep'], + default_options: ['default_library=static']) +deps += lz4_dep +blosc2_args += ['-DHAVE_LZ4'] + +zstd_dep = dependency('libzstd', required: get_option('zstd'), + fallback: ['zstd', 'libzstd_dep'], + default_options: ['default_library=static']) +if zstd_dep.found() + deps += zstd_dep + blosc2_args += ['-DHAVE_ZSTD'] +endif + +zlib_dep = dependency('zlib', required: get_option('zlib'), + fallback: ['zlib', 'zlib_dep'], + default_options: ['default_library=static']) +if zlib_dep.found() + deps += zlib_dep + blosc2_args += ['-DHAVE_ZLIB'] +endif + +sources = files( + 'blosc/blosc2.c', 'blosc/blosclz.c', 'blosc/fastcopy.c', 'blosc/schunk.c', + 'blosc/frame.c', 'blosc/stune.c', 'blosc/delta.c', 'blosc/shuffle.c', + 'blosc/shuffle-generic.c', 'blosc/bitshuffle-generic.c', + 'blosc/trunc-prec.c', 'blosc/timestamp.c', 'blosc/sframe.c', + 'blosc/directories.c', 'blosc/blosc2-stdio.c', 'blosc/b2nd.c', + 'blosc/b2nd_utils.c', + # plugins: registries and the plugins they register (ZFP excluded) + 'plugins/codecs/codecs-registry.c', + 'plugins/codecs/ndlz/ndlz.c', 'plugins/codecs/ndlz/ndlz4x4.c', + 'plugins/codecs/ndlz/ndlz8x8.c', 'plugins/codecs/ndlz/xxhash.c', + 'plugins/filters/filters-registry.c', + 'plugins/filters/bytedelta/bytedelta.c', + 'plugins/filters/int_trunc/int_trunc.c', + 'plugins/filters/ndcell/ndcell.c', 'plugins/filters/ndmean/ndmean.c', + 'plugins/tuners/tuners-registry.c', +) +if host_system == 'windows' + sources += files('blosc/win32/threading.c') +endif + +# --- SIMD kernels: detected by compiler capability (cross-compile safe) --- +# Each kernel file gets its ISA flag; shuffle.c only learns that it may +# dispatch to them at run time (SHUFFLE_*_ENABLED) and is built portably. +shuffle_defs = [] +simd_libs = [] +msvc = cc.get_argument_syntax() == 'msvc' + +if cpu in ['x86', 'x86_64'] + if not get_option('sse2').disabled() and cc.has_argument(msvc ? '/arch:SSE2' : '-msse2') + simd_libs += static_library('blosc2_sse2', + files('blosc/shuffle-sse2.c', 'blosc/bitshuffle-sse2.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + (msvc ? [] : ['-msse2']), + dependencies: deps) + shuffle_defs += '-DSHUFFLE_SSE2_ENABLED' + endif + if not get_option('avx2').disabled() and cc.has_argument(msvc ? '/arch:AVX2' : '-mavx2') + simd_libs += static_library('blosc2_avx2', + files('blosc/shuffle-avx2.c', 'blosc/bitshuffle-avx2.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + (msvc ? ['/arch:AVX2'] : ['-mavx2']), + dependencies: deps) + shuffle_defs += '-DSHUFFLE_AVX2_ENABLED' + endif + if not get_option('avx512').disabled() and not msvc and cc.has_argument('-mavx512bw') + simd_libs += static_library('blosc2_avx512', + files('blosc/bitshuffle-avx512.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + ['-mavx512f', '-mavx512bw'], + dependencies: deps) + shuffle_defs += '-DSHUFFLE_AVX512_ENABLED' + endif +endif + +if cpu == 'aarch64' and not get_option('neon').disabled() + simd_libs += static_library('blosc2_neon', + files('blosc/shuffle-neon.c'), + include_directories: include_directories('include', 'blosc'), + c_args: blosc2_args + ['-flax-vector-conversions'], + dependencies: deps) + shuffle_defs += '-DSHUFFLE_NEON_ENABLED' +endif + +blosc2_inc = include_directories('include') + +blosc2_lib = static_library('blosc2', sources, + include_directories: [blosc2_inc, include_directories('blosc')], + c_args: blosc2_args + shuffle_defs, + dependencies: deps, + link_whole: simd_libs, + pic: true, +) + +blosc2_dep = declare_dependency( + link_with: blosc2_lib, + include_directories: blosc2_inc, + dependencies: deps, + compile_args: ['-DBUILD_STATIC'], +) +meson.override_dependency('blosc2', blosc2_dep) diff --git a/subprojects/packagefiles/c-blosc2/meson.options b/subprojects/packagefiles/c-blosc2/meson.options new file mode 100644 index 000000000..45623a184 --- /dev/null +++ b/subprojects/packagefiles/c-blosc2/meson.options @@ -0,0 +1,12 @@ +option('zstd', type: 'feature', value: 'enabled', + description: 'Build Zstandard support into Blosc2') +option('zlib', type: 'feature', value: 'enabled', + description: 'Build zlib support into Blosc2') +option('sse2', type: 'feature', value: 'auto', + description: 'Enable SSE2 shuffle kernels') +option('avx2', type: 'feature', value: 'auto', + description: 'Enable AVX2 shuffle kernels') +option('avx512', type: 'feature', value: 'auto', + description: 'Enable AVX512 bitshuffle kernels') +option('neon', type: 'feature', value: 'auto', + description: 'Enable NEON shuffle kernels') diff --git a/subprojects/packagefiles/c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch b/subprojects/packagefiles/c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch new file mode 100644 index 000000000..04671582b --- /dev/null +++ b/subprojects/packagefiles/c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch @@ -0,0 +1,21 @@ +From: numcodecs contributors +Subject: [PATCH] Copy the leftover bytes when bit-unshuffling a version 2 block + +A Blosc1 (format version 2) block whose element count is a multiple of 8 +stores the bytes that do not fill a whole element after the shuffled part. +The reader skipped them, so chunks written by C-Blosc 1.x with a typesize +that does not divide the block size were decoded with garbage at the end. +--- +diff -ruN a/blosc/shuffle.c b/blosc/shuffle.c +--- a/blosc/shuffle.c ++++ b/blosc/shuffle.c +@@ -498,6 +498,9 @@ + BLOSC_TRACE_ERROR("the impossible happened: the bitunshuffle filter failed!"); + return ret; + } ++ /* Copy the leftovers, as Blosc1 does */ ++ size_t offset = size * typesize; ++ memcpy((uint8_t *) dest + offset, (const uint8_t *) src + offset, blocksize - offset); + } + else { + memcpy(dest, src, blocksize); diff --git a/subprojects/packagefiles/c-blosc2/patches/0002-blosc1-format-flag.patch b/subprojects/packagefiles/c-blosc2/patches/0002-blosc1-format-flag.patch new file mode 100644 index 000000000..8ca87d0b7 --- /dev/null +++ b/subprojects/packagefiles/c-blosc2/patches/0002-blosc1-format-flag.patch @@ -0,0 +1,95 @@ +From: numcodecs contributors +Subject: [PATCH] Add an opt-in switch to write Blosc1 (format version 2) chunks + +A compression context gets a blosc1_format flag. When set, the chunk header is +the 16-byte Blosc1 header with format version 2. The flag is set with the new +blosc2_ctx_set_blosc1_format(), and for the global context by the existing +BLOSC_BLOSC1_COMPAT environment variable. Off by default; no change to the +default Blosc2 output. The filter and block-split changes that the version-2 +format also requires follow in the next patches. +--- +diff -ruN a/blosc/context.h b/blosc/context.h +--- a/blosc/context.h ++++ b/blosc/context.h +@@ -40,6 +40,7 @@ + uint8_t header_flags; /* Flags for header */ + uint8_t blosc2_flags; /* Flags specific for blosc2 */ + uint8_t blosc2_flags2; /* Secondary flags specific for blosc2 */ ++ bool blosc1_format; /* Write Blosc1 (format version 2) chunks, readable by C-Blosc 1.x */ + int32_t sourcesize; /* Number of bytes in source buffer */ + int32_t header_overhead; /* The number of bytes in chunk header */ + int32_t nblocks; /* Number of total blocks in buffer */ +diff -ruN a/include/blosc2.h b/include/blosc2.h +--- a/include/blosc2.h ++++ b/include/blosc2.h +@@ -2719,6 +2719,20 @@ + */ + BLOSC_EXPORT void blosc1_set_splitmode(int splitmode); + ++/** ++ * @brief Make a compression context write Blosc1 chunks. ++ * ++ * The chunks use format version 2 and the 16-byte header, so that they can also be ++ * decoded by C-Blosc 1.x. Only plain shuffle/bitshuffle filters and the blosclz, ++ * lz4, lz4hc, zlib and zstd codecs can be used. Contexts are not Blosc1 by default. ++ * ++ * @param context A compression context. ++ * @param enable Whether to write Blosc1 chunks. ++ * ++ * @return 0 on success, a negative error code otherwise. ++ */ ++BLOSC_EXPORT int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable); ++ + + /** + * @brief Get the offsets of a frame in a super-chunk. +diff -ruN a/blosc/blosc2.c b/blosc/blosc2.c +--- a/blosc/blosc2.c ++++ b/blosc/blosc2.c +@@ -1008,6 +1008,9 @@ + header->version = (context->blosc2_flags2 & BLOSC2_VL_BLOCKS) ? + BLOSC2_VERSION_FORMAT_VL_BLOCKS : + BLOSC2_VERSION_FORMAT_STABLE; ++ if (context->blosc1_format) { ++ header->version = BLOSC1_VERSION_FORMAT; ++ } + header->versionlz = compcode_to_compversion(context->compcode); + header->flags = context->header_flags; + header->typesize = (uint8_t)context->typesize; +@@ -3153,6 +3161,15 @@ + } + + ++int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable) { ++ if (context == NULL || context->do_compress != 1) { ++ return BLOSC2_ERROR_INVALID_PARAM; ++ } ++ context->blosc1_format = enable; ++ return BLOSC2_ERROR_SUCCESS; ++} ++ ++ + /* The public secure routine for compression with context. */ + int blosc2_compress_ctx(blosc2_context* context, const void* src, int32_t srcsize, + void* dest, int32_t destsize) { +@@ -3173,8 +3190,8 @@ + return error; + } + +- /* Write the extended header */ +- error = write_compression_header(context, true); ++ /* Write the extended header, or the Blosc1 header if requested */ ++ error = write_compression_header(context, !context->blosc1_format); + if (error < 0) { + return error; + } +@@ -3896,6 +3913,9 @@ + } + + blosc2_pthread_mutex_lock(&global_comp_mutex); ++ ++ /* Blosc1 compatibility mode is read before anything depends on it (e.g. block splitting) */ ++ g_global_context->blosc1_format = (getenv("BLOSC_BLOSC1_COMPAT") != NULL); + + /* Initialize a context compression */ + uint8_t* filters = calloc(1, BLOSC2_MAX_FILTERS); diff --git a/subprojects/packagefiles/c-blosc2/patches/0003-blosc1-format-bitshuffle.patch b/subprojects/packagefiles/c-blosc2/patches/0003-blosc1-format-bitshuffle.patch new file mode 100644 index 000000000..9146d0bc0 --- /dev/null +++ b/subprojects/packagefiles/c-blosc2/patches/0003-blosc1-format-bitshuffle.patch @@ -0,0 +1,24 @@ +From: numcodecs contributors +Subject: [PATCH] Follow the version 2 bitshuffle rule when writing Blosc1 chunks + +Format version 2 bit-shuffles a block only when its number of elements is a +multiple of 8, and stores it unshuffled otherwise. Blosc2 shuffles the largest +multiple-of-8 prefix instead, which C-Blosc 1.x decodes to wrong data. +--- +diff -ruN a/blosc/blosc2.c b/blosc/blosc2.c +--- a/blosc/blosc2.c ++++ b/blosc/blosc2.c +@@ -1119,7 +1122,12 @@ + blosc2_shuffle(filters_meta[i] == 0 ? typesize : filters_meta[i], bsize, _src, _dest); + break; + case BLOSC_BITSHUFFLE: +- if (blosc2_bitshuffle(typesize, bsize, _src, _dest) < 0) { ++ if (context->blosc1_format && ((bsize / typesize) % 8) != 0) { ++ // Format version 2 (Blosc1) only bit-shuffles blocks whose number of ++ // elements is a multiple of 8; other blocks are stored unshuffled. ++ memcpy(_dest, _src, (size_t)bsize); ++ } ++ else if (blosc2_bitshuffle(typesize, bsize, _src, _dest) < 0) { + return NULL; + } + break; diff --git a/subprojects/packagefiles/c-blosc2/patches/0004-blosc1-format-split-rule.patch b/subprojects/packagefiles/c-blosc2/patches/0004-blosc1-format-split-rule.patch new file mode 100644 index 000000000..09c4bcac5 --- /dev/null +++ b/subprojects/packagefiles/c-blosc2/patches/0004-blosc1-format-split-rule.patch @@ -0,0 +1,33 @@ +From: numcodecs contributors +Subject: [PATCH] Use the C-Blosc 1.x block split rule when writing Blosc1 chunks + +A C-Blosc 1.x reader recomputes whether a block is split from the typesize and +blocksize (typesize <= 16 and at least 128 elements); Blosc2 uses a smaller +minimum, so it could write split blocks the reader treats as unsplit. +--- +diff -ruN a/blosc/stune.c b/blosc/stune.c +--- a/blosc/stune.c ++++ b/blosc/stune.c +@@ -184,6 +184,22 @@ + } + + int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) { ++ if (context->blosc1_format) { ++ // C-Blosc 1.x readers infer splitting from these sizes (and the "dont split" flag), ++ // so a split is only possible if this condition holds. ++ int blosc1_splittable = (typesize <= MAX_STREAMS) && (blocksize / typesize) >= 128; ++ switch (context->splitmode) { ++ case BLOSC_ALWAYS_SPLIT: ++ return blosc1_splittable; ++ case BLOSC_NEVER_SPLIT: ++ return 0; ++ case BLOSC_AUTO_SPLIT: ++ return blosc1_splittable && context->compcode == BLOSC_BLOSCLZ; ++ default: ++ // The zstd support was introduced at the same time than the split flag ++ return blosc1_splittable && context->compcode != BLOSC_ZSTD; ++ } ++ } + switch (context->splitmode) { + case BLOSC_ALWAYS_SPLIT: + return 1; diff --git a/subprojects/zlib.wrap b/subprojects/zlib.wrap new file mode 100644 index 000000000..0626401ac --- /dev/null +++ b/subprojects/zlib.wrap @@ -0,0 +1,14 @@ +[wrap-file] +directory = zlib-1.3.2 +source_url = https://zlib.net/zlib-1.3.2.tar.xz +source_fallback_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.2-1/get_source/zlib-1.3.2.tar.xz +source_filename = zlib-1.3.2.tar.xz +source_hash = d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3 +patch_filename = zlib_1.3.2-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.2-1/get_patch +patch_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zlib_1.3.2-1/zlib_1.3.2-1_patch.zip +patch_hash = 5ae7a2e92f823df118cfb8c1b23d94e3117864392b3446581d669049b2fba6dd +wrapdb_version = 1.3.2-1 + +[provide] +dependency_names = zlib diff --git a/subprojects/zstd.wrap b/subprojects/zstd.wrap new file mode 100644 index 000000000..fc6e4fcb7 --- /dev/null +++ b/subprojects/zstd.wrap @@ -0,0 +1,13 @@ +[wrap-file] +directory = zstd-1.5.7 +source_url = https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz +source_filename = zstd-1.5.7.tar.gz +source_hash = eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 +patch_filename = zstd_1.5.7-3_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zstd_1.5.7-3/get_patch +patch_hash = 75ccb7b5bd26ae38b9d2db40ea4915504137835c15857ea833e70af1fe6f8686 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zstd_1.5.7-3/zstd-1.5.7.tar.gz +wrapdb_version = 1.5.7-3 + +[provide] +libzstd = libzstd_dep From f65406b7481b776e2b35c244ed7e242d6be34a5f Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 20 Oct 2026 16:52:06 -0400 Subject: [PATCH 02/10] Add an optional c-blosc2 backend for the Blosc codec -Dblosc_backend=blosc2 builds blosc.pyx against the patched c-blosc2 instead of c-blosc 1.x (still the default). A small C shim gives both libraries one API. zstd, lz4 and zlib are resolved separately from Blosc, so the zstd and lz4 codecs share them, and -Dblosc_zstd/-Dblosc_zlib=disabled leave them out of Blosc only. Chunks are always written in the Blosc1 format. Also ports the frame validation and empty-buffer handling from #854, the zero-itemsize fix from #870 and the shuffle round-trip test from #864, clamps a requested blocksize like c-blosc 1.x does, and adds tests for Blosc1 output and for builds without zstd or zlib. Co-Authored-By: Claude Sonnet 5.5 --- docs/release.md | 15 ++ meson.options | 8 +- pyproject.toml | 6 + src/numcodecs/_blosc_shim.h | 146 ++++++++++++++++ src/numcodecs/blosc.pyx | 151 ++++++++++------ src/numcodecs/meson.build | 334 +++++++++++++++++++++--------------- tests/test_blosc.py | 76 +++++++- 7 files changed, 533 insertions(+), 203 deletions(-) create mode 100644 src/numcodecs/_blosc_shim.h diff --git a/docs/release.md b/docs/release.md index ee7709335..b9012d370 100644 --- a/docs/release.md +++ b/docs/release.md @@ -18,6 +18,21 @@ ## 0.17.1 +### Enhancements + +* Add an optional c-blosc2 backend for the `Blosc` codec, selected at build time with + `-Dblosc_backend=blosc2` (default is still c-blosc 1.x). c-blosc2 is built from a meson + wrap (`subprojects/c-blosc2.wrap`) with a small patch so that it writes Blosc1-format + chunks that C-Blosc 1.x can decode (stock c-blosc2 output is not readable by C-Blosc 1.x). + zstd, lz4 and zlib are resolved separately from Blosc (system libraries or WrapDB wraps, + `-Dsystem_*`) and can be left out of Blosc with `-Dblosc_zstd=disabled` and + `-Dblosc_zlib=disabled`. `numcodecs.blosc.LIBRARY` reports the library in use; c-blosc2 + does not provide `snappy`. +* Validate Blosc frames before decompression, allowing valid zero-length frames to round-trip + while rejecting truncated frames, by {user}`Igor Stadnyk `, {issue}`831` (#854). +* `Blosc` can encode arrays whose dtype has an itemsize of 0, by + {user}`Raashish Aggarwal `, {issue}`516` (#870). + ### Security * `Pickle.decode` now refuses to unpickle data unless the process opts in, because diff --git a/meson.options b/meson.options index f055af877..e353f976d 100644 --- a/meson.options +++ b/meson.options @@ -1,5 +1,11 @@ +option('blosc_backend', type: 'combo', choices: ['blosc1', 'blosc2'], value: 'blosc1', + description: 'C library behind the Blosc codec. Both write Blosc1-format chunks.') option('system_blosc', type: 'feature', value: 'disabled', - description: 'Use system-installed Blosc library') + description: 'Use system-installed Blosc library (c-blosc 1.x for blosc_backend=blosc1, c-blosc2 for blosc2)') +option('blosc_zstd', type: 'feature', value: 'enabled', + description: 'blosc_backend=blosc2 only: build Zstandard support into Blosc') +option('blosc_zlib', type: 'feature', value: 'enabled', + description: 'blosc_backend=blosc2 only: build zlib support into Blosc') option('system_lz4', type: 'feature', value: 'disabled', description: 'Use system-installed LZ4 library') option('system_zlib', type: 'feature', value: 'disabled', diff --git a/pyproject.toml b/pyproject.toml index 185ecc3a5..453abf583 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -253,3 +253,9 @@ conflicts = [ { group = "test-zarr-main" } ] ] + +[tool.meson-python.args] +# Wrapped subprojects (c-blosc2, zstd, lz4, zlib) are static build dependencies: +# do not install their artifacts, but ship their sources in the sdist. +install = ['--skip-subprojects'] +dist = ['--include-subprojects'] diff --git a/src/numcodecs/_blosc_shim.h b/src/numcodecs/_blosc_shim.h new file mode 100644 index 000000000..49162db83 --- /dev/null +++ b/src/numcodecs/_blosc_shim.h @@ -0,0 +1,146 @@ +/* Common C API for the two Blosc backends that blosc.pyx can be built against. + * + * - c-blosc 1.x (default): thin aliases for the blosc_* API. + * - c-blosc2 (-DNUMCODECS_BLOSC2): the blosc1_*, blosc2_* API. Chunks are always written + * in the Blosc1 format (format version 2, 16-byte header) so that they stay readable + * by C-Blosc 1.x; this needs the blosc1-format patch applied to c-blosc2 (see + * subprojects/packagefiles/c-blosc2/blosc1-format.patch). c-blosc2 reads both formats. + */ +#ifndef NUMCODECS_BLOSC_SHIM_H +#define NUMCODECS_BLOSC_SHIM_H + +#include +#include + +#ifdef NUMCODECS_BLOSC2 + +#include + +#define NC_BLOSC_LIBRARY "c-blosc2" +#define NC_BLOSC_VERSION_STRING BLOSC2_VERSION_STRING +#define NC_BLOSC_VERSION_DATE BLOSC2_VERSION_DATE +/* Blosc1 chunk header size; what is written. Buffers are sized with NC_BLOSC_DEST_OVERHEAD. */ +#define NC_BLOSC_MAX_OVERHEAD BLOSC_MIN_HEADER_LENGTH +#define NC_BLOSC_DEST_OVERHEAD BLOSC2_MAX_OVERHEAD +#define NC_BLOSC_MAX_BUFFERSIZE BLOSC2_MAX_BUFFERSIZE +#define NC_BLOSC_MAX_THREADS 256 +#define NC_BLOSC_MAX_TYPESIZE BLOSC_MAX_TYPESIZE +#define NC_BLOSC_MIN_HEADER_LENGTH BLOSC_MIN_HEADER_LENGTH + +#define nc_blosc_init blosc2_init +#define nc_blosc_destroy blosc2_destroy +#define nc_blosc_get_nthreads blosc2_get_nthreads +#define nc_blosc_set_nthreads blosc2_set_nthreads +#define nc_blosc_set_compressor blosc1_set_compressor +#define nc_blosc_set_blocksize blosc1_set_blocksize +#define nc_blosc_list_compressors blosc2_list_compressors +#define nc_blosc_compress blosc1_compress +#define nc_blosc_decompress blosc1_decompress +#define nc_blosc_cbuffer_sizes blosc1_cbuffer_sizes +#define nc_blosc_cbuffer_validate blosc1_cbuffer_validate +#define nc_blosc_cbuffer_complib blosc2_cbuffer_complib +#define nc_blosc_cbuffer_metainfo blosc1_cbuffer_metainfo + +/* The global context writes Blosc1 chunks when this environment variable is set. It is + * read on each compression call, so set it before the first one. */ +static inline void nc_blosc_setup(void) { +#ifdef _WIN32 + _putenv_s("BLOSC_BLOSC1_COMPAT", "1"); +#else + setenv("BLOSC_BLOSC1_COMPAT", "1", 1); +#endif +} + +/* Equivalent of blosc_compress_ctx: no global state, so no locking is needed. */ +static inline int nc_blosc_compress_ctx( + int clevel, int doshuffle, size_t typesize, size_t nbytes, const void* src, + void* dest, size_t destsize, const char* compressor, size_t blocksize, + int numinternalthreads) { + int compcode = blosc2_compname_to_compcode(compressor); + if (compcode < 0) { + return -1; + } + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.compcode = (uint8_t)compcode; + cparams.clevel = (uint8_t)clevel; + cparams.typesize = (int32_t)typesize; + cparams.blocksize = (int32_t)blocksize; + cparams.nthreads = (int16_t)numinternalthreads; + memset(cparams.filters, 0, sizeof(cparams.filters)); + memset(cparams.filters_meta, 0, sizeof(cparams.filters_meta)); + if (doshuffle == BLOSC_SHUFFLE) { + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + } + else if (doshuffle == BLOSC_BITSHUFFLE) { + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_BITSHUFFLE; + } + blosc2_context* ctx = blosc2_create_cctx(cparams); + if (ctx == NULL) { + return -1; + } + int rc = blosc2_ctx_set_blosc1_format(ctx, true); + if (rc < 0) { + blosc2_free_ctx(ctx); + return rc; + } + rc = blosc2_compress_ctx(ctx, src, (int32_t)nbytes, dest, (int32_t)destsize); + blosc2_free_ctx(ctx); + return rc; +} + +/* Equivalent of blosc_decompress_ctx; `srcsize` is the number of valid source bytes. */ +static inline int nc_blosc_decompress_ctx( + const void* src, size_t srcsize, void* dest, size_t destsize, + int numinternalthreads) { + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = (int16_t)numinternalthreads; + blosc2_context* ctx = blosc2_create_dctx(dparams); + if (ctx == NULL) { + return -1; + } + int rc = blosc2_decompress_ctx(ctx, src, (int32_t)srcsize, dest, (int32_t)destsize); + blosc2_free_ctx(ctx); + return rc; +} + +#else /* c-blosc 1.x */ + +#include + +#define NC_BLOSC_LIBRARY "c-blosc" +#define NC_BLOSC_VERSION_STRING BLOSC_VERSION_STRING +#define NC_BLOSC_VERSION_DATE BLOSC_VERSION_DATE +#define NC_BLOSC_MAX_OVERHEAD BLOSC_MAX_OVERHEAD +#define NC_BLOSC_DEST_OVERHEAD BLOSC_MAX_OVERHEAD +#define NC_BLOSC_MAX_BUFFERSIZE BLOSC_MAX_BUFFERSIZE +#define NC_BLOSC_MAX_THREADS BLOSC_MAX_THREADS +#define NC_BLOSC_MAX_TYPESIZE BLOSC_MAX_TYPESIZE +#define NC_BLOSC_MIN_HEADER_LENGTH BLOSC_MIN_HEADER_LENGTH + +#define nc_blosc_init blosc_init +#define nc_blosc_destroy blosc_destroy +#define nc_blosc_get_nthreads blosc_get_nthreads +#define nc_blosc_set_nthreads blosc_set_nthreads +#define nc_blosc_set_compressor blosc_set_compressor +#define nc_blosc_set_blocksize blosc_set_blocksize +#define nc_blosc_list_compressors blosc_list_compressors +#define nc_blosc_compress blosc_compress +#define nc_blosc_decompress blosc_decompress +#define nc_blosc_cbuffer_sizes blosc_cbuffer_sizes +#define nc_blosc_cbuffer_validate blosc_cbuffer_validate +#define nc_blosc_cbuffer_complib blosc_cbuffer_complib +#define nc_blosc_cbuffer_metainfo blosc_cbuffer_metainfo +#define nc_blosc_compress_ctx blosc_compress_ctx + +static inline void nc_blosc_setup(void) {} + +static inline int nc_blosc_decompress_ctx( + const void* src, size_t srcsize, void* dest, size_t destsize, + int numinternalthreads) { + (void)srcsize; + return blosc_decompress_ctx(src, dest, destsize, numinternalthreads); +} + +#endif + +#endif /* NUMCODECS_BLOSC_SHIM_H */ diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index 8c52dd95c..20099c49a 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -18,50 +18,57 @@ from .compat import ensure_contiguous_ndarray from .abc import Codec -cdef extern from "blosc.h": +cdef extern from "_blosc_shim.h": cdef enum: - BLOSC_MAX_OVERHEAD, - BLOSC_VERSION_STRING, - BLOSC_VERSION_DATE, + NC_BLOSC_MAX_OVERHEAD, + NC_BLOSC_DEST_OVERHEAD, + NC_BLOSC_MIN_HEADER_LENGTH, + NC_BLOSC_VERSION_STRING, + NC_BLOSC_VERSION_DATE, + NC_BLOSC_MAX_BUFFERSIZE, + NC_BLOSC_MAX_THREADS, + NC_BLOSC_MAX_TYPESIZE, BLOSC_NOSHUFFLE, BLOSC_SHUFFLE, BLOSC_BITSHUFFLE, - BLOSC_MAX_BUFFERSIZE, - BLOSC_MAX_THREADS, - BLOSC_MAX_TYPESIZE, BLOSC_DOSHUFFLE, BLOSC_DOBITSHUFFLE, BLOSC_MEMCPYED - void blosc_init() - void blosc_destroy() - int blosc_get_nthreads() - int blosc_set_nthreads(int nthreads) - int blosc_set_compressor(const char *compname) - void blosc_set_blocksize(size_t blocksize) - char* blosc_list_compressors() - int blosc_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes, - void* src, void* dest, size_t destsize) nogil - int blosc_decompress(void *src, void *dest, size_t destsize) nogil - int blosc_getitem(void* src, int start, int nitems, void* dest) - int blosc_compress_ctx(int clevel, int doshuffle, size_t typesize, size_t nbytes, - const void* src, void* dest, size_t destsize, - const char* compressor, size_t blocksize, - int numinternalthreads) nogil - int blosc_decompress_ctx(const void* src, void* dest, size_t destsize, - int numinternalthreads) nogil - void blosc_cbuffer_sizes(const void* cbuffer, size_t* nbytes, size_t* cbytes, - size_t* blocksize) - char* blosc_cbuffer_complib(const void* cbuffer) - void blosc_cbuffer_metainfo(const void* cbuffer, size_t* typesize, int* flags) - - -MAX_OVERHEAD = BLOSC_MAX_OVERHEAD -MAX_BUFFERSIZE = BLOSC_MAX_BUFFERSIZE -MAX_THREADS = BLOSC_MAX_THREADS -MAX_TYPESIZE = BLOSC_MAX_TYPESIZE -VERSION_STRING = BLOSC_VERSION_STRING -VERSION_DATE = BLOSC_VERSION_DATE + const char* NC_BLOSC_LIBRARY + + void nc_blosc_setup() + void nc_blosc_init() + void nc_blosc_destroy() + int nc_blosc_get_nthreads() + int nc_blosc_set_nthreads(int nthreads) + int nc_blosc_set_compressor(const char *compname) + void nc_blosc_set_blocksize(size_t blocksize) + char* nc_blosc_list_compressors() + int nc_blosc_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes, + void* src, void* dest, size_t destsize) nogil + int nc_blosc_decompress(void *src, void *dest, size_t destsize) nogil + int nc_blosc_compress_ctx(int clevel, int doshuffle, size_t typesize, size_t nbytes, + const void* src, void* dest, size_t destsize, + const char* compressor, size_t blocksize, + int numinternalthreads) nogil + int nc_blosc_decompress_ctx(const void* src, size_t srcsize, void* dest, + size_t destsize, int numinternalthreads) nogil + int nc_blosc_cbuffer_validate(const void* cbuffer, size_t cbytes, size_t* nbytes) + void nc_blosc_cbuffer_sizes(const void* cbuffer, size_t* nbytes, size_t* cbytes, + size_t* blocksize) + const char* nc_blosc_cbuffer_complib(const void* cbuffer) + void nc_blosc_cbuffer_metainfo(const void* cbuffer, size_t* typesize, int* flags) + + +MAX_OVERHEAD = NC_BLOSC_MAX_OVERHEAD +MAX_BUFFERSIZE = NC_BLOSC_MAX_BUFFERSIZE +MAX_THREADS = NC_BLOSC_MAX_THREADS +MAX_TYPESIZE = NC_BLOSC_MAX_TYPESIZE +VERSION_STRING = NC_BLOSC_VERSION_STRING +VERSION_DATE = NC_BLOSC_VERSION_DATE +# name of the C library this module is built against: "c-blosc" or "c-blosc2" +LIBRARY = ( NC_BLOSC_LIBRARY).decode() VERSION_STRING = VERSION_STRING.decode() VERSION_DATE = VERSION_DATE.decode() __version__ = VERSION_STRING @@ -74,7 +81,8 @@ AUTOSHUFFLE = -1 AUTOBLOCKS = 0 # computed once at import; blosc_list_compressors() fills a static buffer without locking -_COMPRESSORS = tuple(blosc_list_compressors().decode('ascii').split(',')) +nc_blosc_setup() +_COMPRESSORS = tuple(nc_blosc_list_compressors().decode('ascii').split(',')) # serializes access to blosc's global context _MUTEX = threading.Lock() @@ -95,13 +103,13 @@ _importer_pid = os.getpid() def _init(): """Initialize the Blosc library environment.""" with _MUTEX: - blosc_init() + nc_blosc_init() def _destroy(): """Destroy the Blosc library environment.""" with _MUTEX: - blosc_destroy() + nc_blosc_destroy() def list_compressors(): @@ -113,7 +121,7 @@ def get_nthreads(): """Get the number of threads that Blosc uses internally for compression and decompression.""" with _MUTEX: - return blosc_get_nthreads() + return nc_blosc_get_nthreads() def set_nthreads(int nthreads): @@ -121,7 +129,7 @@ def set_nthreads(int nthreads): decompression.""" # blosc_set_nthreads re-creates the global context with _MUTEX: - return blosc_set_nthreads(nthreads) + return nc_blosc_set_nthreads(nthreads) def _cbuffer_sizes(source): @@ -146,7 +154,7 @@ def _cbuffer_sizes(source): source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - blosc_cbuffer_sizes(source_pb.buf, &nbytes, &cbytes, &blocksize) + nc_blosc_cbuffer_sizes(source_pb.buf, &nbytes, &cbytes, &blocksize) return nbytes, cbytes, blocksize @@ -162,7 +170,7 @@ def cbuffer_complib(source): source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - complib = blosc_cbuffer_complib(source_pb.buf) + complib = nc_blosc_cbuffer_complib(source_pb.buf) complib = complib.decode('ascii') @@ -192,7 +200,7 @@ def _cbuffer_metainfo(source): source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - blosc_cbuffer_metainfo(source_pb.buf, &typesize, &flags) + nc_blosc_cbuffer_metainfo(source_pb.buf, &typesize, &flags) # decompose flags if flags & BLOSC_DOSHUFFLE: @@ -267,6 +275,20 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, itemsize = typesize else: itemsize = source_pb.itemsize + if itemsize == 0: + # blosc does not accept a typesize of 0, which zero-size dtypes such as a + # structured dtype with a zero-length field give; the buffer is empty in + # that case, so any typesize works + itemsize = 1 + + # A requested blocksize must not exceed the buffer and must be a multiple of the + # typesize. c-blosc 1.x enforces this itself; c-blosc2 does not and then writes + # chunks it cannot read back. + if blocksize > 0: + if blocksize > nbytes and nbytes > 0: + blocksize = nbytes + if blocksize > itemsize: + blocksize -= blocksize % itemsize # determine shuffle if shuffle == AUTOSHUFFLE: @@ -281,7 +303,7 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, try: # setup destination - dest = PyBytes_FromStringAndSize(NULL, nbytes + BLOSC_MAX_OVERHEAD) + dest = PyBytes_FromStringAndSize(NULL, nbytes + NC_BLOSC_DEST_OVERHEAD) dest_ptr = PyBytes_AS_STRING(dest) # perform compression @@ -294,24 +316,24 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, with _MUTEX: # set compressor - compressor_set = blosc_set_compressor(cname) + compressor_set = nc_blosc_set_compressor(cname) if compressor_set < 0: # shouldn't happen if we checked against list of compressors # already, but just in case _err_bad_cname(cname_str) # set blocksize - blosc_set_blocksize(blocksize) + nc_blosc_set_blocksize(blocksize) # perform compression with nogil: - cbytes = blosc_compress(clevel, shuffle, itemsize, nbytes, source_ptr, - dest_ptr, nbytes + BLOSC_MAX_OVERHEAD) + cbytes = nc_blosc_compress(clevel, shuffle, itemsize, nbytes, source_ptr, + dest_ptr, nbytes + NC_BLOSC_DEST_OVERHEAD) else: with nogil: - cbytes = blosc_compress_ctx(clevel, shuffle, itemsize, nbytes, source_ptr, - dest_ptr, nbytes + BLOSC_MAX_OVERHEAD, + cbytes = nc_blosc_compress_ctx(clevel, shuffle, itemsize, nbytes, source_ptr, + dest_ptr, nbytes + NC_BLOSC_DEST_OVERHEAD, cname, blocksize, 1) finally: @@ -334,7 +356,8 @@ def decompress(source, dest=None): ---------- source : bytes-like Compressed data, including blosc header. Can be any object supporting the buffer - protocol. + protocol. Bytes after the first complete Blosc frame are ignored for backward + compatibility. dest : array-like, optional Object to decompress into. @@ -361,8 +384,16 @@ def decompress(source, dest=None): # get source pointer source_ptr = source_pb.buf - # determine buffer size - blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize) + # Read the declared frame size only after proving the complete header is present. + # Validate exactly that frame so trailing bytes remain backward compatible. + if source_pb.len < NC_BLOSC_MIN_HEADER_LENGTH: + raise RuntimeError('invalid blosc frame: buffer is too small') + nc_blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize) + if cbytes > source_pb.len: + raise RuntimeError('invalid blosc frame: buffer is truncated') + ret = nc_blosc_cbuffer_validate(source_ptr, cbytes, &nbytes) + if ret != 0: + raise RuntimeError('invalid blosc frame: header validation failed') # setup destination buffer if dest is None: @@ -384,21 +415,27 @@ def decompress(source, dest=None): raise ValueError('destination buffer too small; expected at least %s, ' 'got %s' % (nbytes, dest_nbytes)) + # Preserve the previous loud failure for a non-empty destination paired + # with an empty frame instead of returning an untouched output buffer. + if nbytes == 0 and dest_nbytes != 0: + raise RuntimeError('cannot decompress an empty blosc frame into a ' + 'non-empty destination buffer') + # perform decompression if _get_use_threads(): # allow blosc to use threads internally with _MUTEX: with nogil: - ret = blosc_decompress(source_ptr, dest_ptr, nbytes) + ret = nc_blosc_decompress(source_ptr, dest_ptr, nbytes) else: with nogil: - ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1) + ret = nc_blosc_decompress_ctx(source_ptr, cbytes, dest_ptr, nbytes, 1) finally: pass # handle errors - if ret <= 0: + if ret < 0 or ret != nbytes: raise RuntimeError('error during blosc decompression: %d' % ret) return dest diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index 41a3a7a2f..f71bd3d1b 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -38,172 +38,219 @@ if host_machine.cpu_family() == 'x86_64' endif endif -# --- Vendored zstd --- -zstd_dep = dependency('libzstd', required: get_option('system_zstd')) - -if not zstd_dep.found() - zstd_sources = files( - # common/ - '../../c-blosc/internal-complibs/zstd-1.5.6/common/debug.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/fse_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/pool.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/threading.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/xxhash.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/common/zstd_common.c', - # compress/ - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/fse_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/hist.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/huf_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_literals.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_sequences.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_superblock.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_double_fast.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_fast.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_lazy.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_ldm.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_opt.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstdmt_compress.c', - # decompress/ - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_ddict.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress_block.c', - # dictBuilder/ - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/cover.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/divsufsort.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/fastcover.c', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/zdict.c', - ) +use_blosc2 = get_option('blosc_backend') == 'blosc2' - # Assembly files — meson handles .S natively - if host_machine.cpu_family() == 'x86_64' - zstd_sources += files( - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S', - ) +if use_blosc2 + # --- Compression libraries from the system or from WrapDB subprojects --- + # Each library is resolved here, once, so that the standalone zstd/lz4 codecs and the + # c-blosc2 subproject share the same build. The subproject is configured first so + # that c-blosc2's own dependency() lookups find it. + sub_opts = ['default_library=static'] + + # zlib first: zstd looks it up as an optional dependency + if get_option('system_zlib').disabled() + subproject('zlib', default_options: sub_opts) + else + dependency('zlib', required: get_option('system_zlib')) endif - zstd_inc = include_directories( - '../../c-blosc/internal-complibs/zstd-1.5.6', - '../../c-blosc/internal-complibs/zstd-1.5.6/common', - '../../c-blosc/internal-complibs/zstd-1.5.6/compress', - '../../c-blosc/internal-complibs/zstd-1.5.6/decompress', - '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', - ) + if get_option('system_lz4').disabled() + lz4_dep = subproject('lz4', default_options: sub_opts).get_variable('liblz4_dep') + else + lz4_dep = dependency('liblz4', required: get_option('system_lz4')) + endif - zstd_lib = static_library('zstd_vendored', zstd_sources, - include_directories: zstd_inc, - c_args: c_args, - ) - zstd_dep = declare_dependency( - link_with: zstd_lib, - include_directories: zstd_inc, - ) -endif + if get_option('system_zstd').disabled() + zstd_dep = subproject('zstd', default_options: sub_opts).get_variable('libzstd_dep') + else + zstd_dep = dependency('libzstd', required: get_option('system_zstd')) + endif -# --- Vendored lz4 --- -lz4_dep = dependency('liblz4', required: get_option('system_lz4')) + blosc_sub_opts = [ + 'zstd=' + (get_option('blosc_zstd').disabled() ? 'disabled' : 'enabled'), + 'zlib=' + (get_option('blosc_zlib').disabled() ? 'disabled' : 'enabled'), + 'sse2=' + (have_sse2 ? 'enabled' : 'disabled'), + 'avx2=' + (have_avx2 ? 'enabled' : 'disabled'), + ] + if get_option('system_blosc').disabled() + blosc_dep = subproject( + 'c-blosc2', + default_options: sub_opts + blosc_sub_opts, + ).get_variable('blosc2_dep') + else + blosc_dep = dependency('blosc2', version: '>=3.0', required: get_option('system_blosc')) + endif + blosc_c_args = ['-DNUMCODECS_BLOSC2'] +else + # --- Vendored zstd --- + zstd_dep = dependency('libzstd', required: get_option('system_zstd')) -if not lz4_dep.found() - lz4_sources = files( - '../../c-blosc/internal-complibs/lz4-1.10.0/lz4.c', - '../../c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', - ) - lz4_inc = include_directories('../../c-blosc/internal-complibs/lz4-1.10.0') + if not zstd_dep.found() + zstd_sources = files( + # common/ + '../../c-blosc/internal-complibs/zstd-1.5.6/common/debug.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/fse_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/pool.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/threading.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/xxhash.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/zstd_common.c', + # compress/ + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/fse_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/hist.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/huf_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_literals.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_sequences.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_superblock.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_double_fast.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_fast.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_lazy.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_ldm.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_opt.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstdmt_compress.c', + # decompress/ + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_ddict.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress_block.c', + # dictBuilder/ + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/cover.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/divsufsort.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/fastcover.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/zdict.c', + ) - lz4_lib = static_library('lz4_vendored', lz4_sources, - include_directories: lz4_inc, - c_args: c_args, - ) - lz4_dep = declare_dependency( - link_with: lz4_lib, - include_directories: lz4_inc, - ) -endif + # Assembly files — meson handles .S natively + if host_machine.cpu_family() == 'x86_64' + zstd_sources += files( + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S', + ) + endif -# --- Vendored blosc --- -blosc_dep = dependency('blosc', required: get_option('system_blosc')) - -if not blosc_dep.found() - # --- Vendored zlib --- - zlib_dep = dependency('zlib', required: get_option('system_zlib')) - - if not zlib_dep.found() - zlib_sources = files( - '../../c-blosc/internal-complibs/zlib-1.3.1/adler32.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/compress.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/crc32.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/deflate.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzclose.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzlib.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzread.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/gzwrite.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/infback.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inffast.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inflate.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/trees.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', - '../../c-blosc/internal-complibs/zlib-1.3.1/zutil.c', + zstd_inc = include_directories( + '../../c-blosc/internal-complibs/zstd-1.5.6', + '../../c-blosc/internal-complibs/zstd-1.5.6/common', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', ) - zlib_inc = include_directories('../../c-blosc/internal-complibs/zlib-1.3.1') - zlib_lib = static_library('zlib_vendored', zlib_sources, - include_directories: zlib_inc, + zstd_lib = static_library('zstd_vendored', zstd_sources, + include_directories: zstd_inc, c_args: c_args, ) - zlib_dep = declare_dependency( - link_with: zlib_lib, - include_directories: zlib_inc, + zstd_dep = declare_dependency( + link_with: zstd_lib, + include_directories: zstd_inc, ) endif - blosc_sources = files( - '../../c-blosc/blosc/blosc.c', - '../../c-blosc/blosc/blosclz.c', - '../../c-blosc/blosc/fastcopy.c', - '../../c-blosc/blosc/shuffle.c', - '../../c-blosc/blosc/shuffle-generic.c', - '../../c-blosc/blosc/bitshuffle-generic.c', - ) + # --- Vendored lz4 --- + lz4_dep = dependency('liblz4', required: get_option('system_lz4')) - blosc_c_args = c_args + ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] + if not lz4_dep.found() + lz4_sources = files( + '../../c-blosc/internal-complibs/lz4-1.10.0/lz4.c', + '../../c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', + ) + lz4_inc = include_directories('../../c-blosc/internal-complibs/lz4-1.10.0') - if have_sse2 - blosc_sources += files( - '../../c-blosc/blosc/shuffle-sse2.c', - '../../c-blosc/blosc/bitshuffle-sse2.c', + lz4_lib = static_library('lz4_vendored', lz4_sources, + include_directories: lz4_inc, + c_args: c_args, + ) + lz4_dep = declare_dependency( + link_with: lz4_lib, + include_directories: lz4_inc, ) - blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] - if host_machine.system() == 'windows' - blosc_c_args += ['-D__SSE2__'] - endif endif - if have_avx2 - blosc_sources += files( - '../../c-blosc/blosc/shuffle-avx2.c', - '../../c-blosc/blosc/bitshuffle-avx2.c', + # --- Vendored blosc --- + blosc_dep = dependency('blosc', required: get_option('system_blosc')) + + if not blosc_dep.found() + # --- Vendored zlib --- + zlib_dep = dependency('zlib', required: get_option('system_zlib')) + + if not zlib_dep.found() + zlib_sources = files( + '../../c-blosc/internal-complibs/zlib-1.3.1/adler32.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/compress.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/crc32.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/deflate.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzclose.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzlib.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzread.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzwrite.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/infback.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inffast.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inflate.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/trees.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/zutil.c', + ) + zlib_inc = include_directories('../../c-blosc/internal-complibs/zlib-1.3.1') + + zlib_lib = static_library('zlib_vendored', zlib_sources, + include_directories: zlib_inc, + c_args: c_args, + ) + zlib_dep = declare_dependency( + link_with: zlib_lib, + include_directories: zlib_inc, + ) + endif + + blosc_sources = files( + '../../c-blosc/blosc/blosc.c', + '../../c-blosc/blosc/blosclz.c', + '../../c-blosc/blosc/fastcopy.c', + '../../c-blosc/blosc/shuffle.c', + '../../c-blosc/blosc/shuffle-generic.c', + '../../c-blosc/blosc/bitshuffle-generic.c', ) - blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] - if host_machine.system() == 'windows' - blosc_c_args += ['-D__AVX2__'] + + blosc_c_args = c_args + ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] + + if have_sse2 + blosc_sources += files( + '../../c-blosc/blosc/shuffle-sse2.c', + '../../c-blosc/blosc/bitshuffle-sse2.c', + ) + blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__SSE2__'] + endif endif - endif - blosc_inc = include_directories('../../c-blosc/blosc') + if have_avx2 + blosc_sources += files( + '../../c-blosc/blosc/shuffle-avx2.c', + '../../c-blosc/blosc/bitshuffle-avx2.c', + ) + blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__AVX2__'] + endif + endif - blosc_lib = static_library('blosc_vendored', blosc_sources, - include_directories: blosc_inc, - dependencies: [lz4_dep, zlib_dep, zstd_dep], - c_args: blosc_c_args, - ) - blosc_dep = declare_dependency( - link_with: blosc_lib, - include_directories: blosc_inc, - ) + blosc_inc = include_directories('../../c-blosc/blosc') + + blosc_lib = static_library('blosc_vendored', blosc_sources, + include_directories: blosc_inc, + dependencies: [lz4_dep, zlib_dep, zstd_dep], + c_args: blosc_c_args, + ) + blosc_dep = declare_dependency( + link_with: blosc_lib, + include_directories: blosc_inc, + ) + endif + + blosc_c_args = [] endif # --- Cython extension modules --- @@ -212,7 +259,8 @@ endif py.extension_module('blosc', 'blosc.pyx', dependencies: [blosc_dep], - c_args: c_args, + include_directories: include_directories('.'), + c_args: c_args + blosc_c_args, link_args: link_args, install: true, subdir: 'numcodecs', diff --git a/tests/test_blosc.py b/tests/test_blosc.py index b920f03e8..ac0d11caa 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -38,6 +38,10 @@ Blosc(shuffle=Blosc.SHUFFLE, blocksize=2**8), Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE, blocksize=2**8), ] +# compressors can be left out of a build (e.g. -Dblosc_zstd=disabled with c-blosc2) +codecs = [ + c if c is None or c.cname in blosc.list_compressors() else None for c in codecs +] # mix of dtypes: integer, float, bool, string @@ -57,6 +61,7 @@ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'), np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'), np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'), + np.empty(0, dtype='u1'), ] @@ -77,6 +82,35 @@ def test_encode_decode(array, codec): check_encode_decode(array, codec) +def test_empty_encode_decode(use_threads): + blosc.use_threads = use_threads + try: + check_encode_decode(np.empty(0, dtype='u1'), Blosc()) + finally: + blosc.use_threads = None + + +def test_empty_decode_rejects_nonempty_destination(): + codec = Blosc() + encoded = codec.encode(b'') + with pytest.raises(RuntimeError, match='non-empty destination buffer'): + codec.decode(encoded, out=bytearray(1)) + + +def test_decompress_allows_trailing_bytes(): + codec = Blosc() + original = b'some data to compress' + encoded = codec.encode(original) + assert codec.decode(encoded + b'padding') == original + + +def test_decompress_rejects_truncated_frame(): + codec = Blosc() + encoded = codec.encode(b'some data to compress') + with pytest.raises(RuntimeError, match='buffer is truncated'): + codec.decode(encoded[:-1]) + + def test_config(): codec = Blosc(cname='zstd', clevel=3, shuffle=1) check_config(codec) @@ -165,7 +199,11 @@ def test_compress_metainfo(dtype, use_threads): enc = blosc.compress(arr, cname.encode(), 1, shuffle) typesize, did_shuffle, _ = blosc._cbuffer_metainfo(enc) assert typesize == arr.dtype.itemsize - assert did_shuffle == shuffle + if typesize == 1 and shuffle == Blosc.SHUFFLE and blosc.LIBRARY == 'c-blosc2': + # c-blosc2 omits the byte shuffle flag when it is a no-op + assert did_shuffle in (Blosc.NOSHUFFLE, shuffle) + else: + assert did_shuffle == shuffle def test_compress_autoshuffle(use_threads): @@ -203,7 +241,8 @@ def test_backwards_compatibility(): def _encode_worker(data): - compressor = Blosc(cname='zlib', clevel=9, shuffle=Blosc.SHUFFLE) + cname = 'zlib' if 'zlib' in blosc.list_compressors() else 'lz4' + compressor = Blosc(cname=cname, clevel=9, shuffle=Blosc.SHUFFLE) return compressor.encode(data) @@ -326,7 +365,40 @@ def test_typesize_less_than_1(): compressor.encode(arr.tobytes()) +@pytest.mark.parametrize("shuffle", [Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE, Blosc.AUTOSHUFFLE]) +def test_zero_itemsize(shuffle): + # a structured dtype with a zero-length field has itemsize 0, which blosc + # rejects as a typesize (and older versions crashed on, #516) + arr = np.zeros((5,), dtype=[("foo", int, (0,))]) + assert arr.dtype.itemsize == 0 + enc = Blosc(shuffle=shuffle).encode(arr) + nbytes, _, _ = blosc._cbuffer_sizes(enc) + assert nbytes == 0 + + def test_config_no_typesize(): codec = Blosc(shuffle=Blosc.SHUFFLE, typesize=5) config = codec.get_config() assert "typesize" not in config + + +@pytest.mark.parametrize('shuffle', [Blosc.SHUFFLE, Blosc.BITSHUFFLE]) +@pytest.mark.parametrize('blocksize', [0, 2**8, 4097]) +@pytest.mark.parametrize('nbytes', [1, 31, 33, 127, 1000, 65537]) +@pytest.mark.parametrize('typesize', [1, 2, 3, 4, 7, 8, 12, 16, 17, 32]) +def test_shuffle_roundtrip_typesizes(typesize, nbytes, blocksize, shuffle): + # Exercises the SIMD (un)shuffle kernels, including the scalar tail used when a + # block is not a multiple of the vector width or the typesize. + src = np.random.default_rng(0).integers(0, 64, nbytes, dtype='u1').tobytes() + codec = Blosc(cname='lz4', clevel=5, shuffle=shuffle, blocksize=blocksize, typesize=typesize) + assert bytes(codec.decode(codec.encode(src))) == src + + +@pytest.mark.parametrize('cname', blosc.list_compressors()) +@pytest.mark.parametrize('shuffle', [Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE]) +@pytest.mark.parametrize('typesize', [1, 2, 3, 8]) +def test_output_is_blosc1_format(cname, shuffle, typesize): + # Chunks must stay readable by C-Blosc 1.x: format version 2 in the first byte + src = np.arange(4096, dtype='u1').tobytes() + enc = Blosc(cname=cname, shuffle=shuffle, typesize=typesize).encode(src) + assert enc[0] == 2 From df7666427d50c368368af4f7cb79da4940f91ef2 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 20 Oct 2026 16:59:38 -0400 Subject: [PATCH 03/10] Fix CI for the c-blosc2 backend The compatibility workflow assumed the wrapped zstd/lz4/zlib static libraries were always built; c-blosc2 uses a system copy when it finds one, so it now links whichever exists. Keep the pre-commit whitespace hook away from .patch files, whose context lines must keep their trailing space, and make the package's scripts pass ruff (including the executable bit for shebangs). Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/blosc2-compat.yaml | 11 +++++- .pre-commit-config.yaml | 2 +- .../c-blosc2-blosc1-compat/tests/c/cons.c | 2 +- .../c-blosc2-blosc1-compat/tests/c/cons2.c | 2 +- .../c-blosc2-blosc1-compat/tests/c/prod.c | 2 +- .../c-blosc2-blosc1-compat/tests/c/prod1.c | 2 +- packages/c-blosc2-blosc1-compat/tests/run.py | 39 ++++++++++++------- .../tools/make_wrap_diff_files.py | 6 ++- .../c-blosc2-blosc1-compat/tools/rebase.py | 18 +++++++-- .../c-blosc2-blosc1-compat/tools/sync_wrap.py | 7 +++- tests/test_blosc.py | 8 ++-- 11 files changed, 68 insertions(+), 31 deletions(-) mode change 100644 => 100755 packages/c-blosc2-blosc1-compat/tests/run.py mode change 100644 => 100755 packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py mode change 100644 => 100755 packages/c-blosc2-blosc1-compat/tools/rebase.py mode change 100644 => 100755 packages/c-blosc2-blosc1-compat/tools/sync_wrap.py diff --git a/.github/workflows/blosc2-compat.yaml b/.github/workflows/blosc2-compat.yaml index e9ea3c5ad..08e15f4d8 100644 --- a/.github/workflows/blosc2-compat.yaml +++ b/.github/workflows/blosc2-compat.yaml @@ -41,11 +41,18 @@ jobs: meson setup b && meson compile -C b - name: Run the compatibility check run: | - B=/tmp/probe/b/subprojects + B=/tmp/probe/b + # c-blosc2 uses a system zstd/lz4/zlib when it finds one, otherwise the wrap's + # static build; link whichever is there + LIBS="$(find $B/subprojects -name libblosc2.a | head -n 1)" + for l in zstd lz4 z; do + a="$(find $B/subprojects -name lib$l.a | head -n 1)" + LIBS="$LIBS ${a:--l$l}" + done python packages/c-blosc2-blosc1-compat/tests/run.py \ --blosc1-include /tmp/c-blosc/blosc --blosc1-lib /tmp/b1/blosc \ --blosc2-include "$(echo /tmp/probe/subprojects/c-blosc2-*/include)" \ - --blosc2-ldflags "$(echo $B/c-blosc2-*/libblosc2.a) $(echo $B/zstd-*/build/meson/lib/libzstd.a) $(echo $B/lz4-*/build/meson/meson/lib/liblz4.a) $(echo $B/zlib-*/libz.a)" + --blosc2-ldflags "$LIBS" rebase: # Informational: does the series still apply to upstream main? Fails only on the schedule. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 59c2c9835..d9a275a6b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: ^fixture/ # files with trailing whitespaces on purpose +exclude: ^fixture/|\.patch$ # trailing whitespace on purpose (fixtures, patch context lines) ci: autoupdate_commit_msg: "chore: update pre-commit hooks" autoupdate_schedule: "monthly" diff --git a/packages/c-blosc2-blosc1-compat/tests/c/cons.c b/packages/c-blosc2-blosc1-compat/tests/c/cons.c index 2a535d4c5..b7316bfef 100644 --- a/packages/c-blosc2-blosc1-compat/tests/c/cons.c +++ b/packages/c-blosc2-blosc1-compat/tests/c/cons.c @@ -7,7 +7,7 @@ int main(int argc, char **argv) { int ok = 0, bad = 0; for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { size_t nb = sizes[z]; int ts = typesizes[t]; - + char fn[256]; snprintf(fn, sizeof fn, "out/%s_%s_s%d_t%d_n%zu.bin", mode, cnames[c], shuffles[s], ts, nb); FILE *f = fopen(fn, "rb"); if (!f) { printf("missing %s\n", fn); continue; } uint8_t *cbuf = malloc(nb + 1024); size_t cb = fread(cbuf, 1, nb + 1024, f); fclose(f); diff --git a/packages/c-blosc2-blosc1-compat/tests/c/cons2.c b/packages/c-blosc2-blosc1-compat/tests/c/cons2.c index 21cb6d884..6318d0b7a 100644 --- a/packages/c-blosc2-blosc1-compat/tests/c/cons2.c +++ b/packages/c-blosc2-blosc1-compat/tests/c/cons2.c @@ -7,7 +7,7 @@ int main(int argc, char **argv) { int ok = 0, bad = 0; for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { size_t nb = sizes[z]; int ts = typesizes[t]; - + char fn[256]; snprintf(fn, sizeof fn, "out/%s_%s_s%d_t%d_n%zu.bin", mode, cnames[c], shuffles[s], ts, nb); FILE *f = fopen(fn, "rb"); if (!f) { printf("missing %s\n", fn); continue; } uint8_t *cbuf = malloc(nb + 1024); size_t cb = fread(cbuf, 1, nb + 1024, f); fclose(f); diff --git a/packages/c-blosc2-blosc1-compat/tests/c/prod.c b/packages/c-blosc2-blosc1-compat/tests/c/prod.c index ab494a1bd..621129b06 100644 --- a/packages/c-blosc2-blosc1-compat/tests/c/prod.c +++ b/packages/c-blosc2-blosc1-compat/tests/c/prod.c @@ -8,7 +8,7 @@ int main(int argc, char **argv) { int n = 0; for (int c = 0; c < 5; c++) for (int s = 0; s < 3; s++) for (int t = 0; t < 7; t++) for (int z = 0; z < 7; z++) { size_t nb = sizes[z]; int ts = typesizes[t]; - + uint8_t *src = malloc(nb), *dst = malloc(nb + BLOSC2_MAX_OVERHEAD); fill(src, nb, ts, 1234); int cb; diff --git a/packages/c-blosc2-blosc1-compat/tests/c/prod1.c b/packages/c-blosc2-blosc1-compat/tests/c/prod1.c index 0b78cd0d4..2bb853eb7 100644 --- a/packages/c-blosc2-blosc1-compat/tests/c/prod1.c +++ b/packages/c-blosc2-blosc1-compat/tests/c/prod1.c @@ -2,7 +2,7 @@ #include "common.h" int main(void){ blosc_init(); blosc_set_nthreads(1); for (int c=0;c<5;c++) for(int s=0;s<3;s++) for(int t=0;t<7;t++) for(int z=0;z<7;z++){ - size_t nb=sizes[z]; int ts=typesizes[t]; + size_t nb=sizes[z]; int ts=typesizes[t]; uint8_t *src=malloc(nb),*dst=malloc(nb+BLOSC_MAX_OVERHEAD); fill(src,nb,ts,1234); blosc_set_compressor(cnames[c]); int cb=blosc_compress(5,shuffles[s],ts,nb,src,dst,nb+BLOSC_MAX_OVERHEAD); if(cb<=0){fprintf(stderr,"fail\n");return 2;} diff --git a/packages/c-blosc2-blosc1-compat/tests/run.py b/packages/c-blosc2-blosc1-compat/tests/run.py old mode 100644 new mode 100755 index cd99919e8..eb4c4fbde --- a/packages/c-blosc2-blosc1-compat/tests/run.py +++ b/packages/c-blosc2-blosc1-compat/tests/run.py @@ -13,6 +13,7 @@ run.py --blosc1-include c-blosc/blosc --blosc1-lib build-b1/blosc \\ --blosc2-include c-blosc2/include --blosc2-ldflags "libblosc2.a libzstd.a liblz4.a -lz" """ + import argparse import os import shlex @@ -29,33 +30,43 @@ def sh(cmd, **kw): def main(): - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) ap.add_argument("--blosc1-include", required=True, help="directory with blosc.h (C-Blosc 1.x)") - ap.add_argument("--blosc1-lib", required=True, help="directory with the C-Blosc 1.x shared library") - ap.add_argument("--blosc2-include", required=True, help="directory with blosc2.h (patched c-blosc2)") - ap.add_argument("--blosc2-ldflags", required=True, help="libraries to link c-blosc2 and its codecs") + ap.add_argument( + "--blosc1-lib", required=True, help="directory with the C-Blosc 1.x shared library" + ) + ap.add_argument( + "--blosc2-include", required=True, help="directory with blosc2.h (patched c-blosc2)" + ) + ap.add_argument( + "--blosc2-ldflags", required=True, help="libraries to link c-blosc2 and its codecs" + ) ap.add_argument("--cc", default=os.environ.get("CC", "cc")) a = ap.parse_args() lib1 = Path(a.blosc1_lib).resolve() - ld1 = ["-L" + str(lib1), "-lblosc", "-Wl,-rpath," + str(lib1)] - ld2 = shlex.split(a.blosc2_ldflags) + ["-lpthread", "-lm"] - inc = lambda d: "-I" + str(Path(d).resolve()) # noqa: E731 + ld1 = [f"-L{lib1}", "-lblosc", f"-Wl,-rpath,{lib1}"] + ld2 = [*shlex.split(a.blosc2_ldflags), "-lpthread", "-lm"] + + def inc(d): + return "-I" + str(Path(d).resolve()) with tempfile.TemporaryDirectory() as tmp: tmp = Path(tmp) (tmp / "out").mkdir() - cc = [a.cc, "-O1", "-I" + str(C)] - sh(cc + [inc(a.blosc2_include), str(C / "prod.c"), "-o", "prod"] + ld2, cwd=tmp) - sh(cc + [inc(a.blosc1_include), str(C / "cons.c"), "-o", "cons"] + ld1, cwd=tmp) - sh(cc + [inc(a.blosc1_include), str(C / "prod1.c"), "-o", "prod1"] + ld1, cwd=tmp) - sh(cc + [inc(a.blosc2_include), str(C / "cons2.c"), "-o", "cons2"] + ld2, cwd=tmp) + cc = [a.cc, "-O1", f"-I{C}"] + sh([*cc, inc(a.blosc2_include), str(C / "prod.c"), "-o", "prod", *ld2], cwd=tmp) + sh([*cc, inc(a.blosc1_include), str(C / "cons.c"), "-o", "cons", *ld1], cwd=tmp) + sh([*cc, inc(a.blosc1_include), str(C / "prod1.c"), "-o", "prod1", *ld1], cwd=tmp) + sh([*cc, inc(a.blosc2_include), str(C / "cons2.c"), "-o", "cons2", *ld2], cwd=tmp) failed = False for mode in ("global", "ctx"): sh(["./prod", mode], cwd=tmp, stdout=subprocess.DEVNULL) - failed |= subprocess.run(["./cons", mode], cwd=tmp).returncode != 0 + failed |= subprocess.run(["./cons", mode], cwd=tmp, check=False).returncode != 0 sh(["./prod1"], cwd=tmp) - failed |= subprocess.run(["./cons2", "b1"], cwd=tmp).returncode != 0 + failed |= subprocess.run(["./cons2", "b1"], cwd=tmp, check=False).returncode != 0 return 1 if failed else 0 diff --git a/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py b/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py old mode 100644 new mode 100755 index d762a92fb..2403ff7c9 --- a/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py +++ b/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py @@ -1,6 +1,10 @@ #!/usr/bin/env python3 """Print the `diff_files` line for subprojects/c-blosc2.wrap from the patch series.""" + from pathlib import Path pkg = Path(__file__).resolve().parents[1] -print("diff_files = " + ", ".join(f"c-blosc2/patches/{p.name}" for p in sorted((pkg / "patches").glob("*.patch")))) +print( + "diff_files = " + + ", ".join(f"c-blosc2/patches/{p.name}" for p in sorted((pkg / "patches").glob("*.patch"))) +) diff --git a/packages/c-blosc2-blosc1-compat/tools/rebase.py b/packages/c-blosc2-blosc1-compat/tools/rebase.py old mode 100644 new mode 100755 index bc5104ada..496572f0b --- a/packages/c-blosc2-blosc1-compat/tools/rebase.py +++ b/packages/c-blosc2-blosc1-compat/tools/rebase.py @@ -3,6 +3,7 @@ Usage: rebase.py [REF] (default: the commit in UPSTREAM) """ + import subprocess import sys import tarfile @@ -14,9 +15,15 @@ def main(): - ref = sys.argv[1] if len(sys.argv) > 1 else dict( - line.split(" = ") for line in (PKG / "UPSTREAM").read_text().split("\n") if " = " in line - )["commit"] + ref = ( + sys.argv[1] + if len(sys.argv) > 1 + else dict( + line.split(" = ") + for line in (PKG / "UPSTREAM").read_text().split("\n") + if " = " in line + )["commit"] + ) url = f"https://github.com/Blosc/c-blosc2/archive/{ref}.tar.gz" with tempfile.TemporaryDirectory() as tmp: tar = Path(tmp) / "src.tar.gz" @@ -28,7 +35,10 @@ def main(): for patch in sorted((PKG / "patches").glob("*.patch")): r = subprocess.run( ["patch", "-p1", "--no-backup-if-mismatch", "-i", str(patch)], - cwd=src, capture_output=True, text=True, + cwd=src, + check=False, + capture_output=True, + text=True, ) print(f"{'ok ' if r.returncode == 0 else 'FAIL'} {patch.name}") if r.returncode: diff --git a/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py b/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py old mode 100644 new mode 100755 index 2c67d94d5..c09155be2 --- a/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py +++ b/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py @@ -5,6 +5,7 @@ so that directory holds copies of what lives here. Run without arguments to update the copies, or with --check (used in CI) to fail if they are out of date. """ + import argparse import filecmp import shutil @@ -28,7 +29,11 @@ def main(): check = ap.parse_args().check expected = {dst for _, dst in sources()} stale = [d for d in DEST.rglob("*") if d.is_file() and d not in expected] - bad = [dst for src, dst in sources() if not dst.exists() or not filecmp.cmp(src, dst, shallow=False)] + bad = [ + dst + for src, dst in sources() + if not dst.exists() or not filecmp.cmp(src, dst, shallow=False) + ] if check: for f in bad + stale: print(f"out of sync: {f}", file=sys.stderr) diff --git a/tests/test_blosc.py b/tests/test_blosc.py index ac0d11caa..088a490f1 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -39,9 +39,7 @@ Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE, blocksize=2**8), ] # compressors can be left out of a build (e.g. -Dblosc_zstd=disabled with c-blosc2) -codecs = [ - c if c is None or c.cname in blosc.list_compressors() else None for c in codecs -] +codecs = [c if c is None or c.cname in blosc.list_compressors() else None for c in codecs] # mix of dtypes: integer, float, bool, string @@ -365,7 +363,9 @@ def test_typesize_less_than_1(): compressor.encode(arr.tobytes()) -@pytest.mark.parametrize("shuffle", [Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE, Blosc.AUTOSHUFFLE]) +@pytest.mark.parametrize( + "shuffle", [Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE, Blosc.AUTOSHUFFLE] +) def test_zero_itemsize(shuffle): # a structured dtype with a zero-length field has itemsize 0, which blosc # rejects as a typesize (and older versions crashed on, #516) From b106205059b735d93509f4bcb8be6448bf89a81c Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 20 Oct 2026 17:15:45 -0400 Subject: [PATCH 04/10] Note the upstream c-blosc2 issue and PR for patch 0001 Co-Authored-By: Claude Sonnet 5.5 --- packages/c-blosc2-blosc1-compat/README.md | 3 ++- packages/c-blosc2-blosc1-compat/TODO.md | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 packages/c-blosc2-blosc1-compat/TODO.md diff --git a/packages/c-blosc2-blosc1-compat/README.md b/packages/c-blosc2-blosc1-compat/README.md index 311400476..8ee669cce 100644 --- a/packages/c-blosc2-blosc1-compat/README.md +++ b/packages/c-blosc2-blosc1-compat/README.md @@ -19,6 +19,7 @@ deleted if upstream accepts the patches. | `meson/` | `meson.build` and `meson.options` for c-blosc2 (upstream only has CMake) | | `tests/` | the forward-compatibility check, see below | | `tools/` | `sync_wrap.py`, `rebase.py` | +| `TODO.md` | upstream items to check | ## The patches @@ -67,5 +68,5 @@ that `UPSTREAM` pins: | Patch | Upstream | |---|---| -| 0001 | not yet submitted | +| 0001 | [issue #812](https://github.com/Blosc/c-blosc2/issues/812), [PR #813](https://github.com/Blosc/c-blosc2/pull/813) (see `TODO.md`) | | 0002–0004 | not yet submitted | diff --git a/packages/c-blosc2-blosc1-compat/TODO.md b/packages/c-blosc2-blosc1-compat/TODO.md new file mode 100644 index 000000000..bb3bbd54d --- /dev/null +++ b/packages/c-blosc2-blosc1-compat/TODO.md @@ -0,0 +1,20 @@ +# TODO + +## Check upstream c-blosc2 status of patch 0001 + +Patch `0001-reader-bitunshuffle-v2-leftovers.patch` is submitted upstream: + +- issue: https://github.com/Blosc/c-blosc2/issues/812 +- pull request: https://github.com/Blosc/c-blosc2/pull/813 + +Check periodically whether #813 has been merged and which c-blosc2 release contains it. +When `UPSTREAM` pins a release that includes it: + +1. delete `patches/0001-reader-bitunshuffle-v2-leftovers.patch`; +2. run `python tools/sync_wrap.py` and update `diff_files` in `subprojects/c-blosc2.wrap` + (`python tools/make_wrap_diff_files.py` prints the line); +3. update the upstream table in `README.md`. + +If upstream asks for changes or rejects it, update the patch and this note to match. + +Patches `0002`-`0004` (the opt-in Blosc1 writer) have not been submitted upstream. From 1e918eefda7ea972377ec143f1f337ffd5300d1f Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 20 Oct 2026 17:16:36 -0400 Subject: [PATCH 05/10] Run the test suite on the c-blosc2 backend in CI Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/blosc2-compat.yaml | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/blosc2-compat.yaml b/.github/workflows/blosc2-compat.yaml index 08e15f4d8..fa3b123a8 100644 --- a/.github/workflows/blosc2-compat.yaml +++ b/.github/workflows/blosc2-compat.yaml @@ -54,6 +54,36 @@ jobs: --blosc2-include "$(echo /tmp/probe/subprojects/c-blosc2-*/include)" \ --blosc2-ldflags "$LIBS" + test-blosc2-backend: + # the numcodecs test suite with Blosc built on c-blosc2 instead of c-blosc 1.x + runs-on: ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + - {platform: ubuntu-latest, python-version: "3.13"} + - {platform: ubuntu-latest, python-version: "3.14t"} + - {platform: ubuntu-22.04-arm, python-version: "3.13"} + - {platform: macos-14, python-version: "3.13"} + - {platform: windows-latest, python-version: "3.13"} + defaults: + run: + shell: bash -e {0} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 # required for version resolution + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install numcodecs with the c-blosc2 backend + run: python -m pip install -v ".[test,test_extras]" -Csetup-args=-Dblosc_backend=blosc2 + - name: Check the backend + run: python -c "import numcodecs.blosc as b; print(b.LIBRARY, b.VERSION_STRING, b.list_compressors()); assert b.LIBRARY == 'c-blosc2'" + - name: Run tests + run: pytest -v + rebase: # Informational: does the series still apply to upstream main? Fails only on the schedule. if: github.event_name == 'schedule' From 814803c26c8573b6a36bc85ef6804a6f8f859199 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 20 Oct 2026 17:18:23 -0400 Subject: [PATCH 06/10] Check out the submodule in the c-blosc2 backend test job pyproject.toml's license-files still lists the c-blosc submodule's licenses, so the build fails without it even with the c-blosc2 backend. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/blosc2-compat.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/blosc2-compat.yaml b/.github/workflows/blosc2-compat.yaml index fa3b123a8..5bd464846 100644 --- a/.github/workflows/blosc2-compat.yaml +++ b/.github/workflows/blosc2-compat.yaml @@ -72,6 +72,8 @@ jobs: steps: - uses: actions/checkout@v6 with: + # pyproject.toml's license-files still lists the c-blosc submodule's licenses + submodules: recursive fetch-depth: 0 # required for version resolution - uses: actions/setup-python@v6 with: From c603e686bd85b6154d0917410f5e49659b75b08d Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 20 Oct 2026 17:21:47 -0400 Subject: [PATCH 07/10] Add tests for Blosc frame validation, blocksize handling and the c-blosc2 package Cover the decompress errors for a short buffer, an invalid header and a truncated frame, blocksizes that are larger than the buffer or not a multiple of the typesize, and the library attributes. A new test module checks that the wrap copies match packages/c-blosc2-blosc1-compat, that diff_files lists the patch series, that the wrap matches UPSTREAM, and that sync_wrap.py detects and fixes a stale copy. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_blosc.py | 45 +++++++++++++++++++ tests/test_blosc2_package.py | 85 ++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 tests/test_blosc2_package.py diff --git a/tests/test_blosc.py b/tests/test_blosc.py index 088a490f1..181477864 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -1,3 +1,4 @@ +import struct import threading import time from multiprocessing import Pool @@ -402,3 +403,47 @@ def test_output_is_blosc1_format(cname, shuffle, typesize): src = np.arange(4096, dtype='u1').tobytes() enc = Blosc(cname=cname, shuffle=shuffle, typesize=typesize).encode(src) assert enc[0] == 2 + + +def _blosc1_header(nbytes, cbytes, typesize=1, blocksize=1): + # version 2, lz4, no flags, then nbytes, blocksize, cbytes (little endian) + return struct.pack(' Date: Thu, 22 Oct 2026 03:12:06 -0400 Subject: [PATCH 08/10] Build c-blosc2 from the Blosc1-format branch instead of patching it Point the c-blosc2 wrap at the blosc1-format-writer branch of mkitti/c-blosc2 (Blosc/c-blosc2#815), pinned to its head commit, instead of a release tarball plus a four-patch series. The reader fix (patch 0001) is already upstream and the branch contains the writer (patches 0002-0004, reworked and with tests). - subprojects/c-blosc2.wrap is now a wrap-git; the patches, make_wrap_diff_files.py and rebase.py are removed, sync_wrap.py only copies the meson overlay - UPSTREAM records the branch and commit; the test checks the wrap agrees - drop the scheduled rebase job from the workflow and the .patch pre-commit exclude - README.md, TODO.md and the release notes describe the branch and what to do once #815 is released (the wrap must not stay on a personal fork for a release) Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/blosc2-compat.yaml | 17 +--- .pre-commit-config.yaml | 2 +- docs/release.md | 2 +- packages/c-blosc2-blosc1-compat/README.md | 82 ++++++++-------- packages/c-blosc2-blosc1-compat/TODO.md | 29 +++--- packages/c-blosc2-blosc1-compat/UPSTREAM | 7 +- ...001-reader-bitunshuffle-v2-leftovers.patch | 21 ---- .../patches/0002-blosc1-format-flag.patch | 95 ------------------- .../0003-blosc1-format-bitshuffle.patch | 24 ----- .../0004-blosc1-format-split-rule.patch | 33 ------- packages/c-blosc2-blosc1-compat/tests/run.py | 4 +- .../tools/make_wrap_diff_files.py | 10 -- .../c-blosc2-blosc1-compat/tools/rebase.py | 52 ---------- .../c-blosc2-blosc1-compat/tools/sync_wrap.py | 6 +- subprojects/c-blosc2.wrap | 11 +-- ...001-reader-bitunshuffle-v2-leftovers.patch | 21 ---- .../patches/0002-blosc1-format-flag.patch | 95 ------------------- .../0003-blosc1-format-bitshuffle.patch | 24 ----- .../0004-blosc1-format-split-rule.patch | 33 ------- tests/test_blosc2_package.py | 21 +--- 20 files changed, 75 insertions(+), 514 deletions(-) delete mode 100644 packages/c-blosc2-blosc1-compat/patches/0001-reader-bitunshuffle-v2-leftovers.patch delete mode 100644 packages/c-blosc2-blosc1-compat/patches/0002-blosc1-format-flag.patch delete mode 100644 packages/c-blosc2-blosc1-compat/patches/0003-blosc1-format-bitshuffle.patch delete mode 100644 packages/c-blosc2-blosc1-compat/patches/0004-blosc1-format-split-rule.patch delete mode 100755 packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py delete mode 100755 packages/c-blosc2-blosc1-compat/tools/rebase.py delete mode 100644 subprojects/packagefiles/c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch delete mode 100644 subprojects/packagefiles/c-blosc2/patches/0002-blosc1-format-flag.patch delete mode 100644 subprojects/packagefiles/c-blosc2/patches/0003-blosc1-format-bitshuffle.patch delete mode 100644 subprojects/packagefiles/c-blosc2/patches/0004-blosc1-format-split-rule.patch diff --git a/.github/workflows/blosc2-compat.yaml b/.github/workflows/blosc2-compat.yaml index 5bd464846..582411f8a 100644 --- a/.github/workflows/blosc2-compat.yaml +++ b/.github/workflows/blosc2-compat.yaml @@ -6,8 +6,6 @@ on: paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h'] pull_request: paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h'] - schedule: - - cron: '0 6 * * 1' # weekly: does the series still apply to upstream c-blosc2 main? workflow_dispatch: jobs: @@ -32,7 +30,7 @@ jobs: cmake -S /tmp/c-blosc -B /tmp/b1 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release cmake --build /tmp/b1 -j - - name: Build the patched c-blosc2 through the wrap + - name: Build c-blosc2 through the wrap run: | mkdir /tmp/probe && cd /tmp/probe ln -s "$GITHUB_WORKSPACE/subprojects" subprojects @@ -51,7 +49,7 @@ jobs: done python packages/c-blosc2-blosc1-compat/tests/run.py \ --blosc1-include /tmp/c-blosc/blosc --blosc1-lib /tmp/b1/blosc \ - --blosc2-include "$(echo /tmp/probe/subprojects/c-blosc2-*/include)" \ + --blosc2-include "/tmp/probe/subprojects/c-blosc2/include" \ --blosc2-ldflags "$LIBS" test-blosc2-backend: @@ -85,14 +83,3 @@ jobs: run: python -c "import numcodecs.blosc as b; print(b.LIBRARY, b.VERSION_STRING, b.list_compressors()); assert b.LIBRARY == 'c-blosc2'" - name: Run tests run: pytest -v - - rebase: - # Informational: does the series still apply to upstream main? Fails only on the schedule. - if: github.event_name == 'schedule' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.13' - - run: python packages/c-blosc2-blosc1-compat/tools/rebase.py main diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d9a275a6b..59c2c9835 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: ^fixture/|\.patch$ # trailing whitespace on purpose (fixtures, patch context lines) +exclude: ^fixture/ # files with trailing whitespaces on purpose ci: autoupdate_commit_msg: "chore: update pre-commit hooks" autoupdate_schedule: "monthly" diff --git a/docs/release.md b/docs/release.md index b9012d370..2dc299048 100644 --- a/docs/release.md +++ b/docs/release.md @@ -22,7 +22,7 @@ * Add an optional c-blosc2 backend for the `Blosc` codec, selected at build time with `-Dblosc_backend=blosc2` (default is still c-blosc 1.x). c-blosc2 is built from a meson - wrap (`subprojects/c-blosc2.wrap`) with a small patch so that it writes Blosc1-format + wrap (`subprojects/c-blosc2.wrap`) of a c-blosc2 branch that can write Blosc1-format chunks that C-Blosc 1.x can decode (stock c-blosc2 output is not readable by C-Blosc 1.x). zstd, lz4 and zlib are resolved separately from Blosc (system libraries or WrapDB wraps, `-Dsystem_*`) and can be left out of Blosc with `-Dblosc_zstd=disabled` and diff --git a/packages/c-blosc2-blosc1-compat/README.md b/packages/c-blosc2-blosc1-compat/README.md index 8ee669cce..19bac2f2e 100644 --- a/packages/c-blosc2-blosc1-compat/README.md +++ b/packages/c-blosc2-blosc1-compat/README.md @@ -1,72 +1,68 @@ # c-blosc2-blosc1-compat -An overlay for [c-blosc2](https://github.com/Blosc/c-blosc2) that makes it usable as the -backend of numcodecs' `Blosc` codec: a meson build, plus a small patch series that lets -c-blosc2 **write chunks that C-Blosc 1.x can decode**. Stock c-blosc2 cannot, even through -its `blosc1_*` API or with `BLOSC_BLOSC1_COMPAT` (its README says so, and +A build overlay for [c-blosc2](https://github.com/Blosc/c-blosc2) that makes it usable as the +backend of numcodecs' `Blosc` codec: a meson build (upstream only has CMake) for a c-blosc2 +branch that can **write chunks that C-Blosc 1.x can decode**. Stock c-blosc2 cannot, even +through its `blosc1_*` API or with `BLOSC_BLOSC1_COMPAT` (its README says so, and `tests/run.py` confirms it). This directory is self-contained (it does not import anything from numcodecs) so it can be -split out with `git subtree split` if it ever needs to live in its own repository, or be -deleted if upstream accepts the patches. +split out with `git subtree split` if it ever needs to live in its own repository. ## Contents | Path | What | |---|---| -| `UPSTREAM` | the pinned c-blosc2 commit and source hash; `subprojects/c-blosc2.wrap` must match | -| `patches/` | the patch series, applied in order; one concern per patch | -| `meson/` | `meson.build` and `meson.options` for c-blosc2 (upstream only has CMake) | +| `UPSTREAM` | the c-blosc2 branch and commit in use; `subprojects/c-blosc2.wrap` must match | +| `meson/` | `meson.build` and `meson.options` for c-blosc2 | | `tests/` | the forward-compatibility check, see below | -| `tools/` | `sync_wrap.py`, `rebase.py` | -| `TODO.md` | upstream items to check | +| `tools/` | `sync_wrap.py` | +| `TODO.md` | what to do once the c-blosc2 work is released | -## The patches +## The c-blosc2 branch -Format version 2 (Blosc1) differs from what c-blosc2 writes in three ways that matter to a -reader, plus one reader bug: +`subprojects/c-blosc2.wrap` clones the `blosc1-format-writer` branch of +[mkitti/c-blosc2](https://github.com/mkitti/c-blosc2), pinned to a commit. The branch is +upstream c-blosc2 plus the change proposed in +[Blosc/c-blosc2#815](https://github.com/Blosc/c-blosc2/pull/815): +`blosc2_ctx_set_blosc1_format()`, which makes a compression context write format version 2 +(Blosc1) chunks. Format version 2 differs from what c-blosc2 writes in three ways that +matter to a reader: -1. `0001` reader: bytes left over after a bit-shuffled version-2 block were not copied. - A plain bug fix, independent of the rest. -2. `0002` an opt-in flag: write the 16-byte Blosc1 header with format version 2. Set with - `blosc2_ctx_set_blosc1_format()` or, for the global context, `BLOSC_BLOSC1_COMPAT`. - Off by default, no ABI change. -3. `0003` bitshuffle: version 2 shuffles a block only if its element count is a multiple - of 8, and stores it unshuffled otherwise. -4. `0004` block splitting: use the C-Blosc 1.x rule, which its reader recomputes. +1. the format version byte and the 16-byte header; +2. bitshuffle: version 2 shuffles a block only if its element count is a multiple of 8, and + stores it unshuffled otherwise; +3. block splitting: a C-Blosc 1.x reader recomputes it from the typesize and blocksize. Only plain shuffle/bitshuffle with blosclz, lz4, lz4hc, zlib and zstd are supported in this mode. The output decodes with C-Blosc 1.x but is not byte-identical to what it writes. +The reader fix for bit-shuffled version 2 blocks with leftover bytes +([#812](https://github.com/Blosc/c-blosc2/issues/812), +[#813](https://github.com/Blosc/c-blosc2/pull/813)) is already in upstream c-blosc2 and so +in the branch. + ## Wiring into the build -Meson only looks for `patch_directory` and `diff_files` in `subprojects/packagefiles`, so -`subprojects/packagefiles/c-blosc2/` holds copies of `meson/` and `patches/`. After editing -here run `python tools/sync_wrap.py`; CI runs it with `--check`. If a patch is added or -removed, update `diff_files` in `subprojects/c-blosc2.wrap` -(`tools/make_wrap_diff_files.py` prints the line). +Meson only looks for `patch_directory` in `subprojects/packagefiles`, so +`subprojects/packagefiles/c-blosc2/` holds a copy of `meson/`. After editing here run +`python tools/sync_wrap.py`; CI runs it with `--check`. ## Testing -`tests/run.py` builds small C programs against C-Blosc 1.x and the patched c-blosc2 and -checks 735 cases in three directions (see its docstring for the command line): -c-blosc2 global-context to C-Blosc 1.x, c-blosc2 context API to C-Blosc 1.x, and C-Blosc 1.x -to c-blosc2. All must decode. +`tests/run.py` builds small C programs against C-Blosc 1.x and c-blosc2 and checks 735 +cases in three directions (see its docstring for the command line): c-blosc2 global-context +to C-Blosc 1.x, c-blosc2 context API to C-Blosc 1.x, and C-Blosc 1.x to c-blosc2. All must +decode. The c-blosc2 branch has its own unit tests (`tests/test_blosc1_format.c`). ## Updating c-blosc2 -`python tools/rebase.py ` downloads that c-blosc2, tries to apply the -series, and reports which patch stops applying. To bump: edit `UPSTREAM` and -`subprojects/c-blosc2.wrap` (directory, URL, hash), fix any patch that fails, re-export it -as a unified diff against the pristine tree (`diff -ruN a b`), run `tools/sync_wrap.py` -and the tests. The scheduled workflow `blosc2-compat-rebase` does the first step weekly. +Edit `UPSTREAM` and the `revision` (and `url`, if the branch moves) in +`subprojects/c-blosc2.wrap` together; a test checks that they agree. Then run +`python tools/sync_wrap.py` and the tests. If upstream changes its source file list, update +`meson/meson.build`. ## Upstreaming -Track each patch here when a PR exists, and delete the patch once it is in a release -that `UPSTREAM` pins: - -| Patch | Upstream | -|---|---| -| 0001 | [issue #812](https://github.com/Blosc/c-blosc2/issues/812), [PR #813](https://github.com/Blosc/c-blosc2/pull/813) (see `TODO.md`) | -| 0002–0004 | not yet submitted | +The wrap depends on a personal fork until Blosc/c-blosc2#815 is merged and released; see +`TODO.md`. diff --git a/packages/c-blosc2-blosc1-compat/TODO.md b/packages/c-blosc2-blosc1-compat/TODO.md index bb3bbd54d..80cf696d7 100644 --- a/packages/c-blosc2-blosc1-compat/TODO.md +++ b/packages/c-blosc2-blosc1-compat/TODO.md @@ -1,20 +1,23 @@ # TODO -## Check upstream c-blosc2 status of patch 0001 +## Move the wrap off the fork once Blosc/c-blosc2#815 is released -Patch `0001-reader-bitunshuffle-v2-leftovers.patch` is submitted upstream: +`subprojects/c-blosc2.wrap` clones the `blosc1-format-writer` branch of +https://github.com/mkitti/c-blosc2 (see `UPSTREAM`). The change is proposed upstream in +https://github.com/Blosc/c-blosc2/pull/815 (it builds on #812 / #813, already merged). -- issue: https://github.com/Blosc/c-blosc2/issues/812 -- pull request: https://github.com/Blosc/c-blosc2/pull/813 +Check periodically whether #815 has been merged and which c-blosc2 release contains it. +When there is such a release: -Check periodically whether #813 has been merged and which c-blosc2 release contains it. -When `UPSTREAM` pins a release that includes it: +1. point the wrap at it: `url = https://github.com/Blosc/c-blosc2.git`, `revision = ` (or switch back to a `[wrap-file]` with `source_url` and `source_hash`); +2. update `UPSTREAM` to match, and check that `meson/meson.build` still lists the right + source files; +3. run `python tools/sync_wrap.py` and `tests/run.py`; +4. update `docs/release.md` and `README.md`. -1. delete `patches/0001-reader-bitunshuffle-v2-leftovers.patch`; -2. run `python tools/sync_wrap.py` and update `diff_files` in `subprojects/c-blosc2.wrap` - (`python tools/make_wrap_diff_files.py` prints the line); -3. update the upstream table in `README.md`. +If upstream asks for changes (for example to the `BLOSC_BLOSC1_COMPAT` behavior) or rejects +it, update the branch, `UPSTREAM` and the wrap's `revision` to match. -If upstream asks for changes or rejects it, update the patch and this note to match. - -Patches `0002`-`0004` (the opt-in Blosc1 writer) have not been submitted upstream. +Do not release numcodecs with the wrap pointing at the fork: a personal branch is not a +stable build dependency. diff --git a/packages/c-blosc2-blosc1-compat/UPSTREAM b/packages/c-blosc2-blosc1-compat/UPSTREAM index 98bf86eba..effa94cf0 100644 --- a/packages/c-blosc2-blosc1-compat/UPSTREAM +++ b/packages/c-blosc2-blosc1-compat/UPSTREAM @@ -1,4 +1,5 @@ -repository = https://github.com/Blosc/c-blosc2 -commit = 38258631914de7f6ca2cf55c9675d771796ab96c +repository = https://github.com/mkitti/c-blosc2 +branch = blosc1-format-writer +commit = a4995c38d1250790ce7f7e4a6debcda41404fb4b version = 3.3.6.dev -sha256 = c5556a00d8fc6a6a7be17e8f54e5c2aae2d304e740ac1aeb245db0f5731950a6 +pull_request = https://github.com/Blosc/c-blosc2/pull/815 diff --git a/packages/c-blosc2-blosc1-compat/patches/0001-reader-bitunshuffle-v2-leftovers.patch b/packages/c-blosc2-blosc1-compat/patches/0001-reader-bitunshuffle-v2-leftovers.patch deleted file mode 100644 index 04671582b..000000000 --- a/packages/c-blosc2-blosc1-compat/patches/0001-reader-bitunshuffle-v2-leftovers.patch +++ /dev/null @@ -1,21 +0,0 @@ -From: numcodecs contributors -Subject: [PATCH] Copy the leftover bytes when bit-unshuffling a version 2 block - -A Blosc1 (format version 2) block whose element count is a multiple of 8 -stores the bytes that do not fill a whole element after the shuffled part. -The reader skipped them, so chunks written by C-Blosc 1.x with a typesize -that does not divide the block size were decoded with garbage at the end. ---- -diff -ruN a/blosc/shuffle.c b/blosc/shuffle.c ---- a/blosc/shuffle.c -+++ b/blosc/shuffle.c -@@ -498,6 +498,9 @@ - BLOSC_TRACE_ERROR("the impossible happened: the bitunshuffle filter failed!"); - return ret; - } -+ /* Copy the leftovers, as Blosc1 does */ -+ size_t offset = size * typesize; -+ memcpy((uint8_t *) dest + offset, (const uint8_t *) src + offset, blocksize - offset); - } - else { - memcpy(dest, src, blocksize); diff --git a/packages/c-blosc2-blosc1-compat/patches/0002-blosc1-format-flag.patch b/packages/c-blosc2-blosc1-compat/patches/0002-blosc1-format-flag.patch deleted file mode 100644 index 8ca87d0b7..000000000 --- a/packages/c-blosc2-blosc1-compat/patches/0002-blosc1-format-flag.patch +++ /dev/null @@ -1,95 +0,0 @@ -From: numcodecs contributors -Subject: [PATCH] Add an opt-in switch to write Blosc1 (format version 2) chunks - -A compression context gets a blosc1_format flag. When set, the chunk header is -the 16-byte Blosc1 header with format version 2. The flag is set with the new -blosc2_ctx_set_blosc1_format(), and for the global context by the existing -BLOSC_BLOSC1_COMPAT environment variable. Off by default; no change to the -default Blosc2 output. The filter and block-split changes that the version-2 -format also requires follow in the next patches. ---- -diff -ruN a/blosc/context.h b/blosc/context.h ---- a/blosc/context.h -+++ b/blosc/context.h -@@ -40,6 +40,7 @@ - uint8_t header_flags; /* Flags for header */ - uint8_t blosc2_flags; /* Flags specific for blosc2 */ - uint8_t blosc2_flags2; /* Secondary flags specific for blosc2 */ -+ bool blosc1_format; /* Write Blosc1 (format version 2) chunks, readable by C-Blosc 1.x */ - int32_t sourcesize; /* Number of bytes in source buffer */ - int32_t header_overhead; /* The number of bytes in chunk header */ - int32_t nblocks; /* Number of total blocks in buffer */ -diff -ruN a/include/blosc2.h b/include/blosc2.h ---- a/include/blosc2.h -+++ b/include/blosc2.h -@@ -2719,6 +2719,20 @@ - */ - BLOSC_EXPORT void blosc1_set_splitmode(int splitmode); - -+/** -+ * @brief Make a compression context write Blosc1 chunks. -+ * -+ * The chunks use format version 2 and the 16-byte header, so that they can also be -+ * decoded by C-Blosc 1.x. Only plain shuffle/bitshuffle filters and the blosclz, -+ * lz4, lz4hc, zlib and zstd codecs can be used. Contexts are not Blosc1 by default. -+ * -+ * @param context A compression context. -+ * @param enable Whether to write Blosc1 chunks. -+ * -+ * @return 0 on success, a negative error code otherwise. -+ */ -+BLOSC_EXPORT int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable); -+ - - /** - * @brief Get the offsets of a frame in a super-chunk. -diff -ruN a/blosc/blosc2.c b/blosc/blosc2.c ---- a/blosc/blosc2.c -+++ b/blosc/blosc2.c -@@ -1008,6 +1008,9 @@ - header->version = (context->blosc2_flags2 & BLOSC2_VL_BLOCKS) ? - BLOSC2_VERSION_FORMAT_VL_BLOCKS : - BLOSC2_VERSION_FORMAT_STABLE; -+ if (context->blosc1_format) { -+ header->version = BLOSC1_VERSION_FORMAT; -+ } - header->versionlz = compcode_to_compversion(context->compcode); - header->flags = context->header_flags; - header->typesize = (uint8_t)context->typesize; -@@ -3153,6 +3161,15 @@ - } - - -+int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable) { -+ if (context == NULL || context->do_compress != 1) { -+ return BLOSC2_ERROR_INVALID_PARAM; -+ } -+ context->blosc1_format = enable; -+ return BLOSC2_ERROR_SUCCESS; -+} -+ -+ - /* The public secure routine for compression with context. */ - int blosc2_compress_ctx(blosc2_context* context, const void* src, int32_t srcsize, - void* dest, int32_t destsize) { -@@ -3173,8 +3190,8 @@ - return error; - } - -- /* Write the extended header */ -- error = write_compression_header(context, true); -+ /* Write the extended header, or the Blosc1 header if requested */ -+ error = write_compression_header(context, !context->blosc1_format); - if (error < 0) { - return error; - } -@@ -3896,6 +3913,9 @@ - } - - blosc2_pthread_mutex_lock(&global_comp_mutex); -+ -+ /* Blosc1 compatibility mode is read before anything depends on it (e.g. block splitting) */ -+ g_global_context->blosc1_format = (getenv("BLOSC_BLOSC1_COMPAT") != NULL); - - /* Initialize a context compression */ - uint8_t* filters = calloc(1, BLOSC2_MAX_FILTERS); diff --git a/packages/c-blosc2-blosc1-compat/patches/0003-blosc1-format-bitshuffle.patch b/packages/c-blosc2-blosc1-compat/patches/0003-blosc1-format-bitshuffle.patch deleted file mode 100644 index 9146d0bc0..000000000 --- a/packages/c-blosc2-blosc1-compat/patches/0003-blosc1-format-bitshuffle.patch +++ /dev/null @@ -1,24 +0,0 @@ -From: numcodecs contributors -Subject: [PATCH] Follow the version 2 bitshuffle rule when writing Blosc1 chunks - -Format version 2 bit-shuffles a block only when its number of elements is a -multiple of 8, and stores it unshuffled otherwise. Blosc2 shuffles the largest -multiple-of-8 prefix instead, which C-Blosc 1.x decodes to wrong data. ---- -diff -ruN a/blosc/blosc2.c b/blosc/blosc2.c ---- a/blosc/blosc2.c -+++ b/blosc/blosc2.c -@@ -1119,7 +1122,12 @@ - blosc2_shuffle(filters_meta[i] == 0 ? typesize : filters_meta[i], bsize, _src, _dest); - break; - case BLOSC_BITSHUFFLE: -- if (blosc2_bitshuffle(typesize, bsize, _src, _dest) < 0) { -+ if (context->blosc1_format && ((bsize / typesize) % 8) != 0) { -+ // Format version 2 (Blosc1) only bit-shuffles blocks whose number of -+ // elements is a multiple of 8; other blocks are stored unshuffled. -+ memcpy(_dest, _src, (size_t)bsize); -+ } -+ else if (blosc2_bitshuffle(typesize, bsize, _src, _dest) < 0) { - return NULL; - } - break; diff --git a/packages/c-blosc2-blosc1-compat/patches/0004-blosc1-format-split-rule.patch b/packages/c-blosc2-blosc1-compat/patches/0004-blosc1-format-split-rule.patch deleted file mode 100644 index 09c4bcac5..000000000 --- a/packages/c-blosc2-blosc1-compat/patches/0004-blosc1-format-split-rule.patch +++ /dev/null @@ -1,33 +0,0 @@ -From: numcodecs contributors -Subject: [PATCH] Use the C-Blosc 1.x block split rule when writing Blosc1 chunks - -A C-Blosc 1.x reader recomputes whether a block is split from the typesize and -blocksize (typesize <= 16 and at least 128 elements); Blosc2 uses a smaller -minimum, so it could write split blocks the reader treats as unsplit. ---- -diff -ruN a/blosc/stune.c b/blosc/stune.c ---- a/blosc/stune.c -+++ b/blosc/stune.c -@@ -184,6 +184,22 @@ - } - - int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) { -+ if (context->blosc1_format) { -+ // C-Blosc 1.x readers infer splitting from these sizes (and the "dont split" flag), -+ // so a split is only possible if this condition holds. -+ int blosc1_splittable = (typesize <= MAX_STREAMS) && (blocksize / typesize) >= 128; -+ switch (context->splitmode) { -+ case BLOSC_ALWAYS_SPLIT: -+ return blosc1_splittable; -+ case BLOSC_NEVER_SPLIT: -+ return 0; -+ case BLOSC_AUTO_SPLIT: -+ return blosc1_splittable && context->compcode == BLOSC_BLOSCLZ; -+ default: -+ // The zstd support was introduced at the same time than the split flag -+ return blosc1_splittable && context->compcode != BLOSC_ZSTD; -+ } -+ } - switch (context->splitmode) { - case BLOSC_ALWAYS_SPLIT: - return 1; diff --git a/packages/c-blosc2-blosc1-compat/tests/run.py b/packages/c-blosc2-blosc1-compat/tests/run.py index eb4c4fbde..204a735d4 100755 --- a/packages/c-blosc2-blosc1-compat/tests/run.py +++ b/packages/c-blosc2-blosc1-compat/tests/run.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Check that chunks written by the patched c-blosc2 can be read by C-Blosc 1.x. +"""Check that chunks written by c-blosc2 in Blosc1 format can be read by C-Blosc 1.x. Compiles the programs in tests/c against both libraries and runs, over every compressor in {blosclz, lz4, lz4hc, zlib, zstd} x shuffle {none, byte, bit} x typesize @@ -38,7 +38,7 @@ def main(): "--blosc1-lib", required=True, help="directory with the C-Blosc 1.x shared library" ) ap.add_argument( - "--blosc2-include", required=True, help="directory with blosc2.h (patched c-blosc2)" + "--blosc2-include", required=True, help="directory with blosc2.h (c-blosc2 with Blosc1 format support)" ) ap.add_argument( "--blosc2-ldflags", required=True, help="libraries to link c-blosc2 and its codecs" diff --git a/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py b/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py deleted file mode 100755 index 2403ff7c9..000000000 --- a/packages/c-blosc2-blosc1-compat/tools/make_wrap_diff_files.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 -"""Print the `diff_files` line for subprojects/c-blosc2.wrap from the patch series.""" - -from pathlib import Path - -pkg = Path(__file__).resolve().parents[1] -print( - "diff_files = " - + ", ".join(f"c-blosc2/patches/{p.name}" for p in sorted((pkg / "patches").glob("*.patch"))) -) diff --git a/packages/c-blosc2-blosc1-compat/tools/rebase.py b/packages/c-blosc2-blosc1-compat/tools/rebase.py deleted file mode 100755 index 496572f0b..000000000 --- a/packages/c-blosc2-blosc1-compat/tools/rebase.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -"""Check that the patch series applies to a c-blosc2 commit or tag. - -Usage: rebase.py [REF] (default: the commit in UPSTREAM) -""" - -import subprocess -import sys -import tarfile -import tempfile -import urllib.request -from pathlib import Path - -PKG = Path(__file__).resolve().parents[1] - - -def main(): - ref = ( - sys.argv[1] - if len(sys.argv) > 1 - else dict( - line.split(" = ") - for line in (PKG / "UPSTREAM").read_text().split("\n") - if " = " in line - )["commit"] - ) - url = f"https://github.com/Blosc/c-blosc2/archive/{ref}.tar.gz" - with tempfile.TemporaryDirectory() as tmp: - tar = Path(tmp) / "src.tar.gz" - urllib.request.urlretrieve(url, tar) - with tarfile.open(tar) as t: - t.extractall(tmp, **({"filter": "data"} if hasattr(tarfile, "data_filter") else {})) - (src,) = [d for d in Path(tmp).iterdir() if d.is_dir()] - status = 0 - for patch in sorted((PKG / "patches").glob("*.patch")): - r = subprocess.run( - ["patch", "-p1", "--no-backup-if-mismatch", "-i", str(patch)], - cwd=src, - check=False, - capture_output=True, - text=True, - ) - print(f"{'ok ' if r.returncode == 0 else 'FAIL'} {patch.name}") - if r.returncode: - print(r.stdout + r.stderr) - status = 1 - break - return status - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py b/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py index c09155be2..f604d3335 100755 --- a/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py +++ b/packages/c-blosc2-blosc1-compat/tools/sync_wrap.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -"""Copy this package's build overlay and patches into subprojects/packagefiles/c-blosc2. +"""Copy this package's build overlay into subprojects/packagefiles/c-blosc2. -Meson wraps only look for `patch_directory` and `diff_files` in subprojects/packagefiles, +Meson wraps only look for `patch_directory` in subprojects/packagefiles, so that directory holds copies of what lives here. Run without arguments to update the copies, or with --check (used in CI) to fail if they are out of date. """ @@ -19,8 +19,6 @@ def sources(): for f in sorted((PKG / "meson").iterdir()): yield f, DEST / f.name - for f in sorted((PKG / "patches").glob("*.patch")): - yield f, DEST / "patches" / f.name def main(): diff --git a/subprojects/c-blosc2.wrap b/subprojects/c-blosc2.wrap index 6654e73fe..e98c50ce9 100644 --- a/subprojects/c-blosc2.wrap +++ b/subprojects/c-blosc2.wrap @@ -1,10 +1,9 @@ -[wrap-file] -directory = c-blosc2-38258631914de7f6ca2cf55c9675d771796ab96c -source_url = https://github.com/Blosc/c-blosc2/archive/38258631914de7f6ca2cf55c9675d771796ab96c.tar.gz -source_filename = c-blosc2-38258631914de7f6ca2cf55c9675d771796ab96c.tar.gz -source_hash = c5556a00d8fc6a6a7be17e8f54e5c2aae2d304e740ac1aeb245db0f5731950a6 +[wrap-git] +directory = c-blosc2 +url = https://github.com/mkitti/c-blosc2.git +# Branch blosc1-format-writer of the fork (Blosc/c-blosc2#815); pinned to its head commit. +revision = a4995c38d1250790ce7f7e4a6debcda41404fb4b patch_directory = c-blosc2 -diff_files = c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch, c-blosc2/patches/0002-blosc1-format-flag.patch, c-blosc2/patches/0003-blosc1-format-bitshuffle.patch, c-blosc2/patches/0004-blosc1-format-split-rule.patch [provide] dependency_names = blosc2 diff --git a/subprojects/packagefiles/c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch b/subprojects/packagefiles/c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch deleted file mode 100644 index 04671582b..000000000 --- a/subprojects/packagefiles/c-blosc2/patches/0001-reader-bitunshuffle-v2-leftovers.patch +++ /dev/null @@ -1,21 +0,0 @@ -From: numcodecs contributors -Subject: [PATCH] Copy the leftover bytes when bit-unshuffling a version 2 block - -A Blosc1 (format version 2) block whose element count is a multiple of 8 -stores the bytes that do not fill a whole element after the shuffled part. -The reader skipped them, so chunks written by C-Blosc 1.x with a typesize -that does not divide the block size were decoded with garbage at the end. ---- -diff -ruN a/blosc/shuffle.c b/blosc/shuffle.c ---- a/blosc/shuffle.c -+++ b/blosc/shuffle.c -@@ -498,6 +498,9 @@ - BLOSC_TRACE_ERROR("the impossible happened: the bitunshuffle filter failed!"); - return ret; - } -+ /* Copy the leftovers, as Blosc1 does */ -+ size_t offset = size * typesize; -+ memcpy((uint8_t *) dest + offset, (const uint8_t *) src + offset, blocksize - offset); - } - else { - memcpy(dest, src, blocksize); diff --git a/subprojects/packagefiles/c-blosc2/patches/0002-blosc1-format-flag.patch b/subprojects/packagefiles/c-blosc2/patches/0002-blosc1-format-flag.patch deleted file mode 100644 index 8ca87d0b7..000000000 --- a/subprojects/packagefiles/c-blosc2/patches/0002-blosc1-format-flag.patch +++ /dev/null @@ -1,95 +0,0 @@ -From: numcodecs contributors -Subject: [PATCH] Add an opt-in switch to write Blosc1 (format version 2) chunks - -A compression context gets a blosc1_format flag. When set, the chunk header is -the 16-byte Blosc1 header with format version 2. The flag is set with the new -blosc2_ctx_set_blosc1_format(), and for the global context by the existing -BLOSC_BLOSC1_COMPAT environment variable. Off by default; no change to the -default Blosc2 output. The filter and block-split changes that the version-2 -format also requires follow in the next patches. ---- -diff -ruN a/blosc/context.h b/blosc/context.h ---- a/blosc/context.h -+++ b/blosc/context.h -@@ -40,6 +40,7 @@ - uint8_t header_flags; /* Flags for header */ - uint8_t blosc2_flags; /* Flags specific for blosc2 */ - uint8_t blosc2_flags2; /* Secondary flags specific for blosc2 */ -+ bool blosc1_format; /* Write Blosc1 (format version 2) chunks, readable by C-Blosc 1.x */ - int32_t sourcesize; /* Number of bytes in source buffer */ - int32_t header_overhead; /* The number of bytes in chunk header */ - int32_t nblocks; /* Number of total blocks in buffer */ -diff -ruN a/include/blosc2.h b/include/blosc2.h ---- a/include/blosc2.h -+++ b/include/blosc2.h -@@ -2719,6 +2719,20 @@ - */ - BLOSC_EXPORT void blosc1_set_splitmode(int splitmode); - -+/** -+ * @brief Make a compression context write Blosc1 chunks. -+ * -+ * The chunks use format version 2 and the 16-byte header, so that they can also be -+ * decoded by C-Blosc 1.x. Only plain shuffle/bitshuffle filters and the blosclz, -+ * lz4, lz4hc, zlib and zstd codecs can be used. Contexts are not Blosc1 by default. -+ * -+ * @param context A compression context. -+ * @param enable Whether to write Blosc1 chunks. -+ * -+ * @return 0 on success, a negative error code otherwise. -+ */ -+BLOSC_EXPORT int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable); -+ - - /** - * @brief Get the offsets of a frame in a super-chunk. -diff -ruN a/blosc/blosc2.c b/blosc/blosc2.c ---- a/blosc/blosc2.c -+++ b/blosc/blosc2.c -@@ -1008,6 +1008,9 @@ - header->version = (context->blosc2_flags2 & BLOSC2_VL_BLOCKS) ? - BLOSC2_VERSION_FORMAT_VL_BLOCKS : - BLOSC2_VERSION_FORMAT_STABLE; -+ if (context->blosc1_format) { -+ header->version = BLOSC1_VERSION_FORMAT; -+ } - header->versionlz = compcode_to_compversion(context->compcode); - header->flags = context->header_flags; - header->typesize = (uint8_t)context->typesize; -@@ -3153,6 +3161,15 @@ - } - - -+int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable) { -+ if (context == NULL || context->do_compress != 1) { -+ return BLOSC2_ERROR_INVALID_PARAM; -+ } -+ context->blosc1_format = enable; -+ return BLOSC2_ERROR_SUCCESS; -+} -+ -+ - /* The public secure routine for compression with context. */ - int blosc2_compress_ctx(blosc2_context* context, const void* src, int32_t srcsize, - void* dest, int32_t destsize) { -@@ -3173,8 +3190,8 @@ - return error; - } - -- /* Write the extended header */ -- error = write_compression_header(context, true); -+ /* Write the extended header, or the Blosc1 header if requested */ -+ error = write_compression_header(context, !context->blosc1_format); - if (error < 0) { - return error; - } -@@ -3896,6 +3913,9 @@ - } - - blosc2_pthread_mutex_lock(&global_comp_mutex); -+ -+ /* Blosc1 compatibility mode is read before anything depends on it (e.g. block splitting) */ -+ g_global_context->blosc1_format = (getenv("BLOSC_BLOSC1_COMPAT") != NULL); - - /* Initialize a context compression */ - uint8_t* filters = calloc(1, BLOSC2_MAX_FILTERS); diff --git a/subprojects/packagefiles/c-blosc2/patches/0003-blosc1-format-bitshuffle.patch b/subprojects/packagefiles/c-blosc2/patches/0003-blosc1-format-bitshuffle.patch deleted file mode 100644 index 9146d0bc0..000000000 --- a/subprojects/packagefiles/c-blosc2/patches/0003-blosc1-format-bitshuffle.patch +++ /dev/null @@ -1,24 +0,0 @@ -From: numcodecs contributors -Subject: [PATCH] Follow the version 2 bitshuffle rule when writing Blosc1 chunks - -Format version 2 bit-shuffles a block only when its number of elements is a -multiple of 8, and stores it unshuffled otherwise. Blosc2 shuffles the largest -multiple-of-8 prefix instead, which C-Blosc 1.x decodes to wrong data. ---- -diff -ruN a/blosc/blosc2.c b/blosc/blosc2.c ---- a/blosc/blosc2.c -+++ b/blosc/blosc2.c -@@ -1119,7 +1122,12 @@ - blosc2_shuffle(filters_meta[i] == 0 ? typesize : filters_meta[i], bsize, _src, _dest); - break; - case BLOSC_BITSHUFFLE: -- if (blosc2_bitshuffle(typesize, bsize, _src, _dest) < 0) { -+ if (context->blosc1_format && ((bsize / typesize) % 8) != 0) { -+ // Format version 2 (Blosc1) only bit-shuffles blocks whose number of -+ // elements is a multiple of 8; other blocks are stored unshuffled. -+ memcpy(_dest, _src, (size_t)bsize); -+ } -+ else if (blosc2_bitshuffle(typesize, bsize, _src, _dest) < 0) { - return NULL; - } - break; diff --git a/subprojects/packagefiles/c-blosc2/patches/0004-blosc1-format-split-rule.patch b/subprojects/packagefiles/c-blosc2/patches/0004-blosc1-format-split-rule.patch deleted file mode 100644 index 09c4bcac5..000000000 --- a/subprojects/packagefiles/c-blosc2/patches/0004-blosc1-format-split-rule.patch +++ /dev/null @@ -1,33 +0,0 @@ -From: numcodecs contributors -Subject: [PATCH] Use the C-Blosc 1.x block split rule when writing Blosc1 chunks - -A C-Blosc 1.x reader recomputes whether a block is split from the typesize and -blocksize (typesize <= 16 and at least 128 elements); Blosc2 uses a smaller -minimum, so it could write split blocks the reader treats as unsplit. ---- -diff -ruN a/blosc/stune.c b/blosc/stune.c ---- a/blosc/stune.c -+++ b/blosc/stune.c -@@ -184,6 +184,22 @@ - } - - int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) { -+ if (context->blosc1_format) { -+ // C-Blosc 1.x readers infer splitting from these sizes (and the "dont split" flag), -+ // so a split is only possible if this condition holds. -+ int blosc1_splittable = (typesize <= MAX_STREAMS) && (blocksize / typesize) >= 128; -+ switch (context->splitmode) { -+ case BLOSC_ALWAYS_SPLIT: -+ return blosc1_splittable; -+ case BLOSC_NEVER_SPLIT: -+ return 0; -+ case BLOSC_AUTO_SPLIT: -+ return blosc1_splittable && context->compcode == BLOSC_BLOSCLZ; -+ default: -+ // The zstd support was introduced at the same time than the split flag -+ return blosc1_splittable && context->compcode != BLOSC_ZSTD; -+ } -+ } - switch (context->splitmode) { - case BLOSC_ALWAYS_SPLIT: - return 1; diff --git a/tests/test_blosc2_package.py b/tests/test_blosc2_package.py index ae99dc863..c30816b6f 100644 --- a/tests/test_blosc2_package.py +++ b/tests/test_blosc2_package.py @@ -1,7 +1,6 @@ """Checks on packages/c-blosc2-blosc1-compat, which only exist in a source checkout.""" import importlib.util -import re import subprocess import sys from pathlib import Path @@ -36,18 +35,6 @@ def test_wrap_copies_are_in_sync(): assert result.returncode == 0, result.stderr -def test_wrap_diff_files_match_patch_series(): - expected = _run("make_wrap_diff_files.py").stdout.strip() - assert expected in WRAP.read_text() - - -def test_patches_are_numbered_in_order(): - names = sorted(p.name for p in (PACKAGE / "patches").glob("*.patch")) - assert names - numbers = [int(re.match(r"(\d{4})-", n).group(1)) for n in names] - assert numbers == list(range(1, len(names) + 1)) - - def test_wrap_matches_upstream_pin(): pin = dict( line.split(" = ") @@ -55,17 +42,15 @@ def test_wrap_matches_upstream_pin(): if " = " in line ) wrap = WRAP.read_text() - assert pin["commit"] in wrap - assert pin["sha256"] in wrap + assert f"url = {pin['repository']}.git" in wrap + assert f"revision = {pin['commit']}" in wrap def test_sync_wrap_detects_a_stale_copy(tmp_path, monkeypatch): sync_wrap = _load("sync_wrap") package = tmp_path / "packages" / "pkg" (package / "meson").mkdir(parents=True) - (package / "patches").mkdir() (package / "meson" / "meson.build").write_text("project('x')\n") - (package / "patches" / "0001-a.patch").write_text("a\n") dest = tmp_path / "subprojects" / "packagefiles" / "c-blosc2" monkeypatch.setattr(sync_wrap, "PKG", package) monkeypatch.setattr(sync_wrap, "DEST", dest) @@ -77,7 +62,7 @@ def test_sync_wrap_detects_a_stale_copy(tmp_path, monkeypatch): monkeypatch.setattr(sys, "argv", ["sync_wrap.py", "--check"]) assert sync_wrap.main() == 0 - (dest / "patches" / "0001-a.patch").write_text("changed\n") + (dest / "meson.build").write_text("changed\n") (dest / "stale.txt").write_text("left over\n") assert sync_wrap.main() == 1 monkeypatch.setattr(sys, "argv", ["sync_wrap.py"]) From e014c8ff14f295b11a885e2fb7b4c45b1f7902c0 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Thu, 22 Oct 2026 03:38:17 -0400 Subject: [PATCH 09/10] Add property-based tests and CI for Blosc1 compatibility Review feedback on #879 asked for very comprehensive property-based tests that the range of payloads seen in zarr survives the swap to c-blosc2. The compressed bytes cannot be identical (the libraries pick different default block sizes), so check the thing that matters: chunks written by numcodecs decode with C-Blosc 1.x to the original bytes, and the reverse. python-blosc, which bundles C-Blosc 1.x, is the reference. tests/test_blosc1_compat.py has hypothesis properties over compressors, levels, shuffles, typesizes 1-300, buffer sizes at the edges that matter (32 bytes, multiples of 8 and 128 elements, leftover bytes), block sizes and data kinds, plus an exhaustive grid. Both numcodecs code paths are covered, the global context and numcodecs' own context (use_threads=False, other threads, forks), which write Blosc1 chunks differently on the c-blosc2 backend: switching the context path off fails 1506 of the tests, which is how a first version that missed it was found. .github/workflows/blosc1-compat.yaml runs it on the c-blosc2 backend on Linux (x86_64, arm64), macOS and Windows, and on the default c-blosc 1.x backend as a control. It is self-contained and can be disabled without touching anything else: set the repository variable BLOSC1_COMPAT_CI to 'false', delete the file, or deselect the blosc1_compat marker. The test module skips itself when python-blosc or hypothesis is missing, so the other workflows are unchanged. Also format packages/c-blosc2-blosc1-compat/tests/run.py with ruff, which pre-commit.ci flagged. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/blosc1-compat.yaml | 97 +++++++ packages/c-blosc2-blosc1-compat/README.md | 25 ++ packages/c-blosc2-blosc1-compat/tests/run.py | 4 +- pyproject.toml | 3 + tests/test_blosc1_compat.py | 257 +++++++++++++++++++ 5 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/blosc1-compat.yaml create mode 100644 tests/test_blosc1_compat.py diff --git a/.github/workflows/blosc1-compat.yaml b/.github/workflows/blosc1-compat.yaml new file mode 100644 index 000000000..e348e23a9 --- /dev/null +++ b/.github/workflows/blosc1-compat.yaml @@ -0,0 +1,97 @@ +name: Blosc1 compatibility + +# Property-based and exhaustive tests (tests/test_blosc1_compat.py) that chunks written by +# numcodecs' Blosc decode with C-Blosc 1.x, and the reverse, using python-blosc (which +# bundles C-Blosc 1.x) as the reference. +# +# This workflow is self-contained and optional. To turn it off, any of: +# * set the repository variable BLOSC1_COMPAT_CI to 'false' +# (Settings > Secrets and variables > Actions > Variables), without a code change; +# * delete this file; nothing else refers to it. tests/test_blosc1_compat.py skips itself +# when python-blosc or hypothesis is missing, so the other workflows are unaffected; +# * run pytest elsewhere with -m "not blosc1_compat". + +on: + push: + branches: [main] + paths: + - '.github/workflows/blosc1-compat.yaml' + - 'tests/test_blosc1_compat.py' + - 'src/numcodecs/blosc.pyx' + - 'src/numcodecs/_blosc_shim.h' + - 'src/numcodecs/meson.build' + - 'subprojects/**' + - 'packages/c-blosc2-blosc1-compat/**' + - 'c-blosc' + pull_request: + paths: + - '.github/workflows/blosc1-compat.yaml' + - 'tests/test_blosc1_compat.py' + - 'src/numcodecs/blosc.pyx' + - 'src/numcodecs/_blosc_shim.h' + - 'src/numcodecs/meson.build' + - 'subprojects/**' + - 'packages/c-blosc2-blosc1-compat/**' + - 'c-blosc' + schedule: + - cron: '0 5 * * 1' # weekly, with many more examples than a pull request + workflow_dispatch: + inputs: + examples: + description: 'Hypothesis examples per property' + default: '1000' + +permissions: + contents: read + +jobs: + compat: + # the repository variable is the off switch + if: vars.BLOSC1_COMPAT_CI != 'false' + name: ${{ matrix.backend }} backend, ${{ matrix.platform }} + runs-on: ${{ matrix.platform }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + # the c-blosc 1.x backend is the control: the tests must pass on it too, which + # shows that they test the format and not an implementation detail + - {backend: blosc1, platform: ubuntu-latest} + - {backend: blosc2, platform: ubuntu-latest} + - {backend: blosc2, platform: ubuntu-22.04-arm} + - {backend: blosc2, platform: macos-14} + - {backend: blosc2, platform: windows-latest} + defaults: + run: + shell: bash -e {0} + env: + BLOSC1_COMPAT_EXAMPLES: ${{ inputs.examples || (github.event_name == 'schedule' && '5000') || '300' }} + steps: + - uses: actions/checkout@v6 + with: + # pyproject.toml's license-files lists the c-blosc submodule's licenses, and the + # blosc1 backend builds it + submodules: recursive + fetch-depth: 0 # required for version resolution + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + cache: 'pip' + - name: Install numcodecs with the ${{ matrix.backend }} backend + run: python -m pip install -v ".[test]" -Csetup-args=-Dblosc_backend=${{ matrix.backend }} + - name: Install the C-Blosc 1.x reference + # python-blosc has wheels for most platforms and builds from source (with cmake) + # for the rest + run: python -m pip install blosc hypothesis cmake + - name: Versions + run: | + python - <<'EOF' + import blosc, hypothesis, numcodecs.blosc as nb + print("numcodecs backend:", nb.LIBRARY, nb.VERSION_STRING, nb.list_compressors()) + print("reference:", "python-blosc", blosc.__version__, "C-Blosc", blosc.VERSION_STRING) + print("hypothesis:", hypothesis.__version__) + assert nb.LIBRARY == ("c-blosc2" if "${{ matrix.backend }}" == "blosc2" else "c-blosc") + EOF + - name: Run the compatibility tests + run: pytest tests/test_blosc1_compat.py -m blosc1_compat --no-cov -v -p no:cacheprovider diff --git a/packages/c-blosc2-blosc1-compat/README.md b/packages/c-blosc2-blosc1-compat/README.md index 19bac2f2e..4e99adadf 100644 --- a/packages/c-blosc2-blosc1-compat/README.md +++ b/packages/c-blosc2-blosc1-compat/README.md @@ -55,6 +55,31 @@ cases in three directions (see its docstring for the command line): c-blosc2 glo to C-Blosc 1.x, c-blosc2 context API to C-Blosc 1.x, and C-Blosc 1.x to c-blosc2. All must decode. The c-blosc2 branch has its own unit tests (`tests/test_blosc1_format.c`). +## Compatibility CI + +`tests/test_blosc1_compat.py` (repository `tests/`, not this directory) checks that chunks +written by numcodecs decode with C-Blosc 1.x and the reverse, with python-blosc, which +bundles C-Blosc 1.x, as the reference. It combines property-based tests (hypothesis) over +compressors, levels, shuffles, typesizes up to 300, buffer sizes around the edges that +matter (32 bytes, multiples of 8 and 128 elements, bytes left over after the last item), +block sizes and data kinds, with an exhaustive grid. Both numcodecs code paths are checked: +the one through blosc's global context and the one through a context of its own. The +compressed bytes are not expected to equal what C-Blosc 1.x writes (the libraries pick +different default block sizes); the decoded bytes are. + +`.github/workflows/blosc1-compat.yaml` runs it on the c-blosc2 backend on Linux (x86_64 and +arm64), macOS and Windows, and on the default c-blosc 1.x backend as a control. It runs 300 +examples per property on a pull request and 5000 weekly. It is independent of everything +else and can be turned off by + +- setting the repository variable `BLOSC1_COMPAT_CI` to `false` (no code change), +- deleting the workflow file (the test module skips itself when python-blosc or hypothesis + is not installed, so no other workflow changes), or +- `pytest -m "not blosc1_compat"` anywhere else. + +To run it locally: `pip install blosc hypothesis`, then +`pytest tests/test_blosc1_compat.py` (`BLOSC1_COMPAT_EXAMPLES` sets the examples). + ## Updating c-blosc2 Edit `UPSTREAM` and the `revision` (and `url`, if the branch moves) in diff --git a/packages/c-blosc2-blosc1-compat/tests/run.py b/packages/c-blosc2-blosc1-compat/tests/run.py index 204a735d4..219d587d4 100755 --- a/packages/c-blosc2-blosc1-compat/tests/run.py +++ b/packages/c-blosc2-blosc1-compat/tests/run.py @@ -38,7 +38,9 @@ def main(): "--blosc1-lib", required=True, help="directory with the C-Blosc 1.x shared library" ) ap.add_argument( - "--blosc2-include", required=True, help="directory with blosc2.h (c-blosc2 with Blosc1 format support)" + "--blosc2-include", + required=True, + help="directory with blosc2.h (c-blosc2 with Blosc1 format support)", ) ap.add_argument( "--blosc2-ldflags", required=True, help="libraries to link c-blosc2 and its codecs" diff --git a/pyproject.toml b/pyproject.toml index 453abf583..9d0428624 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,6 +148,9 @@ norecursedirs = [ ] log_cli_level = "INFO" xfail_strict = true +markers = [ + "blosc1_compat: compatibility of Blosc with C-Blosc 1.x (needs python-blosc and hypothesis; see .github/workflows/blosc1-compat.yaml)", +] filterwarnings = [ "error", "ignore:As the c extension couldn't be imported:RuntimeWarning", # Ignore warning about pure python google_crc32c (on Python 3.14) diff --git a/tests/test_blosc1_compat.py b/tests/test_blosc1_compat.py new file mode 100644 index 000000000..5bfa3679d --- /dev/null +++ b/tests/test_blosc1_compat.py @@ -0,0 +1,257 @@ +"""Compatibility of numcodecs' Blosc with C-Blosc 1.x, using python-blosc as the reference. + +python-blosc (the ``blosc`` package) bundles C-Blosc 1.x, so it stands in for every other +reader and writer of the Blosc1 format, among them the Blosc codec of zarr v3. Whatever +backend ``numcodecs.blosc`` is built with, these must hold: + +* chunks written by numcodecs decode with C-Blosc 1.x to the original bytes (forward); +* chunks written by C-Blosc 1.x decode with numcodecs to the original bytes (backward); +* numcodecs round-trips its own chunks, and its chunks have a sound Blosc1 header. + +The compressed bytes are *not* expected to be identical to what C-Blosc 1.x writes (the +libraries choose different default block sizes, for one); the decoded bytes are. + +This module is optional. It is skipped unless ``blosc`` and ``hypothesis`` are installed, +and it is only run by .github/workflows/blosc1-compat.yaml. To leave it out anywhere else +run ``pytest -m "not blosc1_compat"``. The number of generated examples can be set with +the BLOSC1_COMPAT_EXAMPLES environment variable. +""" + +import contextlib +import itertools +import os + +import numpy as np +import pytest + +pytest.importorskip("hypothesis") +blosc1 = pytest.importorskip("blosc", reason="python-blosc (C-Blosc 1.x) is the reference") + +from hypothesis import HealthCheck, given, settings # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +from numcodecs import blosc as nc_blosc # noqa: E402 + +pytestmark = pytest.mark.blosc1_compat + +EXAMPLES = int(os.environ.get("BLOSC1_COMPAT_EXAMPLES", "300")) +MAX_TYPESIZE = 255 # C-Blosc stores a larger typesize as 1 + +# compressors that both libraries have: a build can leave some out (e.g. zstd, zlib) +CNAMES = sorted( + set(nc_blosc.list_compressors()) + & set(blosc1.compressor_list()) + & {"blosclz", "lz4", "lz4hc", "zlib", "zstd"} +) +SHUFFLES = [nc_blosc.NOSHUFFLE, nc_blosc.SHUFFLE, nc_blosc.BITSHUFFLE] + +# the sizes where things change: the buffer minimum (32), element counts around multiples +# of 8 (bitshuffle) and around 128 (the C-Blosc 1.x split rule), and typesizes around 16 +# (the largest splittable one) and 255 +EDGE_ITEMSIZES = [1, 2, 3, 4, 5, 7, 8, 12, 16, 17, 24, 32, 255, 256, 300] +EDGE_ELEMENTS = [0, 1, 2, 7, 8, 9, 15, 16, 17, 31, 32, 33, 127, 128, 129, 255, 256, 257, 1023, 1024] +EDGE_BLOCKSIZES = [0, 1, 7, 32, 33, 100, 128, 1000, 4096, 2**15 + 1, 2**16, 2**18, 2**20] +KINDS = ["zeros", "constant", "random", "ramp", "periodic", "sparse", "sorted"] + +SETTINGS = settings( + max_examples=EXAMPLES, + deadline=None, + database=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], + print_blob=True, +) + + +def make_payload(kind, seed, itemsize, nelements, leftover): + """Bytes of ``nelements`` items of ``itemsize`` bytes plus ``leftover`` extra bytes.""" + rng = np.random.default_rng(seed) + items = np.zeros((nelements, itemsize), dtype=np.uint8) + if kind == "constant": + items[:] = rng.integers(1, 256, dtype=np.uint8) + elif kind == "random": + items[:] = rng.integers(0, 256, size=items.shape, dtype=np.uint8) + elif kind in ("ramp", "sorted"): + # integers growing by a small step, little-endian across the item's bytes + step = rng.integers(0, 4 if kind == "ramp" else 2**20, size=nelements, dtype=np.uint64) + values = np.cumsum(step, dtype=np.uint64) + for j in range(min(itemsize, 8)): + items[:, j] = (values >> np.uint64(8 * j)) & np.uint64(0xFF) + elif kind == "periodic": + period = rng.integers(1, 9) + pattern = rng.integers(0, 256, size=(period, itemsize), dtype=np.uint8) + items[:] = pattern[np.arange(nelements) % period] + elif kind == "sparse": + mask = rng.random(items.shape) < 0.02 + items[mask] = rng.integers(0, 256, size=int(mask.sum()), dtype=np.uint8) + tail = rng.integers(0, 256, size=leftover, dtype=np.uint8) + return np.concatenate([items.reshape(-1), tail]).tobytes() + + +@st.composite +def payloads(draw): + itemsize = draw(st.sampled_from(EDGE_ITEMSIZES) | st.integers(1, 300)) + nelements = draw(st.sampled_from(EDGE_ELEMENTS) | st.integers(0, 4096)) + # the bytes that do not make up a whole item are copied through unfiltered + leftover = draw(st.sampled_from([0, 0, 1, itemsize - 1, itemsize // 2])) + leftover = max(0, min(leftover, itemsize - 1)) + kind = draw(st.sampled_from(KINDS)) + seed = draw(st.integers(0, 2**32 - 1)) + return itemsize, make_payload(kind, seed, itemsize, nelements, leftover) + + +compress_params = st.fixed_dictionaries( + { + "cname": st.sampled_from(CNAMES), + "clevel": st.integers(0, 9), + "shuffle": st.sampled_from(SHUFFLES), + "blocksize": st.sampled_from(EDGE_BLOCKSIZES) | st.integers(0, 2**20), + } +) + + +def v1_typesize(itemsize): + return itemsize if itemsize <= MAX_TYPESIZE else 1 + + +def v1_compress(data, itemsize, cname, clevel, shuffle, blocksize): + """Compress with C-Blosc 1.x.""" + blosc1.set_nthreads(1) + blosc1.set_blocksize(blocksize) + try: + return blosc1.compress( + data, typesize=v1_typesize(itemsize), clevel=clevel, shuffle=shuffle, cname=cname + ) + finally: + blosc1.set_blocksize(0) + + +@contextlib.contextmanager +def numcodecs_path(path): + """Make numcodecs compress and decompress through one of its two code paths. + + numcodecs uses blosc's global context when it may use threads, which is the default in + the main thread, and a context of its own otherwise (``use_threads = False``, other + threads, forked processes). With the c-blosc2 backend the two differ in how they ask for + Blosc1 chunks, so both have to be checked. ``use_threads`` is a global, so this is a + context manager and not a (function-scoped) fixture, which hypothesis does not allow. + """ + saved = nc_blosc.use_threads + nc_blosc.use_threads = path == "global" + try: + yield + finally: + nc_blosc.use_threads = saved + + +PATHS = pytest.mark.parametrize("path", ["global", "context"]) + + +def nc_compress(data, itemsize, cname, clevel, shuffle, blocksize): + """Compress with numcodecs, the way zarr does: bytes and an explicit typesize.""" + return nc_blosc.Blosc( + cname=cname, clevel=clevel, shuffle=shuffle, blocksize=blocksize, typesize=itemsize + ).encode(np.frombuffer(data, dtype=np.uint8)) + + +def check_header(chunk, data, itemsize): + """A chunk is a sound Blosc1 chunk of ``data``, whoever wrote it.""" + assert chunk[0] == 2, "format version is not the Blosc1 one" + nbytes, cbytes, blocksize = blosc1.get_cbuffer_sizes(chunk) + assert nbytes == len(data) + assert cbytes == len(chunk) + if data: + assert blocksize <= len(data) + assert chunk[3] == v1_typesize(itemsize) + + +@PATHS +@SETTINGS +@given(payload=payloads(), params=compress_params) +def test_decodes_in_c_blosc1(path, payload, params): + """Forward: what numcodecs writes, C-Blosc 1.x reads.""" + itemsize, data = payload + with numcodecs_path(path): + chunk = nc_compress(data, itemsize, **params) + check_header(chunk, data, itemsize) + assert blosc1.decompress(chunk) == data + + +@PATHS +@SETTINGS +@given(payload=payloads(), params=compress_params) +def test_decodes_from_c_blosc1(path, payload, params): + """Backward: what C-Blosc 1.x writes, numcodecs reads.""" + itemsize, data = payload + chunk = v1_compress(data, itemsize, **params) + with numcodecs_path(path): + assert bytes(nc_blosc.decompress(chunk)) == data + + +@PATHS +@SETTINGS +@given(payload=payloads(), params=compress_params) +def test_numcodecs_roundtrip(path, payload, params): + itemsize, data = payload + with numcodecs_path(path): + chunk = nc_compress(data, itemsize, **params) + assert bytes(nc_blosc.decompress(chunk)) == data + # the codec without a typesize takes it from the array, here bytes + codec = nc_blosc.Blosc(**params) + assert bytes(codec.decode(codec.encode(np.frombuffer(data, dtype=np.uint8)))) == data + + +@PATHS +@pytest.mark.parametrize( + ("itemsize", "dtype"), [(1, "u1"), (2, "u2"), (4, "f4"), (8, "f8"), (16, "c16")] +) +@SETTINGS +@given( + params=compress_params, + nelements=st.sampled_from(EDGE_ELEMENTS) | st.integers(0, 2048), + seed=st.integers(0, 2**32 - 1), +) +def test_typed_arrays(path, itemsize, dtype, params, nelements, seed): + """Arrays give numcodecs their typesize themselves, without a typesize argument.""" + data = make_payload("random" if seed % 2 else "ramp", seed, itemsize, nelements, 0) + array = np.frombuffer(data, dtype=dtype) + with numcodecs_path(path): + chunk = nc_blosc.compress( + array, + params["cname"].encode(), + params["clevel"], + params["shuffle"], + # a blocksize is not an array property, so ask for the automatic one here + 0, + ) + assert bytes(nc_blosc.decompress(chunk)) == data + check_header(chunk, data, itemsize) + assert blosc1.decompress(chunk) == data + + +# A deterministic grid for the same, so that a failure names its parameters and runs the +# same everywhere: every compressor x shuffle x typesize x size, in both directions +GRID = list( + itertools.product( + CNAMES, + SHUFFLES, + [1, 2, 3, 4, 7, 8, 16, 17, 255, 300], + [0, 1, 31, 32, 33, 127, 1001, 4096, 65537, 100003], + ) +) + + +@PATHS +@pytest.mark.parametrize(("cname", "shuffle", "itemsize", "nbytes"), GRID) +def test_grid(path, cname, shuffle, itemsize, nbytes): + data = make_payload("ramp", 1234, itemsize, nbytes // itemsize, nbytes % itemsize) + assert len(data) == nbytes + params = {"cname": cname, "clevel": 5, "shuffle": shuffle, "blocksize": 0} + + with numcodecs_path(path): + chunk = nc_compress(data, itemsize, **params) + check_header(chunk, data, itemsize) + assert blosc1.decompress(chunk) == data + + chunk = v1_compress(data, itemsize, **params) + with numcodecs_path(path): + assert bytes(nc_blosc.decompress(chunk)) == data From cb5bb755c1de04d4f8caeaed06fc99f0eeab30cc Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Thu, 22 Oct 2026 04:36:22 -0400 Subject: [PATCH 10/10] Update the c-blosc2 pin to the reviewed Blosc1 format writer Move the wrap to c919bfe6, the head of blosc1-format-writer after the review on Blosc/c-blosc2#815: a typesize guard, a codec whitelist, the filter accepted in any slot, blosc2_ctx_get_blosc1_format(), and multi-threaded tests. numcodecs only passes a positive typesize, the shuffle filter in the last slot and a supported codec, so nothing changes for it; the compatibility tests pass on this commit. Co-Authored-By: Claude Sonnet 5.5 --- packages/c-blosc2-blosc1-compat/UPSTREAM | 2 +- subprojects/c-blosc2.wrap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/c-blosc2-blosc1-compat/UPSTREAM b/packages/c-blosc2-blosc1-compat/UPSTREAM index effa94cf0..15aac53e9 100644 --- a/packages/c-blosc2-blosc1-compat/UPSTREAM +++ b/packages/c-blosc2-blosc1-compat/UPSTREAM @@ -1,5 +1,5 @@ repository = https://github.com/mkitti/c-blosc2 branch = blosc1-format-writer -commit = a4995c38d1250790ce7f7e4a6debcda41404fb4b +commit = c919bfe6834cd05ca853a4b63340b693bed9892f version = 3.3.6.dev pull_request = https://github.com/Blosc/c-blosc2/pull/815 diff --git a/subprojects/c-blosc2.wrap b/subprojects/c-blosc2.wrap index e98c50ce9..a484923b5 100644 --- a/subprojects/c-blosc2.wrap +++ b/subprojects/c-blosc2.wrap @@ -2,7 +2,7 @@ directory = c-blosc2 url = https://github.com/mkitti/c-blosc2.git # Branch blosc1-format-writer of the fork (Blosc/c-blosc2#815); pinned to its head commit. -revision = a4995c38d1250790ce7f7e4a6debcda41404fb4b +revision = c919bfe6834cd05ca853a4b63340b693bed9892f patch_directory = c-blosc2 [provide]