Optional c-blosc2 backend for Blosc that writes Blosc1-format chunks

.github/workflows/blosc2-compat.yaml

@@ -0,0 +1,98 @@
+name: c-blosc2 Blosc1 compatibility
+
+on:
+  push:
+    branches: [main]
+    paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h']
+  pull_request:
+    paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h']
+  schedule:
+    - cron: '0 6 * * 1'  # weekly: does the series still apply to upstream c-blosc2 main?
+  workflow_dispatch:
+
+jobs:
+  wrap-in-sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: python packages/c-blosc2-blosc1-compat/tools/sync_wrap.py --check
+
+  forward-compat:
+    # c-blosc2 chunks must be readable by C-Blosc 1.x
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - run: pip install meson ninja cmake
+      - name: Build C-Blosc 1.x
+        run: |
+          git clone --depth 1 --branch v1.21.6 https://github.com/Blosc/c-blosc.git /tmp/c-blosc
+          cmake -S /tmp/c-blosc -B /tmp/b1 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
+            -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release
+          cmake --build /tmp/b1 -j
+      - name: Build the patched c-blosc2 through the wrap
+        run: |
+          mkdir /tmp/probe && cd /tmp/probe
+          ln -s "$GITHUB_WORKSPACE/subprojects" subprojects
+          printf "project('probe','c')\nb = dependency('blosc2')\nexecutable('probe', files('p.c'), dependencies: b)\n" > meson.build
+          printf '#include <blosc2.h>\nint main(void){return 0;}\n' > p.c
+          meson setup b && meson compile -C b
+      - name: Run the compatibility check
+        run: |
+          B=/tmp/probe/b
+          # c-blosc2 uses a system zstd/lz4/zlib when it finds one, otherwise the wrap's
+          # static build; link whichever is there
+          LIBS="$(find $B/subprojects -name libblosc2.a | head -n 1)"
+          for l in zstd lz4 z; do
+            a="$(find $B/subprojects -name lib$l.a | head -n 1)"
+            LIBS="$LIBS ${a:--l$l}"
+          done
+          python packages/c-blosc2-blosc1-compat/tests/run.py \
+            --blosc1-include /tmp/c-blosc/blosc --blosc1-lib /tmp/b1/blosc \
+            --blosc2-include "$(echo /tmp/probe/subprojects/c-blosc2-*/include)" \
+            --blosc2-ldflags "$LIBS"
+
+  test-blosc2-backend:
+    # the numcodecs test suite with Blosc built on c-blosc2 instead of c-blosc 1.x
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {platform: ubuntu-latest, python-version: "3.13"}
+          - {platform: ubuntu-latest, python-version: "3.14t"}
+          - {platform: ubuntu-22.04-arm, python-version: "3.13"}
+          - {platform: macos-14, python-version: "3.13"}
+          - {platform: windows-latest, python-version: "3.13"}
+    defaults:
+      run:
+        shell: bash -e {0}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # pyproject.toml's license-files still lists the c-blosc submodule's licenses
+          submodules: recursive
+          fetch-depth: 0 # required for version resolution
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Install numcodecs with the c-blosc2 backend
+        run: python -m pip install -v ".[test,test_extras]" -Csetup-args=-Dblosc_backend=blosc2
+      - name: Check the backend
+        run: python -c "import numcodecs.blosc as b; print(b.LIBRARY, b.VERSION_STRING, b.list_compressors()); assert b.LIBRARY == 'c-blosc2'"
+      - name: Run tests
+        run: pytest -v
+
+  rebase:
+    # Informational: does the series still apply to upstream main? Fails only on the schedule.
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - run: python packages/c-blosc2-blosc1-compat/tools/rebase.py main

.gitignore

@@ -99,4 +99,9 @@ ENV/
 .pixi/*
 *.egg-info
 pixi.lock
-uv.lock
+uv.lock
+# Wrapped subprojects: only the .wrap files and packagefiles are tracked
+/subprojects/*/
+!/subprojects/packagefiles/
+/subprojects/packagecache/
+/subprojects/.wraplock

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: ^fixture/  # files with trailing whitespaces on purpose
+exclude: ^fixture/|\.patch$  # trailing whitespace on purpose (fixtures, patch context lines)
 ci:
   autoupdate_commit_msg: "chore: update pre-commit hooks"
   autoupdate_schedule: "monthly"

docs/release.md

@@ -18,6 +18,21 @@
 
 ## 0.17.1
 
+### Enhancements
+
+* Add an optional c-blosc2 backend for the `Blosc` codec, selected at build time with
+  `-Dblosc_backend=blosc2` (default is still c-blosc 1.x). c-blosc2 is built from a meson
+  wrap (`subprojects/c-blosc2.wrap`) with a small patch so that it writes Blosc1-format
+  chunks that C-Blosc 1.x can decode (stock c-blosc2 output is not readable by C-Blosc 1.x).
+  zstd, lz4 and zlib are resolved separately from Blosc (system libraries or WrapDB wraps,
+  `-Dsystem_*`) and can be left out of Blosc with `-Dblosc_zstd=disabled` and
+  `-Dblosc_zlib=disabled`. `numcodecs.blosc.LIBRARY` reports the library in use; c-blosc2
+  does not provide `snappy`.
+* Validate Blosc frames before decompression, allowing valid zero-length frames to round-trip
+  while rejecting truncated frames, by {user}`Igor Stadnyk <fallenmi>`, {issue}`831` (#854).
+* `Blosc` can encode arrays whose dtype has an itemsize of 0, by
+  {user}`Raashish Aggarwal <raashish1601>`, {issue}`516` (#870).
+
 ### Security
 
 * `Pickle.decode` now refuses to unpickle data unless the process opts in, because

meson.options

@@ -1,5 +1,11 @@
+option('blosc_backend', type: 'combo', choices: ['blosc1', 'blosc2'], value: 'blosc1',
+       description: 'C library behind the Blosc codec. Both write Blosc1-format chunks.')
 option('system_blosc', type: 'feature', value: 'disabled',
-       description: 'Use system-installed Blosc library')
+       description: 'Use system-installed Blosc library (c-blosc 1.x for blosc_backend=blosc1, c-blosc2 for blosc2)')
+option('blosc_zstd', type: 'feature', value: 'enabled',
+       description: 'blosc_backend=blosc2 only: build Zstandard support into Blosc')
+option('blosc_zlib', type: 'feature', value: 'enabled',
+       description: 'blosc_backend=blosc2 only: build zlib support into Blosc')
 option('system_lz4', type: 'feature', value: 'disabled',
        description: 'Use system-installed LZ4 library')
 option('system_zlib', type: 'feature', value: 'disabled',

packages/c-blosc2-blosc1-compat/README.md

@@ -0,0 +1,72 @@
+# c-blosc2-blosc1-compat
+
+An overlay for [c-blosc2](https://github.com/Blosc/c-blosc2) that makes it usable as the
+backend of numcodecs' `Blosc` codec: a meson build, plus a small patch series that lets
+c-blosc2 **write chunks that C-Blosc 1.x can decode**. Stock c-blosc2 cannot, even through
+its `blosc1_*` API or with `BLOSC_BLOSC1_COMPAT` (its README says so, and
+`tests/run.py` confirms it).
+
+This directory is self-contained (it does not import anything from numcodecs) so it can be
+split out with `git subtree split` if it ever needs to live in its own repository, or be
+deleted if upstream accepts the patches.
+
+## Contents
+
+| Path | What |
+|---|---|
+| `UPSTREAM` | the pinned c-blosc2 commit and source hash; `subprojects/c-blosc2.wrap` must match |
+| `patches/` | the patch series, applied in order; one concern per patch |
+| `meson/` | `meson.build` and `meson.options` for c-blosc2 (upstream only has CMake) |
+| `tests/` | the forward-compatibility check, see below |
+| `tools/` | `sync_wrap.py`, `rebase.py` |
+| `TODO.md` | upstream items to check |
+
+## The patches
+
+Format version 2 (Blosc1) differs from what c-blosc2 writes in three ways that matter to a
+reader, plus one reader bug:
+
+1. `0001` reader: bytes left over after a bit-shuffled version-2 block were not copied.
+   A plain bug fix, independent of the rest.
+2. `0002` an opt-in flag: write the 16-byte Blosc1 header with format version 2. Set with
+   `blosc2_ctx_set_blosc1_format()` or, for the global context, `BLOSC_BLOSC1_COMPAT`.
+   Off by default, no ABI change.
+3. `0003` bitshuffle: version 2 shuffles a block only if its element count is a multiple
+   of 8, and stores it unshuffled otherwise.
+4. `0004` block splitting: use the C-Blosc 1.x rule, which its reader recomputes.
+
+Only plain shuffle/bitshuffle with blosclz, lz4, lz4hc, zlib and zstd are supported in this
+mode. The output decodes with C-Blosc 1.x but is not byte-identical to what it writes.
+
+## Wiring into the build
+
+Meson only looks for `patch_directory` and `diff_files` in `subprojects/packagefiles`, so
+`subprojects/packagefiles/c-blosc2/` holds copies of `meson/` and `patches/`. After editing
+here run `python tools/sync_wrap.py`; CI runs it with `--check`. If a patch is added or
+removed, update `diff_files` in `subprojects/c-blosc2.wrap`
+(`tools/make_wrap_diff_files.py` prints the line).
+
+## Testing
+
+`tests/run.py` builds small C programs against C-Blosc 1.x and the patched c-blosc2 and
+checks 735 cases in three directions (see its docstring for the command line):
+c-blosc2 global-context to C-Blosc 1.x, c-blosc2 context API to C-Blosc 1.x, and C-Blosc 1.x
+to c-blosc2. All must decode.
+
+## Updating c-blosc2
+
+`python tools/rebase.py <commit-or-tag>` downloads that c-blosc2, tries to apply the
+series, and reports which patch stops applying. To bump: edit `UPSTREAM` and
+`subprojects/c-blosc2.wrap` (directory, URL, hash), fix any patch that fails, re-export it
+as a unified diff against the pristine tree (`diff -ruN a b`), run `tools/sync_wrap.py`
+and the tests. The scheduled workflow `blosc2-compat-rebase` does the first step weekly.
+
+## Upstreaming
+
+Track each patch here when a PR exists, and delete the patch once it is in a release
+that `UPSTREAM` pins:
+
+| Patch | Upstream |
+|---|---|
+| 0001 | [issue #812](https://github.com/Blosc/c-blosc2/issues/812), [PR #813](https://github.com/Blosc/c-blosc2/pull/813) (see `TODO.md`) |
+| 0002–0004 | not yet submitted |

packages/c-blosc2-blosc1-compat/TODO.md

@@ -0,0 +1,20 @@
+# TODO
+
+## Check upstream c-blosc2 status of patch 0001
+
+Patch `0001-reader-bitunshuffle-v2-leftovers.patch` is submitted upstream:
+
+- issue: https://github.com/Blosc/c-blosc2/issues/812
+- pull request: https://github.com/Blosc/c-blosc2/pull/813
+
+Check periodically whether #813 has been merged and which c-blosc2 release contains it.
+When `UPSTREAM` pins a release that includes it:
+
+1. delete `patches/0001-reader-bitunshuffle-v2-leftovers.patch`;
+2. run `python tools/sync_wrap.py` and update `diff_files` in `subprojects/c-blosc2.wrap`
+   (`python tools/make_wrap_diff_files.py` prints the line);
+3. update the upstream table in `README.md`.
+
+If upstream asks for changes or rejects it, update the patch and this note to match.
+
+Patches `0002`-`0004` (the opt-in Blosc1 writer) have not been submitted upstream.

packages/c-blosc2-blosc1-compat/UPSTREAM

@@ -0,0 +1,4 @@
+repository = https://github.com/Blosc/c-blosc2
+commit = 38258631914de7f6ca2cf55c9675d771796ab96c
+version = 3.3.6.dev
+sha256 = c5556a00d8fc6a6a7be17e8f54e5c2aae2d304e740ac1aeb245db0f5731950a6

packages/c-blosc2-blosc1-compat/meson/meson.build

@@ -0,0 +1,130 @@
+# Meson build for c-blosc2, which upstream only builds with CMake.
+# Only the core library plus the built-in codec/filter/tuner plugins are built
+# (no ZFP, tests, benchmarks or examples). Used via subprojects/c-blosc2.wrap.
+project(
+  'c-blosc2', 'c',
+  version: '3.3.6.dev',
+  meson_version: '>=1.6.0',
+  default_options: ['c_std=gnu11', 'warning_level=0'],
+)
+
+cc = meson.get_compiler('c')
+host_system = host_machine.system()
+cpu = host_machine.cpu_family()
+
+blosc2_args = ['-DBUILD_STATIC']
+if host_system != 'windows'
+  blosc2_args += ['-D_GNU_SOURCE']
+else
+  blosc2_args += ['-D_CRT_SECURE_NO_WARNINGS']
+endif
+
+deps = [dependency('threads')]
+if host_system != 'windows'
+  deps += cc.find_library('m', required: false)
+  deps += cc.find_library('dl', required: false)
+endif
+
+# --- compression libraries (resolved by the parent project or WrapDB) ---
+lz4_dep = dependency('liblz4', fallback: ['lz4', 'liblz4_dep'],
+                      default_options: ['default_library=static'])
+deps += lz4_dep
+blosc2_args += ['-DHAVE_LZ4']
+
+zstd_dep = dependency('libzstd', required: get_option('zstd'),
+                      fallback: ['zstd', 'libzstd_dep'],
+                      default_options: ['default_library=static'])
+if zstd_dep.found()
+  deps += zstd_dep
+  blosc2_args += ['-DHAVE_ZSTD']
+endif
+
+zlib_dep = dependency('zlib', required: get_option('zlib'),
+                      fallback: ['zlib', 'zlib_dep'],
+                      default_options: ['default_library=static'])
+if zlib_dep.found()
+  deps += zlib_dep
+  blosc2_args += ['-DHAVE_ZLIB']
+endif
+
+sources = files(
+  'blosc/blosc2.c', 'blosc/blosclz.c', 'blosc/fastcopy.c', 'blosc/schunk.c',
+  'blosc/frame.c', 'blosc/stune.c', 'blosc/delta.c', 'blosc/shuffle.c',
+  'blosc/shuffle-generic.c', 'blosc/bitshuffle-generic.c',
+  'blosc/trunc-prec.c', 'blosc/timestamp.c', 'blosc/sframe.c',
+  'blosc/directories.c', 'blosc/blosc2-stdio.c', 'blosc/b2nd.c',
+  'blosc/b2nd_utils.c',
+  # plugins: registries and the plugins they register (ZFP excluded)
+  'plugins/codecs/codecs-registry.c',
+  'plugins/codecs/ndlz/ndlz.c', 'plugins/codecs/ndlz/ndlz4x4.c',
+  'plugins/codecs/ndlz/ndlz8x8.c', 'plugins/codecs/ndlz/xxhash.c',
+  'plugins/filters/filters-registry.c',
+  'plugins/filters/bytedelta/bytedelta.c',
+  'plugins/filters/int_trunc/int_trunc.c',
+  'plugins/filters/ndcell/ndcell.c', 'plugins/filters/ndmean/ndmean.c',
+  'plugins/tuners/tuners-registry.c',
+)
+if host_system == 'windows'
+  sources += files('blosc/win32/threading.c')
+endif
+
+# --- SIMD kernels: detected by compiler capability (cross-compile safe) ---
+# Each kernel file gets its ISA flag; shuffle.c only learns that it may
+# dispatch to them at run time (SHUFFLE_*_ENABLED) and is built portably.
+shuffle_defs = []
+simd_libs = []
+msvc = cc.get_argument_syntax() == 'msvc'
+
+if cpu in ['x86', 'x86_64']
+  if not get_option('sse2').disabled() and cc.has_argument(msvc ? '/arch:SSE2' : '-msse2')
+    simd_libs += static_library('blosc2_sse2',
+      files('blosc/shuffle-sse2.c', 'blosc/bitshuffle-sse2.c'),
+      include_directories: include_directories('include', 'blosc'),
+      c_args: blosc2_args + (msvc ? [] : ['-msse2']),
+      dependencies: deps)
+    shuffle_defs += '-DSHUFFLE_SSE2_ENABLED'
+  endif
+  if not get_option('avx2').disabled() and cc.has_argument(msvc ? '/arch:AVX2' : '-mavx2')
+    simd_libs += static_library('blosc2_avx2',
+      files('blosc/shuffle-avx2.c', 'blosc/bitshuffle-avx2.c'),
+      include_directories: include_directories('include', 'blosc'),
+      c_args: blosc2_args + (msvc ? ['/arch:AVX2'] : ['-mavx2']),
+      dependencies: deps)
+    shuffle_defs += '-DSHUFFLE_AVX2_ENABLED'
+  endif
+  if not get_option('avx512').disabled() and not msvc and cc.has_argument('-mavx512bw')
+    simd_libs += static_library('blosc2_avx512',
+      files('blosc/bitshuffle-avx512.c'),
+      include_directories: include_directories('include', 'blosc'),
+      c_args: blosc2_args + ['-mavx512f', '-mavx512bw'],
+      dependencies: deps)
+    shuffle_defs += '-DSHUFFLE_AVX512_ENABLED'
+  endif
+endif
+
+if cpu == 'aarch64' and not get_option('neon').disabled()
+  simd_libs += static_library('blosc2_neon',
+    files('blosc/shuffle-neon.c'),
+    include_directories: include_directories('include', 'blosc'),
+    c_args: blosc2_args + ['-flax-vector-conversions'],
+    dependencies: deps)
+  shuffle_defs += '-DSHUFFLE_NEON_ENABLED'
+endif
+
+blosc2_inc = include_directories('include')
+
+blosc2_lib = static_library('blosc2', sources,
+  include_directories: [blosc2_inc, include_directories('blosc')],
+  c_args: blosc2_args + shuffle_defs,
+  dependencies: deps,
+  link_whole: simd_libs,
+  pic: true,
+)
+
+blosc2_dep = declare_dependency(
+  link_with: blosc2_lib,
+  include_directories: blosc2_inc,
+  dependencies: deps,
+  compile_args: ['-DBUILD_STATIC'],
+)
+meson.override_dependency('blosc2', blosc2_dep)

packages/c-blosc2-blosc1-compat/meson/meson.options

@@ -0,0 +1,12 @@
+option('zstd', type: 'feature', value: 'enabled',
+       description: 'Build Zstandard support into Blosc2')
+option('zlib', type: 'feature', value: 'enabled',
+       description: 'Build zlib support into Blosc2')
+option('sse2', type: 'feature', value: 'auto',
+       description: 'Enable SSE2 shuffle kernels')
+option('avx2', type: 'feature', value: 'auto',
+       description: 'Enable AVX2 shuffle kernels')
+option('avx512', type: 'feature', value: 'auto',
+       description: 'Enable AVX512 bitshuffle kernels')
+option('neon', type: 'feature', value: 'auto',
+       description: 'Enable NEON shuffle kernels')