From f9897f4c460059b000c22e44a9169adff72a65eb Mon Sep 17 00:00:00 2001 From: fishidaho Date: Wed, 2 Sep 2026 17:36:03 -0700 Subject: [PATCH 1/3] Categorical-encode obs/var strings on write, as anndata's writers do `numeric_only_compression` exists because Blosc2 crashes on variable-length-string HDF5 datasets, so every string column in obs/var is written uncompressed. That left the package with no size strategy for metadata at all. Investigating what to do about it turned up a plain bug rather than a codec question. `anndata.AnnData.write_h5ad` converts string columns to categoricals before writing (its own `convert_strings_to_categoricals=True`). `VCSCAnnData` writes field-by-field rather than delegating -- deliberately, for the reasons in `_write_group`'s comment -- and so never did. A low-cardinality annotation (cell type, sample ID, batch) therefore landed as one variable-length string per row, uncompressed and unrecoverable. On 200k cells x 2k genes with three such columns: before (plain strings) file 41.72 MB after (categorical) file 13.48 MB Adds the same parameter with the same name and default to `write_h5ad` and `write_zarr`. anndata only converts columns with fewer categories than rows, so a per-row-unique column (barcodes, gene symbols) is left alone and this can never make a column larger. The codec question was answered too, by writing a 20k-element vlen string dataset under each filter, one subprocess each, against h5py 3.16.0 / HDF5 2.0.0 / hdf5plugin 7.0.0: none, gzip and lzf all fine; blosc2 still dies with SIGFPE. So the workaround stays, and it's specifically Blosc2 rather than HDF5 filters in general. gzip/lzf would be safe but aren't worth adopting -- vlen payloads live in HDF5's global heap where per-dataset compression doesn't reach them well, and the categorical encoding above is worth far more than any string codec could be. Both findings are recorded in `_compression`'s module docstring. No upstream issue filed yet: reproducing this outside the h5py/hdf5plugin combination in use here needs a check against a current hdf5plugin build first. Co-Authored-By: Claude Sonnet 5 --- src/vsparse/_anndata_class.py | 40 +++++++++-- src/vsparse/_compression.py | 35 +++++++++- tests/test_metadata_encoding.py | 119 ++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 tests/test_metadata_encoding.py diff --git a/src/vsparse/_anndata_class.py b/src/vsparse/_anndata_class.py index 789b172..d2a5a52 100644 --- a/src/vsparse/_anndata_class.py +++ b/src/vsparse/_anndata_class.py @@ -244,10 +244,19 @@ def _write_group( g: Any, *, format: str = "vcsc", + convert_strings_to_categoricals: bool = True, dataset_kwargs: Mapping[str, Any] = MappingProxyType({}), ) -> None: if format not in _STORE_FORMATS: raise ValueError(f"format must be one of {_STORE_FORMATS}, got {format!r}") + if convert_strings_to_categoricals: + # anndata's own write_h5ad/write_zarr do this (same parameter, same + # default); writing field-by-field means we have to do it too, or + # low-cardinality obs/var columns land as one variable-length + # string per row. That costs several times the whole file -- + # they're also the one thing here that can't be Blosc-compressed + # (see vsparse._compression), so nothing downstream recovers it. + self.strings_to_categoricals() write_array = ad.io.write_elem if format == "vcsc" else _io.write_ivcs_elem if self._vcs_X is not None: write_array(g, "X", self._vcs_X, dataset_kwargs=dataset_kwargs) @@ -276,6 +285,7 @@ def write_h5ad( # ty: ignore[invalid-method-override] filename: str | PathLike[str], *, format: str = "vcsc", + convert_strings_to_categoricals: bool = True, dataset_kwargs: Mapping[str, Any] | None = None, **_kwargs: Any, ) -> None: @@ -290,6 +300,15 @@ def write_h5ad( # ty: ignore[invalid-method-override] the cost of extra work on write/read. Either way, ``X``/``raw_X`` come back from :meth:`read_h5ad` as ordinary VCSCArray/VCSRArray objects -- ``"ivcsc"`` is purely an on-disk storage format. + convert_strings_to_categoricals + Convert ``obs``/``var`` string columns to categorical before + writing, in place, exactly as ``anndata``'s own writers do (same + name, same default). Only columns with fewer categories than rows + are converted, so it never makes a column larger -- and for the + typical low-cardinality annotation (cell type, sample ID) it is + worth several times the size of the whole file, because + variable-length strings are the one thing here that can't be + compressed (see :mod:`vsparse._compression`). dataset_kwargs Passed to ``h5py.Group.create_dataset`` for every array written. Defaults to Blosc2+LZ4 compression; pass ``{}`` to store @@ -304,7 +323,12 @@ def write_h5ad( # ty: ignore[invalid-method-override] _compression.numeric_only_compression("h5"), h5py.File(filename, "w") as f, ): - self._write_group(f, format=format, dataset_kwargs=dataset_kwargs) + self._write_group( + f, + format=format, + convert_strings_to_categoricals=convert_strings_to_categoricals, + dataset_kwargs=dataset_kwargs, + ) @classmethod def read_h5ad(cls, filename: str | PathLike[str]) -> VCSCAnnData: @@ -319,14 +343,15 @@ def write_zarr( store: Any, *, format: str = "vcsc", + convert_strings_to_categoricals: bool = True, dataset_kwargs: Mapping[str, Any] | None = None, **_kwargs: Any, ) -> None: """Write to a zarr store. Read back with :meth:`read_zarr`. - See :meth:`write_h5ad` for ``format``/``dataset_kwargs`` (including the - numeric-only compression behavior); the default compression here is - Blosc+LZ4 via ``numcodecs``. + See :meth:`write_h5ad` for ``format``/``convert_strings_to_categoricals``/ + ``dataset_kwargs`` (including the numeric-only compression behavior); + the default compression here is Blosc+LZ4 via ``numcodecs``. """ import zarr @@ -334,7 +359,12 @@ def write_zarr( dataset_kwargs = _compression.zarr_dataset_kwargs() with _compression.numeric_only_compression("zarr"): f = zarr.open_group(store, mode="w") - self._write_group(f, format=format, dataset_kwargs=dataset_kwargs) + self._write_group( + f, + format=format, + convert_strings_to_categoricals=convert_strings_to_categoricals, + dataset_kwargs=dataset_kwargs, + ) @classmethod def read_zarr(cls, store: Any) -> VCSCAnnData: diff --git a/src/vsparse/_compression.py b/src/vsparse/_compression.py index d88a9d3..9e5b217 100644 --- a/src/vsparse/_compression.py +++ b/src/vsparse/_compression.py @@ -11,11 +11,42 @@ ``_index``). At least some HDF5 filter-plugin builds (seen in this environment: h5py 3.16 / HDF5 2.0.0 / hdf5plugin's Blosc2) segfault (``SIGFPE``) when the Blosc2 filter is applied to a variable-length-string -dataset -- and there's no benefit to compressing already-tiny label arrays -anyway. :func:`numeric_only_compression` patches the relevant +dataset. :func:`numeric_only_compression` patches the relevant ``create_dataset``/``create_array`` calls for the duration of a write so string/object arrays always land uncompressed, regardless of what ``dataset_kwargs`` was passed in -- callers don't have to know about this. + +Why strings stay uncompressed, and what to do about size instead +---------------------------------------------------------------- + +The crash was re-checked against h5py 3.16.0 / HDF5 2.0.0 / hdf5plugin +7.0.0 by writing a 20k-element variable-length string dataset under each +available filter, one subprocess per filter: + +=========== =========================================== +``none`` fine +``gzip`` fine +``lzf`` fine +``blosc2`` dies with ``SIGFPE`` before returning +=========== =========================================== + +So the workaround is still required, and it is specifically Blosc2 -- not +HDF5 filters generally. gzip and lzf *would* be safe here, but neither is +worth adopting: this codec choice only ever applies to the string arrays +themselves, and the size problem those cause isn't a compression problem. +A low-cardinality annotation stored as one variable-length string per row +(cell type, sample ID, batch) pays for the string *and* its heap entry on +every row, and vlen payloads live in HDF5's global heap where per-dataset +compression doesn't reach them well in the first place. + +Encoding those columns as pandas categoricals fixes it at the source: the +per-row data becomes an integer code array -- numeric, so the existing +Blosc2 path compresses it -- and the labels are stored once. Measured on +200k cells x 2k genes with three low-cardinality string ``obs`` columns, +that takes the whole file from **41.82 MB to 13.61 MB**. This is why +:meth:`vsparse.VCSCAnnData.write_h5ad` converts them by default (its +``convert_strings_to_categoricals`` parameter, matching anndata's own +writers), and it is a bigger win than any string codec could be. """ from __future__ import annotations diff --git a/tests/test_metadata_encoding.py b/tests/test_metadata_encoding.py new file mode 100644 index 0000000..56b82fb --- /dev/null +++ b/tests/test_metadata_encoding.py @@ -0,0 +1,119 @@ +"""obs/var string columns are categorical-encoded on write, as anndata's writers do. + +Variable-length strings are the one thing the write path can't compress +(Blosc2 crashes on them -- see vsparse._compression), so a low-cardinality +annotation stored one string per row is pure, unrecoverable file size. +""" + +from __future__ import annotations + +import anndata as ad +import numpy as np +import pandas as pd +import pytest +import scipy.sparse as sp + +from vsparse import VCSCAnnData, VCSRArray + + +def _adata(n_cells: int = 400, n_genes: int = 20) -> ad.AnnData: + rng = np.random.default_rng(0) + obs = pd.DataFrame( + { + "cell_type": [f"type_{i}" for i in rng.integers(0, 5, n_cells)], + "sample_id": [f"SAMPLE_{i:03d}" for i in rng.integers(0, 12, n_cells)], + "total_counts": rng.normal(1000, 50, n_cells), + }, + index=[f"cell_{i:05d}" for i in range(n_cells)], + ) + var = pd.DataFrame( + { + "gene_symbol": [f"GENE{i:04d}" for i in range(n_genes)], # unique per gene + "chromosome": [f"chr{i}" for i in rng.integers(1, 5, n_genes)], # repeats + }, + index=[f"ENSG{i:05d}" for i in range(n_genes)], + ) + X = sp.random_array((n_cells, n_genes), density=0.2, format="csr", random_state=0) + X.data = np.round(X.data * 8 + 1) + return ad.AnnData(X=X, obs=obs, var=var) + + +@pytest.mark.parametrize("fmt", ["vcsc", "ivcsc"]) +def test_string_columns_are_written_as_categoricals(tmp_path, fmt): + va = VCSCAnnData.from_anndata(_adata(), format="csr") + assert not isinstance(va.obs["cell_type"].dtype, pd.CategoricalDtype) + + path = tmp_path / f"data.{fmt}.h5ad" + va.write_h5ad(path, format=fmt) + back = VCSCAnnData.read_h5ad(path) + + assert isinstance(back.obs["cell_type"].dtype, pd.CategoricalDtype) + assert isinstance(back.obs["sample_id"].dtype, pd.CategoricalDtype) + assert isinstance(back.var["chromosome"].dtype, pd.CategoricalDtype) + # gene_symbol is unique per gene: nothing to gain, so it's left alone. + assert not isinstance(back.var["gene_symbol"].dtype, pd.CategoricalDtype) + + +def test_values_survive_the_conversion(tmp_path): + original = _adata() + va = VCSCAnnData.from_anndata(original, format="csr") + path = tmp_path / "data.h5ad" + va.write_h5ad(path) + back = VCSCAnnData.read_h5ad(path) + + for col in ("cell_type", "sample_id"): + pd.testing.assert_series_equal( + back.obs[col].astype(str), original.obs[col].astype(str), check_names=False + ) + np.testing.assert_allclose(back.obs["total_counts"], original.obs["total_counts"]) + np.testing.assert_array_equal(back.obs_names, original.obs_names) + np.testing.assert_array_equal(back.var_names, original.var_names) + assert isinstance(back.X, VCSRArray) + np.testing.assert_allclose(back.X.toarray(), sp.csr_array(original.X).toarray()) + + +def test_conversion_can_be_turned_off(tmp_path): + va = VCSCAnnData.from_anndata(_adata(), format="csr") + path = tmp_path / "raw_strings.h5ad" + va.write_h5ad(path, convert_strings_to_categoricals=False) + back = VCSCAnnData.read_h5ad(path) + + assert not isinstance(back.obs["cell_type"].dtype, pd.CategoricalDtype) + assert not isinstance(va.obs["cell_type"].dtype, pd.CategoricalDtype) # not mutated + + +def test_categorical_encoding_shrinks_the_file(tmp_path): + """The size claim, on the shape of annotation that actually appears in practice.""" + va_plain = VCSCAnnData.from_anndata(_adata(n_cells=4000), format="csr") + va_cat = VCSCAnnData.from_anndata(_adata(n_cells=4000), format="csr") + + plain = tmp_path / "plain.h5ad" + cat = tmp_path / "cat.h5ad" + va_plain.write_h5ad(plain, convert_strings_to_categoricals=False) + va_cat.write_h5ad(cat, convert_strings_to_categoricals=True) + + assert cat.stat().st_size < plain.stat().st_size + + +def test_high_cardinality_columns_are_left_alone(tmp_path): + """A column with a distinct value per row gains nothing and is not converted.""" + adata = _adata(n_cells=100) + adata.obs["barcode"] = [f"barcode_{i}" for i in range(adata.n_obs)] + va = VCSCAnnData.from_anndata(adata, format="csr") + + path = tmp_path / "unique.h5ad" + va.write_h5ad(path) + back = VCSCAnnData.read_h5ad(path) + + assert not isinstance(back.obs["barcode"].dtype, pd.CategoricalDtype) + np.testing.assert_array_equal(back.obs["barcode"], adata.obs["barcode"]) + + +def test_zarr_write_converts_too(tmp_path): + va = VCSCAnnData.from_anndata(_adata(), format="csr") + store = tmp_path / "data.zarr" + va.write_zarr(store) + back = VCSCAnnData.read_zarr(store) + + assert isinstance(back.obs["cell_type"].dtype, pd.CategoricalDtype) + np.testing.assert_array_equal(back.obs["cell_type"].astype(str), va.obs["cell_type"].astype(str)) From d32e2cf8e872f423081060ea37c9cbdbf21a4487 Mon Sep 17 00:00:00 2001 From: fishidaho Date: Thu, 3 Sep 2026 11:18:25 -0700 Subject: [PATCH 2/3] Merge origin/main into investigate/metadata-compression Clean merge. #23 touched _anndata_class only in _subset_2d's comment; this branch touches _write_group/write_h5ad/write_zarr, so the two don't overlap. Nothing in #22/#23 addresses obs/var metadata encoding, and the categorical conversion is unaffected by the indexing changes. From fd60abaa962141b73cb19033262240f35557c7d5 Mon Sep 17 00:00:00 2001 From: fishidaho Date: Thu, 3 Sep 2026 17:10:18 -0700 Subject: [PATCH 3/3] Trim comments and docstrings --- src/vsparse/_anndata_class.py | 19 +++++------------ src/vsparse/_compression.py | 37 ++++++--------------------------- tests/test_metadata_encoding.py | 11 ++-------- 3 files changed, 13 insertions(+), 54 deletions(-) diff --git a/src/vsparse/_anndata_class.py b/src/vsparse/_anndata_class.py index 48fcc52..58e0ca2 100644 --- a/src/vsparse/_anndata_class.py +++ b/src/vsparse/_anndata_class.py @@ -251,12 +251,8 @@ def _write_group( if format not in _STORE_FORMATS: raise ValueError(f"format must be one of {_STORE_FORMATS}, got {format!r}") if convert_strings_to_categoricals: - # anndata's own write_h5ad/write_zarr do this (same parameter, same - # default); writing field-by-field means we have to do it too, or - # low-cardinality obs/var columns land as one variable-length - # string per row. That costs several times the whole file -- - # they're also the one thing here that can't be Blosc-compressed - # (see vsparse._compression), so nothing downstream recovers it. + # Writing field-by-field skips what anndata's own writers do here, + # leaving low-cardinality columns as one string per row. self.strings_to_categoricals() write_array = ad.io.write_elem if format == "vcsc" else _io.write_ivcs_elem if self._vcs_X is not None: @@ -302,14 +298,9 @@ def write_h5ad( # ty: ignore[invalid-method-override] come back from :meth:`read_h5ad` as ordinary VCSCArray/VCSRArray objects -- ``"ivcsc"`` is purely an on-disk storage format. convert_strings_to_categoricals - Convert ``obs``/``var`` string columns to categorical before - writing, in place, exactly as ``anndata``'s own writers do (same - name, same default). Only columns with fewer categories than rows - are converted, so it never makes a column larger -- and for the - typical low-cardinality annotation (cell type, sample ID) it is - worth several times the size of the whole file, because - variable-length strings are the one thing here that can't be - compressed (see :mod:`vsparse._compression`). + Convert ``obs``/``var`` string columns to categorical in place + before writing, as ``anndata``'s own writers do. Only columns + with fewer categories than rows are converted. dataset_kwargs Passed to ``h5py.Group.create_dataset`` for every array written. Defaults to Blosc2+LZ4 compression; pass ``{}`` to store diff --git a/src/vsparse/_compression.py b/src/vsparse/_compression.py index 9e5b217..22c116d 100644 --- a/src/vsparse/_compression.py +++ b/src/vsparse/_compression.py @@ -16,37 +16,12 @@ string/object arrays always land uncompressed, regardless of what ``dataset_kwargs`` was passed in -- callers don't have to know about this. -Why strings stay uncompressed, and what to do about size instead ----------------------------------------------------------------- - -The crash was re-checked against h5py 3.16.0 / HDF5 2.0.0 / hdf5plugin -7.0.0 by writing a 20k-element variable-length string dataset under each -available filter, one subprocess per filter: - -=========== =========================================== -``none`` fine -``gzip`` fine -``lzf`` fine -``blosc2`` dies with ``SIGFPE`` before returning -=========== =========================================== - -So the workaround is still required, and it is specifically Blosc2 -- not -HDF5 filters generally. gzip and lzf *would* be safe here, but neither is -worth adopting: this codec choice only ever applies to the string arrays -themselves, and the size problem those cause isn't a compression problem. -A low-cardinality annotation stored as one variable-length string per row -(cell type, sample ID, batch) pays for the string *and* its heap entry on -every row, and vlen payloads live in HDF5's global heap where per-dataset -compression doesn't reach them well in the first place. - -Encoding those columns as pandas categoricals fixes it at the source: the -per-row data becomes an integer code array -- numeric, so the existing -Blosc2 path compresses it -- and the labels are stored once. Measured on -200k cells x 2k genes with three low-cardinality string ``obs`` columns, -that takes the whole file from **41.82 MB to 13.61 MB**. This is why -:meth:`vsparse.VCSCAnnData.write_h5ad` converts them by default (its -``convert_strings_to_categoricals`` parameter, matching anndata's own -writers), and it is a bigger win than any string codec could be. +Strings are left uncompressed rather than given a different codec: gzip and +lzf are safe on variable-length strings here, but the size problem is the +per-row string itself, not its compression. Encoding low-cardinality +columns as categoricals turns the per-row data numeric, which the existing +Blosc2 path then compresses -- see +:meth:`vsparse.VCSCAnnData.write_h5ad`'s ``convert_strings_to_categoricals``. """ from __future__ import annotations diff --git a/tests/test_metadata_encoding.py b/tests/test_metadata_encoding.py index 56b82fb..94740c2 100644 --- a/tests/test_metadata_encoding.py +++ b/tests/test_metadata_encoding.py @@ -1,10 +1,3 @@ -"""obs/var string columns are categorical-encoded on write, as anndata's writers do. - -Variable-length strings are the one thing the write path can't compress -(Blosc2 crashes on them -- see vsparse._compression), so a low-cardinality -annotation stored one string per row is pure, unrecoverable file size. -""" - from __future__ import annotations import anndata as ad @@ -83,7 +76,7 @@ def test_conversion_can_be_turned_off(tmp_path): def test_categorical_encoding_shrinks_the_file(tmp_path): - """The size claim, on the shape of annotation that actually appears in practice.""" + """Categorical codes compress where per-row strings cannot.""" va_plain = VCSCAnnData.from_anndata(_adata(n_cells=4000), format="csr") va_cat = VCSCAnnData.from_anndata(_adata(n_cells=4000), format="csr") @@ -96,7 +89,7 @@ def test_categorical_encoding_shrinks_the_file(tmp_path): def test_high_cardinality_columns_are_left_alone(tmp_path): - """A column with a distinct value per row gains nothing and is not converted.""" + """A column with a distinct value per row is left alone.""" adata = _adata(n_cells=100) adata.obs["barcode"] = [f"barcode_{i}" for i in range(adata.n_obs)] va = VCSCAnnData.from_anndata(adata, format="csr")