Read pre-1.3.0 / CSV-only Xenium bundles

CHANGELOG.md

@@ -16,6 +16,12 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page.
 ### Added
 
 - `spatialdata_io` ships a `py.typed` marker, so downstream type checkers use its annotations.
+- `xenium` reads pre-1.3.0 (XOA < 1.3.0) and CSV-only bundles: when the parquet files and
+  `cells.zarr.zip` are absent, the table, boundaries, and transcripts are read from the `cells.csv`,
+  `cell_boundaries.csv`, `nucleus_boundaries.csv` and `transcripts.csv` outputs (`.gz` or plain),
+  and the raster cell/nucleus labels are reconstructed by rasterizing the boundary polygons. The
+  cell feature matrix falls back to a `cell_feature_matrix/` MatrixMarket directory or
+  `cell_feature_matrix.tar.gz` when `cell_feature_matrix.h5` is missing.
 
 ### Changed
 

src/spatialdata_io/_constants/_constants.py

@@ -106,8 +106,14 @@ class XeniumKeys(ModeEnum):
     BOUNDARIES_VERTEX_X = "vertex_x"
     BOUNDARIES_VERTEX_Y = "vertex_y"
 
+    # CSV outputs of pre-1.3.0 (XOA < 1.3.0) bundles, used as a fallback when the parquet files
+    # (and cells.zarr.zip) are absent. Same columns as the parquet equivalents.
+    NUCLEUS_BOUNDARIES_FILE_CSV = "nucleus_boundaries.csv.gz"
+    CELL_BOUNDARIES_FILE_CSV = "cell_boundaries.csv.gz"
+
     # transcripts
     TRANSCRIPTS_FILE = "transcripts.parquet"
+    TRANSCRIPTS_FILE_CSV = "transcripts.csv.gz"
     TRANSCRIPTS_X = "x_location"
     TRANSCRIPTS_Y = "y_location"
     TRANSCRIPTS_Z = "z_location"
@@ -117,7 +123,11 @@ class XeniumKeys(ModeEnum):
 
     # cell features matrix
     CELL_FEATURE_MATRIX_FILE = "cell_feature_matrix.h5"
+    # MatrixMarket fallbacks for bundles without the HDF5 matrix (e.g. GEO deposits).
+    CELL_FEATURE_MATRIX_DIR = "cell_feature_matrix"
+    CELL_FEATURE_MATRIX_TAR = "cell_feature_matrix.tar.gz"
     CELL_METADATA_FILE = "cells.parquet"
+    CELL_METADATA_FILE_CSV = "cells.csv.gz"
     CELL_X = "x_centroid"
     CELL_Y = "y_centroid"
     CELL_AREA = "cell_area"

tests/fixtures/ATTRIBUTION.md

@@ -0,0 +1,32 @@
+# Test fixture attribution
+
+The `xenium-*-csv-tiny` fixtures are small subsets of public 10x Genomics Xenium
+example datasets, used only to test reading of CSV-only / pre-1.3.0 bundles.
+
+Both source datasets are © 10x Genomics and licensed under
+[Creative Commons Attribution 4.0 International (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/).
+
+## `xenium-1.0.2-csv-tiny/`
+
+- **Source:** Fresh Frozen Mouse Brain for Xenium Explorer Demo (Xenium Onboard Analysis 1.0.2),
+  region `Xenium_V1_FF_Mouse_Brain_Coronal_Subset_CTX+HP`.
+  <https://www.10xgenomics.com/datasets/fresh-frozen-mouse-brain-for-xenium-explorer-demo-1-standard>
+- **Author:** 10x Genomics.
+- **License:** CC BY 4.0.
+- **Edits:** subset to 30 cells; kept only the CSV-form outputs
+  (`cell_boundaries`, `nucleus_boundaries`, `cells`, `transcripts` as `.csv.gz`),
+  the `cell_feature_matrix.h5` (subset to the same cells), and `experiment.xenium`;
+  removed the zarr/parquet outputs, images, and auxiliary outputs. No values were altered.
+
+## `xenium-3.0.0-csv-hex-tiny/`
+
+- **Source:** Fresh Frozen Mouse Brain Hemisphere, 5K Mouse Pan Tissue & Pathways Panel
+  (Xenium Prime, Xenium Onboard Analysis 3.0.0).
+  <https://www.10xgenomics.com/datasets/xenium-prime-fresh-frozen-mouse-brain>
+- **Author:** 10x Genomics.
+- **License:** CC BY 4.0.
+- **Edits:** subset to 12 cells; kept only the CSV-form outputs
+  (`cell_boundaries`, `nucleus_boundaries`, `cells` as `.csv.gz`, carrying the hex
+  `cell_id` and `label_id` columns), the `cell_feature_matrix.h5` (subset to the same
+  cells), and `experiment.xenium`; removed the zarr/parquet outputs, transcripts,
+  images, and auxiliary outputs. No values were altered.

tests/fixtures/xenium-1.0.2-csv-tiny/experiment.xenium

@@ -0,0 +1,38 @@
+{
+    "run_name": "mouse_brain_ff",
+    "run_start_time": "2022-10-21T18:26:22Z",
+    "region_name": "Xenium_V1_FF_Mouse_Brain_Coronal_Subset_CTX+HP",
+    "preservation_method": "fresh_frozen",
+    "num_cells": 36602,
+    "transcripts_per_cell": 211.0,
+    "transcripts_per_100um": 79.86883303595566,
+    "cassette_name": "sim2",
+    "slide_id": "00002131",
+    "panel_design_id": "mBrain_v1",
+    "panel_name": "Xenium Mouse Brain GEX",
+    "panel_organism": "Mouse",
+    "panel_tissue_type": "Brain",
+    "panel_num_targets_predesigned": 248,
+    "panel_num_targets_custom": 0,
+    "pixel_size": 0.2125,
+    "instrument_sn": "XETG00001",
+    "instrument_sw_version": "1.0.0.3",
+    "analysis_sw_version": "xenium-1.0.2",
+    "experiment_uuid": "14d2c2b1-fe7b-4bc1-9c02-e7dabf91d5ec",
+    "cassette_uuid": "287f823e-665c-46de-81e7-149ed1c087b9",
+    "roi_uuid": "81aa6ae0-7648-4160-8f38-20d1cda7d4dd",
+    "z_step_size": 3.0,
+    "well_uuid": "def_60baf884_rec_7186106a",
+    "images": {
+        "morphology_filepath": "morphology.ome.tif",
+        "morphology_mip_filepath": "morphology_mip.ome.tif",
+        "morphology_focus_filepath": "morphology_focus.ome.tif"
+    },
+    "xenium_explorer_files": {
+        "transcripts_zarr_filepath": "transcripts.zarr.zip",
+        "cells_zarr_filepath": "cells.zarr.zip",
+        "cell_features_zarr_filepath": "cell_feature_matrix.zarr.zip",
+        "analysis_zarr_filepath": "analysis.zarr.zip",
+        "analysis_summary_filepath": "analysis_summary.html"
+    }
+}

tests/fixtures/xenium-3.0.0-csv-hex-tiny/experiment.xenium

@@ -0,0 +1,56 @@
+{
+    "major_version": 5,
+    "minor_version": 0,
+    "patch_version": 0,
+    "run_name": "Mouse FF Brain Hemisphere (Coronal) 5K",
+    "run_start_time": "2024-06-28T20:28:29Z",
+    "region_name": "Mouse FF Brain Hemisphere (Coronal)",
+    "preservation_method": "FF",
+    "num_cells": 63173,
+    "transcripts_per_cell": 1088,
+    "transcripts_per_100um": 1518.0164540474093,
+    "region_area": 33737853.4071875,
+    "total_cell_area": 5699917.268307667,
+    "thickness_of_high_quality_decoded_transcripts": 4.592364546878305,
+    "nuclear_transcripts_per_100um": 1692.2772242796623,
+    "fraction_transcripts_assigned": 0.5795513432527851,
+    "segmented_cell_stain_frac": 0.9485539708419737,
+    "segmented_cell_boundary_frac": 0.009592705744542764,
+    "segmented_cell_interior_frac": 0.9389612650974308,
+    "segmented_cell_nuc_expansion_frac": 0.051446029158026374,
+    "imported_cell_frac": 0.0,
+    "cassette_name": "N/A",
+    "slide_id": "N/A",
+    "panel_type": "predesigned",
+    "panel_design_id": "mAtlas_v1",
+    "panel_predesigned_id": "",
+    "panel_name": "Xenium Mouse 5K Pan Tissue & Pathways Panel",
+    "panel_organism": "Mouse",
+    "panel_tissue_type": "Multi",
+    "panel_num_targets_predesigned": 5006,
+    "panel_num_targets_custom": 0,
+    "chemistry_version": "v2",
+    "pixel_size": 0.2125,
+    "instrument_sn": "R&D",
+    "instrument_sw_version": "R&D",
+    "analysis_sw_version": "xenium-3.0.0.15",
+    "analysis_uuid": "84fee636-44cb-45be-8dfa-b73458348494",
+    "experiment_uuid": "91c670fc-a075-44a0-9e8d-7239e5c6c31b",
+    "cassette_uuid": "6326d085-0228-472a-8b8c-ec24a854f2a2",
+    "roi_uuid": "f9f6e19b-8c59-457c-a311-aefa3440093c",
+    "z_step_size": 3.0,
+    "well_uuid": "def_e7e9ca67_rec_42c0e00f",
+    "calibration_uuid": "c0cfdf3d-39bc-411c-a243-02ef93edb21e",
+    "images": {
+        "morphology_filepath": "morphology.ome.tif",
+        "morphology_focus_filepath": "morphology_focus/morphology_focus_0000.ome.tif"
+    },
+    "xenium_explorer_files": {
+        "transcripts_zarr_filepath": "transcripts.zarr.zip",
+        "cells_zarr_filepath": "cells.zarr.zip",
+        "cell_features_zarr_filepath": "cell_feature_matrix.zarr.zip",
+        "analysis_zarr_filepath": "analysis.zarr.zip",
+        "analysis_summary_filepath": "analysis_summary.html"
+    },
+    "segmentation_stain": "Xenium Multi-Tissue Stain"
+}

tests/test_xenium.py

@@ -1,8 +1,10 @@
 import math
+import warnings
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
 import numpy as np
+import pandas as pd
 import pytest
 from click.testing import CliRunner
 from pytest_mock import MockerFixture
@@ -12,6 +14,7 @@
 from spatialdata_io.__main__ import xenium_wrapper
 from spatialdata_io.readers.xenium import (
     _cell_id_str_from_prefix_suffix_uint32_reference,
+    _warn_if_scaled_protein,
     cell_id_str_from_prefix_suffix_uint32,
     prefix_suffix_uint32_from_cell_id_str,
     xenium,
@@ -211,6 +214,132 @@ def test_cli_xenium(runner: CliRunner, dataset: str) -> None:
         _ = read_zarr(output_zarr)
 
 
+# A real pre-1.3.0 (XOA 1.0.2) bundle reduced to the CSV outputs a GEO deposit typically keeps:
+# cells.csv.gz, cell/nucleus_boundaries.csv.gz, transcripts.csv.gz, cell_feature_matrix.h5,
+# experiment.xenium -- no parquet and no cells.zarr.zip. Subset to 30 cells of the 10x Mouse Brain
+# dataset (sddb 104cz).
+XENIUM_CSV_ONLY = Path(__file__).parent / "fixtures" / "xenium-1.0.2-csv-tiny"
+
+
+def test_xenium_csv_only_bundle() -> None:
+    # defaults: cells_as_circles=False, cells_labels/nucleus_labels=True. The labels, which live only
+    # in cells.zarr.zip, are reconstructed by rasterizing the boundary polygons read from the CSVs.
+    sdata = xenium(
+        XENIUM_CSV_ONLY,
+        morphology_mip=False,
+        morphology_focus=False,
+        aligned_images=False,
+    )
+    assert sdata["table"].n_obs == 30
+    assert len(sdata["cell_boundaries"]) == 30
+    assert len(sdata["nucleus_boundaries"]) == 30
+    assert len(sdata["transcripts"]) > 0
+    # labels reconstructed from the boundaries, keyed by the 30 integer cell_ids
+    for name in ("cell_labels", "nucleus_labels"):
+        assert _label_ids(sdata[name]) == set(range(1, 31))
+
+
+def test_xenium_csv_only_circles() -> None:
+    sdata = xenium(
+        XENIUM_CSV_ONLY,
+        cells_as_circles=True,
+        morphology_mip=False,
+        morphology_focus=False,
+        aligned_images=False,
+    )
+    assert len(sdata["cell_circles"]) == 30
+
+
+def test_xenium_csv_only_uncompressed(tmp_path: Path) -> None:
+    # some GEO deposits ship uncompressed .csv rather than .csv.gz
+    import gzip
+    import shutil
+
+    for f in XENIUM_CSV_ONLY.iterdir():
+        if f.suffixes[-2:] == [".csv", ".gz"]:
+            with gzip.open(f, "rb") as src, open(tmp_path / f.stem, "wb") as dst:  # f.stem drops .gz
+                shutil.copyfileobj(src, dst)
+        else:
+            shutil.copy(f, tmp_path / f.name)
+
+    sdata = xenium(tmp_path, morphology_mip=False, morphology_focus=False, aligned_images=False)
+    assert sdata["table"].n_obs == 30
+    assert len(sdata["cell_boundaries"]) == 30
+
+
+def test_xenium_csv_only_mtx_matrix(tmp_path: Path) -> None:
+    # some GEO deposits ship the MatrixMarket cell_feature_matrix/ dir instead of the .h5
+    import gzip
+    import shutil
+
+    import scanpy as sc
+    from scipy.io import mmwrite
+
+    for f in XENIUM_CSV_ONLY.iterdir():
+        if f.name != "cell_feature_matrix.h5":
+            shutil.copy(f, tmp_path / f.name)
+    adata = sc.read_10x_h5(XENIUM_CSV_ONLY / "cell_feature_matrix.h5", gex_only=False)
+    mdir = tmp_path / "cell_feature_matrix"
+    mdir.mkdir()
+    with gzip.open(mdir / "matrix.mtx.gz", "wb") as fh:
+        mmwrite(fh, adata.X.T)  # type: ignore[arg-type]  # MatrixMarket is features x barcodes
+    with gzip.open(mdir / "features.tsv.gz", "wt") as fh:
+        for gid, name, ft in zip(adata.var["gene_ids"], adata.var_names, adata.var["feature_types"], strict=True):
+            fh.write(f"{gid}\t{name}\t{ft}\n")
+    with gzip.open(mdir / "barcodes.tsv.gz", "wt") as fh:
+        fh.writelines(f"{b}\n" for b in adata.obs_names)
+
+    sdata = xenium(tmp_path, morphology_mip=False, morphology_focus=False, aligned_images=False)
+    assert sdata["table"].n_obs == 30
+    assert "cell_labels" in sdata.labels
+
+
+def test_warn_if_scaled_protein() -> None:
+    # protein counts read outside HDF5 cannot be descaled (the factor lives only in the .h5), so warn
+    from anndata import AnnData
+
+    gex = AnnData(np.zeros((2, 1)), var=pd.DataFrame({"feature_types": ["Gene Expression"]}))
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        _warn_if_scaled_protein(gex)  # no protein -> no warning
+
+    prot = AnnData(np.zeros((2, 1)), var=pd.DataFrame({"feature_types": ["Protein Expression"]}))
+    with pytest.warns(UserWarning, match="scaled units"):
+        _warn_if_scaled_protein(prot)
+
+
+# A v2/v3 (XOA 3.0.0) CSV-only bundle: hex cell_ids, and a label_id column in the boundary CSVs.
+# Subset to 12 cells of the 10x Xenium Prime Mouse Brain dataset (sddb efli2b4vhc).
+XENIUM_CSV_HEX = Path(__file__).parent / "fixtures" / "xenium-3.0.0-csv-hex-tiny"
+
+
+def _label_ids(element: object) -> set[int]:
+    import xarray as xr
+    from xarray import DataTree
+
+    if isinstance(element, DataTree):  # multiscale: take full-res scale0
+        node = element["scale0"]
+        element = node[str(next(iter(node.data_vars)))]
+    assert isinstance(element, xr.DataArray)
+    return set(np.unique(np.asarray(element.data)).tolist()) - {0}
+
+
+def test_xenium_csv_only_hex_labels() -> None:
+    # for hex cell_ids the integer raster label comes from the boundary CSVs' label_id column, and
+    # multinucleate cells yield more nucleus labels than cells
+    sdata = xenium(
+        XENIUM_CSV_HEX, morphology_mip=False, morphology_focus=False, aligned_images=False, transcripts=False
+    )
+    assert sdata["table"].n_obs == 12
+    assert isinstance(sdata["table"].obs["cell_id"].iloc[0], str)  # hex ids
+    # the raster labels must be the true (non-contiguous) label_id, and must match the table column
+    # the instance key resolves against -- not a 1..N rank (which would silently mis-join)
+    cell_label_ids = _label_ids(sdata["cell_labels"])
+    assert cell_label_ids == set(sdata["table"].obs["cell_labels"])
+    assert max(cell_label_ids) > 12  # genuine label_ids have gaps; a 1..12 rank would fail this
+    assert len(_label_ids(sdata["nucleus_labels"])) >= 12  # >= cells (multinucleate)
+
+
 @skip_if_below_python_version()
 @pytest.mark.parametrize(
     (