Source code for photon_mosaic.core.zarrrois

"""Zarr format for ROI data.

Classes
-------
ZarrRois
    ROI extractor backed by a zarr group on disk.

Functions
---------
add_rois_to_zarr_group
    Save ROI masks and metadata to a zarr group.
"""

from pathlib import Path

import numpy as np
import sparse
from spikeinterface.core.zarrextractors import create_zarr_path_for_write

from .baserois import BaseRois


[docs] class ZarrRois(BaseRois): """ROI extractor backed by a zarr group on disk. Parameters ---------- zarr_path : str or Path Path to the zarr store. zarr_group_name : str, default: "rois" Name of the group within the store containing ROI data. storage_options : dict | None, default: None fsspec storage options for remote zarr stores. """ def __init__(self, zarr_path, zarr_group_name: str = "rois", storage_options: dict | None = None): from spikeinterface.core.zarrextractors import super_zarr_open zarr_root = super_zarr_open(str(zarr_path), mode="r", storage_options=storage_options or {}) rois_group = zarr_root[zarr_group_name] roi_ids = np.array(rois_group["roi_ids"]) sampling_frequency = rois_group.attrs["sampling_frequency"] shape = tuple(rois_group.attrs["shape"]) BaseRois.__init__( self, sampling_frequency=sampling_frequency, shape=shape, roi_ids=roi_ids, ) self._rois_group = rois_group # Load properties if "properties" in rois_group: for key in rois_group["properties"].keys(): values = np.array(rois_group["properties"][key]) self.set_property(key, values) # Load annotations if "annotations" in rois_group.attrs: self.annotate(**rois_group.attrs["annotations"]) self._kwargs = dict( zarr_path=str(zarr_path), zarr_group_name=zarr_group_name, storage_options=storage_options, )
[docs] def get_roi_image_masks(self, roi_ids=None): if self._rois_group.attrs.get("roi_image_masks_sparse", False): return self._get_sparse_roi_image_masks(roi_ids) # Index the still-lazy zarr array *before* materialising, so a request for a few # ROIs only reads their own chunks (each ROI is its own chunk, see add_rois_to_zarr_group) # instead of loading every ROI's mask just to throw most of them away. roi_image_masks = self._rois_group["roi_image_masks"] if roi_ids is None: return np.array(roi_image_masks) roi_indices = self.ids_to_indices(roi_ids) return np.array(roi_image_masks[roi_indices])
def _get_sparse_roi_image_masks(self, roi_ids): """Reconstruct a GCXS array from its on-disk (indptr, indices, data) components. These are exactly the CSR-style arrays `sparse.GCXS(compressed_axes=(0,))` already keeps in memory -- `indptr` marks each ROI's own slice of the flat `indices`/`data` arrays. Reading a subset of ROIs therefore only needs `indptr` (always loaded, tiny: one int per ROI) plus the requested ROIs' own slices of `indices`/`data`, read from the still-lazy zarr arrays -- never the full arrays, unlike reconstructing via COO. """ shape = tuple(self._rois_group.attrs["roi_image_masks_shape"]) indptr = np.array(self._rois_group["roi_image_masks_indptr"]) indices = self._rois_group["roi_image_masks_indices"] data = self._rois_group["roi_image_masks_data"] if roi_ids is None: return sparse.GCXS((np.array(data), np.array(indices), indptr), shape=shape, compressed_axes=(0,)) roi_indices = self.ids_to_indices(roi_ids) indices_parts = [] data_parts = [] counts = [] for i in roi_indices: start, end = int(indptr[i]), int(indptr[i + 1]) indices_parts.append(np.array(indices[start:end])) data_parts.append(np.array(data[start:end])) counts.append(end - start) new_indptr = np.concatenate([[0], np.cumsum(counts)]).astype(indptr.dtype) new_indices = np.concatenate(indices_parts) if indices_parts else np.array([], dtype=indptr.dtype) new_data = np.concatenate(data_parts) if data_parts else np.array([], dtype=data.dtype) new_shape = (len(roi_indices), *shape[1:]) return sparse.GCXS((new_data, new_indices, new_indptr), shape=new_shape, compressed_axes=(0,))
[docs] @staticmethod def write_rois( rois: BaseRois, folder_path: str | Path, overwrite: bool = False, storage_options: dict | None = None, **kwargs, ): """Write ROI data to a Zarr store and return a reloaded ROIs. Parameters ---------- rois : BaseRois ROIs instance to write. folder_path : str or pathlib.Path Output path for the Zarr store. overwrite : bool, default: False If ``True``, overwrite an existing store at ``folder_path``. storage_options : dict or None, default: None Optional storage options passed to Zarr I/O. **kwargs Additional dataset creation options forwarded to :func:`add_rois_to_zarr_group` (for example, compression/chunking options). Returns ------- ZarrRois A :class:`ZarrRois` instance opened from the written store. Notes ----- Implemented as a static method so it can be called by :meth:`BaseRois.save` without instantiating :class:`ZarrRois`. This is its only intended use. """ import zarr from spikeinterface.core.core_tools import retrieve_importing_provenance from .zarrrois import ZarrRois, add_rois_to_zarr_group zarr_path = create_zarr_path_for_write(folder_path, overwrite=overwrite) storage_options = storage_options zarr_root = zarr.open(str(zarr_path), mode="w", storage_options=storage_options) # Lets spikeinterface's own read_zarr() (called by BaseExtractor.save_to_zarr() # right after this) reconstruct a ZarrRois directly, instead of falling back to its # channel_ids/unit_ids recording/sorting check, which ROI data doesn't match. zarr_root.attrs["zarr_class_info"] = retrieve_importing_provenance(ZarrRois) rois_group = zarr_root.create_group("rois") add_rois_to_zarr_group(rois, rois_group, **kwargs) zarr.consolidate_metadata(zarr_root.store) return ZarrRois(zarr_path, storage_options=storage_options)
[docs] def add_rois_to_zarr_group(rois: BaseRois, zarr_group, **kwargs) -> None: """Save ROI masks and metadata to a zarr group. Parameters ---------- rois : BaseRois The ROIs object to save. zarr_group : zarr.hierarchy.Group The zarr group to write to. **kwargs : Additional kwargs Additional zarr dataset creation options (e.g., compressor). """ image_masks = rois.get_roi_image_masks() if isinstance(image_masks, sparse.SparseArray): # zarr can't store a sparse array as a dataset value directly. Persist GCXS's own # (indptr, indices, data) components rather than round-tripping through COO: indptr # already marks each ROI's own boundary in the flat indices/data arrays (the same # CSR-style structure get_roi_image_masks needs to load a handful of ROIs without # reading every ROI's mask -- see ZarrRois._get_sparse_roi_image_masks). if isinstance(image_masks, sparse.GCXS): gcxs = image_masks if image_masks.compressed_axes == (0,) else image_masks.change_compressed_axes((0,)) else: gcxs = sparse.GCXS.from_coo(image_masks.tocoo(), compressed_axes=(0,)) zarr_group.attrs["roi_image_masks_sparse"] = True zarr_group.attrs["roi_image_masks_shape"] = list(gcxs.shape) zarr_group.create_dataset("roi_image_masks_indptr", data=gcxs.indptr, compressor=None) # Chunk indices/data to roughly a handful of ROIs' worth of entries, unless the caller # has already specified a chunk layout. zarr's own auto-chunking picks a chunk size # based on total array size with no notion of our per-ROI access pattern (indptr), so # a single-ROI request can still force decompressing a chunk sized for hundreds of # ROIs -- confirmed empirically to scale peak memory with total ROI count otherwise. sparse_saving_options = kwargs if "chunks" not in kwargs: avg_roi_nnz = max(1, gcxs.nnz // max(gcxs.shape[0], 1)) entries_per_chunk = max(8 * avg_roi_nnz, 1024) sparse_saving_options = {**kwargs, "chunks": (entries_per_chunk,)} zarr_group.create_dataset("roi_image_masks_indices", data=gcxs.indices, **sparse_saving_options) zarr_group.create_dataset("roi_image_masks_data", data=gcxs.data, **sparse_saving_options) else: zarr_group.attrs["roi_image_masks_sparse"] = False # Chunk along the ROI axis (first dimension) for efficient per-ROI access, # unless the caller has already specified a chunk layout. if "chunks" not in kwargs: roi_chunks = (1,) + image_masks.shape[1:] kwargs = {**kwargs, "chunks": roi_chunks} zarr_group.create_dataset("roi_image_masks", data=image_masks, **kwargs) roi_ids = np.array(rois.roi_ids) if roi_ids.dtype.kind == "U": import numcodecs zarr_group.create_dataset("roi_ids", data=roi_ids.astype(object), object_codec=numcodecs.JSON()) else: zarr_group.create_dataset("roi_ids", data=roi_ids, compressor=None) zarr_group.attrs["sampling_frequency"] = float(rois.sampling_frequency) zarr_group.attrs["shape"] = list(rois.shape) # Save properties prop_group = zarr_group.create_group("properties") for key in rois.get_property_keys(): values = rois.get_property(key) if values.dtype.kind == "O": continue # skip non-serializable object-dtype properties prop_group.create_dataset(key, data=values, compressor=None) # Save annotations annotations = rois.get_annotation_keys() if annotations: ann_dict = {key: rois.get_annotation(key) for key in annotations} zarr_group.attrs["annotations"] = ann_dict