Source code for photon_mosaic.core.zarrimaging

from pathlib import Path

import numpy as np
import zarr
from spikeinterface.core.core_tools import check_json, retrieve_importing_provenance
from spikeinterface.core.job_tools import split_job_kwargs
from spikeinterface.core.time_series_tools import _write_time_series_to_zarr
from spikeinterface.core.zarrextractors import (
    add_properties_and_annotations,
    get_default_zarr_compressor,
    resolve_zarr_path,
    super_zarr_open,
)

from photon_mosaic.core.baseimaging import BaseImaging, BaseImagingEpoch


[docs] class ZarrImaging(BaseImaging): """An Imaging object that is backed by a zarr array on disk. This allows for memory efficient access to large imaging datasets that do not fit in memory. Parameters ---------- folder_path: str | Path The path to the folder containing the zarr group with the imaging data. storage_options: dict | None Optional storage options for opening the zarr group. """ def __init__( self, folder_path: str | Path, storage_options: dict | None = None, load_compression_ratio: bool = False ): folder_path, folder_path_kwarg = resolve_zarr_path(folder_path) # super_zarr_open is a helper function that deals with consolidated/non-consolidated stores # and anonymous/non-anonymous cloud access self._root = super_zarr_open(folder_path, mode="r", storage_options=storage_options) sampling_frequency = self._root.attrs.get("sampling_frequency", None) num_epochs = self._root.attrs.get("num_epochs", None) assert "shape" in self._root.keys(), "'shape' dataset not found!" shapes = self._root["shape"][:] assert sampling_frequency is not None, "'sampling_frequency' attribute not found!" assert num_epochs is not None, "'num_epochs' attribute not found!" BaseImaging.__init__( self, sampling_frequency=sampling_frequency, shape=shapes, dtype=self._root["video_epoch0"].dtype ) t_starts = self._root.get("t_starts", None) if load_compression_ratio: total_nbytes = 0 total_nbytes_stored = 0 cr_by_epoch = {} for epoch_index in range(num_epochs): video_name = f"video_epoch{epoch_index}" time_kwargs = {} time_vector = self._root.get(f"times_epoch{epoch_index}", None) if time_vector is not None: time_kwargs["time_vector"] = time_vector else: if t_starts is None: t_start = None else: t_start = t_starts[epoch_index] if np.isnan(t_start): t_start = None # pragma: no cover time_kwargs["t_start"] = t_start time_kwargs["sampling_frequency"] = sampling_frequency epoch = ZarrImagingEpoch(self._root, video_name, **time_kwargs) self.add_epoch(epoch) if load_compression_ratio: nbytes_epoch = self._root[video_name].nbytes nbytes_stored_epoch = self._root[video_name].nbytes_stored if nbytes_stored_epoch > 0: cr_by_epoch[epoch_index] = nbytes_epoch / nbytes_stored_epoch else: cr_by_epoch[epoch_index] = np.nan # pragma: no cover total_nbytes += nbytes_epoch total_nbytes_stored += nbytes_stored_epoch # load properties if "properties" in self._root: prop_group = self._root["properties"] for key in prop_group.keys(): values = self._root["properties"][key] self.set_property(key, values) # load annotations annotations = self._root.attrs.get("annotations", None) if annotations is not None: self.annotate(**annotations) if load_compression_ratio: # annotate compression ratios if total_nbytes_stored > 0: cr = total_nbytes / total_nbytes_stored else: cr = np.nan # pragma: no cover self.annotate(compression_ratio=cr, compression_ratio_epochs=cr_by_epoch) self._kwargs = { "folder_path": folder_path_kwarg, "storage_options": storage_options, "load_compression_ratio": load_compression_ratio, }
[docs] @staticmethod def write_imaging(imaging: BaseImaging, folder_path: str, verbose=False, **kwargs): """ Write imaging data to a Zarr folder. Parameters ---------- imaging : BaseImaging Imaging object to write. folder_path : str Destination folder path for the Zarr store. verbose : bool, default: False Whether to print verbose output during writing. **kwargs Additional keyword arguments, including ``storage_options``. Returns ------- ZarrImaging A :class:`ZarrImaging` instance opened from ``folder_path``. Notes ----- Implemented as a static method so it can be called by :meth:`BaseImaging.save` without instantiating :class:`ZarrImaging`. This is its only intended use. """ import zarr storage_options = kwargs.get("storage_options", None) zarr_root = zarr.open(str(folder_path), mode="w", storage_options=storage_options) add_imaging_to_zarr_group(imaging, zarr_root, verbose=verbose, **kwargs) return ZarrImaging(folder_path=folder_path)
[docs] class ZarrImagingEpoch(BaseImagingEpoch): def __init__(self, root, dataset_name, **time_kwargs): BaseImagingEpoch.__init__(self, **time_kwargs) self._video = root[dataset_name]
[docs] def get_num_samples(self) -> int: """Returns the number of samples in this signal block Returns: SampleIndex : Number of samples in the signal block """ return self._video.shape[0]
[docs] def get_series( self, start_frame: int, end_frame: int, plane_indices: slice | np.ndarray | None = None, ) -> np.ndarray: if plane_indices is None: plane_indices = slice(None) video = self._video[start_frame:end_frame, :, :, plane_indices] return np.asarray(video)
[docs] def add_imaging_to_zarr_group( imaging: BaseImaging, zarr_group: zarr.hierarchy.Group, verbose=False, dtype=None, **kwargs ): """Adds an Imaging object to a zarr group. Parameters ---------- imaging: BaseImaging The Imaging object to add to the zarr group. zarr_group: zarr.hierarchy.Group The zarr group to which the imaging data should be added. verbose: bool Whether to print verbose output during the writing process. dtype: np.dtype | None The dtype to use for the video datasets. If None, the dtype of the imaging data will be used. **kwargs: Additional keyword arguments to pass to the _write_time_series_to_zarr function. This can include zarr-specific arguments (e.g., compressor, filters) as well as job-related arguments (e.g., n_jobs). """ zarr_kwargs, job_kwargs = split_job_kwargs(kwargs) if imaging.check_serializability("json"): zarr_group.attrs["provenance"] = check_json(imaging.to_dict(recursive=True)) else: zarr_group.attrs["provenance"] = None # pragma: no cover # save data (done the subclass) zarr_group.attrs["sampling_frequency"] = float(imaging.get_sampling_frequency()) zarr_group.attrs["num_epochs"] = int(imaging.get_num_epochs()) zarr_group.create_dataset(name="shape", data=imaging.shape, compressor=None) dataset_paths = [f"video_epoch{i}" for i in range(imaging.get_num_epochs())] dataset_timestamps_paths: list | None = None if any(imaging.has_time_vector(i) for i in range(imaging.get_num_epochs())): dataset_timestamps_paths = [] for i in range(imaging.get_num_epochs()): if imaging.has_time_vector(i): dataset_timestamps_paths.append(f"times_epoch{i}") else: dataset_timestamps_paths.append(None) dtype = imaging.get_dtype() if dtype is None else dtype extra_chunks = zarr_kwargs.get("extra_chunks", None) global_compressor = zarr_kwargs.pop("compressor", get_default_zarr_compressor()) compressor_by_dataset = zarr_kwargs.pop("compressor_by_dataset", {}) global_filters = zarr_kwargs.pop("filters", None) filters_by_dataset = zarr_kwargs.pop("filters_by_dataset", {}) compressor_videos = compressor_by_dataset.get("videos", global_compressor) filters_videos = filters_by_dataset.get("videos", global_filters) compressor_times = compressor_by_dataset.get("times", global_compressor) filters_times = filters_by_dataset.get("times", global_filters) _write_time_series_to_zarr( time_series=imaging, zarr_group=zarr_group, dataset_paths=dataset_paths, dataset_timestamps_paths=dataset_timestamps_paths, compressor_data=compressor_videos, filters_data=filters_videos, compressor_times=compressor_times, filters_times=filters_times, dtype=dtype, extra_chunks=extra_chunks, verbose=verbose, **job_kwargs, ) add_properties_and_annotations(zarr_group, imaging) zarr_group.attrs["zarr_class_info"] = retrieve_importing_provenance(ZarrImaging)