Source code for photon_mosaic.core.roianalyzer

"""
RoiAnalyzer: core class for pairing BaseRois and BaseImaging for analysis.

Adapted from SpikeInterface's SortingAnalyzer to work with photon-mosaic's
BaseRois and BaseImaging instead of BaseSorting and BaseRecording.
"""

import importlib
import json
import os
import pickle
import shutil
import warnings
import weakref
from copy import copy
from itertools import chain
from pathlib import Path
from time import perf_counter
from typing import Any, Literal

import numpy as np
from spikeinterface.core.core_tools import (
    check_json,
    clean_zarr_folder_name,
    is_path_remote,
    retrieve_importing_provenance,
)
from spikeinterface.core.job_tools import split_job_kwargs

import photon_mosaic

from .baseimaging import BaseImaging
from .baserois import BaseRois
from .binaryrois import BinaryFolderRois
from .imaging_tools import do_imaging_attributes_match, get_imaging_attributes
from .numpyimaging import NumpyRois
from .zarrrois import ZarrRois, add_rois_to_zarr_group

# ---------------------------------------------------------------------------
# High-level factory functions
# ---------------------------------------------------------------------------


[docs] def create_roi_analyzer( rois: BaseRois, imaging: BaseImaging, format: str = "memory", folder=None, overwrite: bool = False, backend_options: dict | None = None, ) -> "RoiAnalyzer": """Create a RoiAnalyzer by pairing a BaseRois and a BaseImaging. The RoiAnalyzer manages a collection of AnalyzerExtension instances for post-processing steps such as trace extraction, dF/F, deconvolution, etc. Parameters ---------- rois : BaseRois The ROIs object. imaging : BaseImaging The imaging object. format : "memory" | "binary_folder" | "zarr", default: "memory" The storage backend. If "binary_folder" or "zarr", ``folder`` must be provided. folder : str | Path | None, default: None The folder where the analyzer is persisted (required for non-memory formats). overwrite : bool, default: False If True, overwrite the folder if it already exists. backend_options : dict | None, default: None Backend-specific options. May contain: * storage_options: dict | None (fsspec storage options) * saving_options: dict | None (e.g. compression for zarr) Returns ------- RoiAnalyzer The created RoiAnalyzer object. """ if format != "memory" and not is_path_remote(folder): folder = clean_zarr_folder_name(folder) if format == "zarr" else folder if Path(folder).is_dir(): if overwrite: shutil.rmtree(folder) else: raise ValueError(f"Folder {folder} already exists! Use overwrite=True to overwrite it.") roi_analyzer = RoiAnalyzer.create( rois, imaging, format=format, # type: ignore[arg-type] folder=folder, backend_options=backend_options, ) return roi_analyzer
[docs] def load_roi_analyzer( folder, load_extensions: bool = True, format: str = "auto", backend_options: dict | None = None ) -> "RoiAnalyzer": """Load a RoiAnalyzer from disk. Parameters ---------- folder : str or Path The folder where the analyzer is stored. load_extensions : bool, default: True Whether to load all saved extensions. format : "auto" | "binary_folder" | "zarr", default: "auto" The format of the stored analyzer. "auto" will guess from the path. backend_options : dict | None, default: None Backend-specific options (e.g. storage_options for remote zarr). Returns ------- RoiAnalyzer The loaded RoiAnalyzer. """ return RoiAnalyzer.load(folder, load_extensions=load_extensions, format=format, backend_options=backend_options)
# --------------------------------------------------------------------------- # RoiAnalyzer # ---------------------------------------------------------------------------
[docs] class RoiAnalyzer: """Pair BaseRois and BaseImaging for extensible post-processing analysis. Maintains a collection of computed AnalyzerExtension instances. Supports three storage backends: ``"memory"``, ``"binary_folder"``, and ``"zarr"``. Do not instantiate directly — use :func:`create_roi_analyzer` or :meth:`RoiAnalyzer.create`. """ def __init__( self, rois: BaseRois, imaging: BaseImaging | None = None, imaging_attributes: dict | None = None, format: str | None = None, backend_options: dict | None = None, ): # Fast init — validation is done in create / load self.rois = rois self._imaging = imaging self.imaging_attributes = imaging_attributes self.format = format self.folder: str | Path | None = None self._temporary_imaging: BaseImaging | None = None self._backend_options = {} if backend_options is None else backend_options # Extensions are not loaded at init self.extensions: dict[str, AnalyzerExtension] = dict() def __repr__(self) -> str: cls_name = self.__class__.__name__ n_rois = self.get_num_rois() n_epochs = self.get_num_epochs() assert self.imaging_attributes is not None shape = tuple(self.imaging_attributes["shape"]) txt = ( f"{cls_name}: {n_rois} ROIs - {shape[0]}x{shape[1]} ({shape[2]} planes) - {n_epochs} epochs - {self.format}" ) if self.format != "memory": if is_path_remote(str(self.folder)): txt += " (remote)" if self.has_imaging(): txt += " - has imaging" if self.has_temporary_imaging(): txt += " - has temporary imaging" ext_txt = f"Loaded {len(self.extensions)} extensions" if len(self.extensions) > 0: ext_txt += f": {', '.join(self.extensions.keys())}" txt += "\n" + ext_txt return txt # ------------------------------------------------------------------ # Create / load # ------------------------------------------------------------------
[docs] @classmethod def create( cls, rois: BaseRois, imaging: BaseImaging, format: Literal["memory", "binary_folder", "zarr"] = "memory", folder=None, backend_options: dict | None = None, ): """Create a new RoiAnalyzer. Parameters ---------- rois : BaseRois The ROIs object. imaging : BaseImaging The imaging object. format : str Storage backend. folder : str | Path | None Required for non-memory formats. backend_options : dict | None Backend-specific options. Returns ------- RoiAnalyzer """ assert imaging is not None, "An imaging object must be provided to create a RoiAnalyzer" # Validate compatibility if not np.isclose(rois.sampling_frequency, imaging.sampling_frequency, atol=0.1): raise ValueError( f"Sampling frequency mismatch: rois={rois.sampling_frequency}, " f"imaging={imaging.sampling_frequency}" ) if rois.shape[:2] != imaging.shape[:2]: raise ValueError(f"Spatial shape mismatch: rois={rois.shape[:2]}, imaging={imaging.shape[:2]}") if rois.num_planes != imaging.num_planes: raise ValueError(f"Number of planes mismatch: rois={rois.num_planes}, imaging={imaging.num_planes}") if format == "memory": return cls.create_memory(rois, imaging, imaging_attributes=None) elif format == "binary_folder": return cls.create_binary_folder( folder, rois, imaging, imaging_attributes=None, backend_options=backend_options ) elif format == "zarr": assert folder is not None, "For format='zarr' folder must be provided" if not is_path_remote(folder): folder = clean_zarr_folder_name(folder) return cls.create_zarr(folder, rois, imaging, imaging_attributes=None, backend_options=backend_options) else: raise ValueError(f"Unknown format: {format}")
[docs] @classmethod def load(cls, folder, imaging=None, load_extensions=True, format="auto", backend_options=None): """Load from folder or zarr. The imaging can be provided if its location has changed. """ if format == "auto": if str(folder).endswith(".zarr"): format = "zarr" else: format = "binary_folder" if format == "binary_folder": analyzer = cls.load_from_binary_folder(folder, imaging=imaging, backend_options=backend_options) elif format == "zarr": analyzer = cls.load_from_zarr(folder, imaging=imaging, backend_options=backend_options) else: raise ValueError(f"Unknown format: {format}") if not is_path_remote(str(folder)): if load_extensions: analyzer.load_all_saved_extension() return analyzer
# ------------------------------------------------------------------ # Memory backend # ------------------------------------------------------------------
[docs] @classmethod def create_memory(cls, rois, imaging, imaging_attributes): if imaging_attributes is None: assert imaging is not None imaging_attributes = get_imaging_attributes(imaging) else: imaging_attributes = imaging_attributes.copy() # Make an in-memory copy of rois for fast access. Pass image masks through as-is # (dense ndarray or sparse array) -- wrapping in np.array() would raise for a sparse # array (which refuses implicit densification via __array__) instead of copying it. rois_copy = NumpyRois( roi_image_masks=rois.get_roi_image_masks(), sampling_frequency=rois.sampling_frequency, roi_ids=np.array(rois.roi_ids), ) rois.copy_metadata(rois_copy, only_main=False, ids=rois_copy.roi_ids) analyzer = RoiAnalyzer( rois=rois_copy, imaging=imaging, imaging_attributes=imaging_attributes, format="memory", ) return analyzer
# ------------------------------------------------------------------ # Binary folder backend # ------------------------------------------------------------------
[docs] @classmethod def create_binary_folder(cls, folder, rois, imaging, imaging_attributes, backend_options): folder = Path(folder) if folder.is_dir(): raise ValueError(f"Folder already exists: {folder}") folder.mkdir(parents=True) # photon-mosaic info info = dict( version=photon_mosaic.__version__, object="RoiAnalyzer", ) with open(folder / "photon_mosaic_info.json", "w") as f: json.dump(check_json(info), f, indent=4) # Save a copy of the rois rois.save(format="binary", folder=folder / "rois") # Save imaging provenance if imaging is not None: if imaging.check_serializability("json"): imaging.dump(folder / "imaging.json", relative_to=folder) elif imaging.check_serializability("pickle"): imaging.dump(folder / "imaging.pickle", relative_to=folder) else: warnings.warn("The imaging is not serializable. The imaging link will be lost on reload.") # Save rois provenance if rois.check_serializability("json"): rois.dump(folder / "rois_provenance.json", relative_to=folder) elif rois.check_serializability("pickle"): rois.dump(folder / "rois_provenance.pickle", relative_to=folder) else: warnings.warn("The rois provenance is not serializable. The link will be lost on reload.") # Imaging attributes attr_folder = folder / "imaging_info" attr_folder.mkdir() if imaging_attributes is None: imaging_attributes = get_imaging_attributes(imaging) attr_file = attr_folder / "imaging_attributes.json" attr_file.write_text(json.dumps(check_json(imaging_attributes), indent=4), encoding="utf8") # Create extensions folder (folder / "extensions").mkdir() return cls.load_from_binary_folder(folder, imaging=imaging, backend_options=backend_options)
[docs] @classmethod def load_from_binary_folder(cls, folder, imaging=None, backend_options=None): from spikeinterface.core.loading import load as si_load folder = Path(folder) assert folder.is_dir(), f"Folder does not exist: {folder}" # Load rois rois = BinaryFolderRois(folder_path=folder / "rois") # Try to load the imaging if not provided if imaging is None: for ext in ("json", "pickle"): fname = folder / f"imaging.{ext}" if fname.exists(): try: imaging = si_load(fname, base_folder=folder) break except Exception: pass # Imaging attributes attr_file = folder / "imaging_info" / "imaging_attributes.json" if not attr_file.exists(): raise ValueError(f"Not a valid RoiAnalyzer binary_folder: {folder}") with open(attr_file, "r") as f: imaging_attributes = json.load(f) analyzer = RoiAnalyzer( rois=rois, imaging=imaging, imaging_attributes=imaging_attributes, format="binary_folder", backend_options=backend_options, ) analyzer.folder = folder return analyzer
# ------------------------------------------------------------------ # Zarr backend # ------------------------------------------------------------------ def _get_zarr_root(self, mode="r+"): from spikeinterface.core.zarrextractors import super_zarr_open storage_options = self._backend_options.get("storage_options", {}) return super_zarr_open(str(self.folder), mode=mode, storage_options=storage_options)
[docs] @classmethod def create_zarr(cls, folder, rois, imaging, imaging_attributes, backend_options): import numcodecs import zarr backend_options = backend_options or {} storage_options = backend_options.get("storage_options", {}) saving_options = backend_options.get("saving_options", {}) remote = is_path_remote(str(folder)) if not remote: folder = clean_zarr_folder_name(folder) if Path(folder).is_dir(): raise ValueError(f"Folder already exists: {folder}") zarr_root = zarr.open(str(folder), mode="w", storage_options=storage_options) # Info info = dict( version=photon_mosaic.__version__, object="RoiAnalyzer", ) zarr_root.attrs["photon_mosaic_info"] = check_json(info) # Save imaging provenance relative_to = folder if not remote else None if imaging is not None: rec_dict = imaging.to_dict(relative_to=relative_to, recursive=True) if imaging.check_serializability("json"): zarr_rec = np.array([check_json(rec_dict)], dtype=object) zarr_root.create_dataset("imaging", data=zarr_rec, object_codec=numcodecs.JSON()) elif imaging.check_serializability("pickle"): zarr_rec = np.array([rec_dict], dtype=object) zarr_root.create_dataset("imaging", data=zarr_rec, object_codec=numcodecs.Pickle()) else: warnings.warn("The imaging is not serializable. The link will be lost on reload.") # Save rois provenance rois_dict = rois.to_dict(relative_to=relative_to, recursive=True) if rois.check_serializability("json"): zarr_rois = np.array([check_json(rois_dict)], dtype=object) zarr_root.create_dataset("rois_provenance", data=zarr_rois, object_codec=numcodecs.JSON()) elif rois.check_serializability("pickle"): zarr_rois = np.array([rois_dict], dtype=object) zarr_root.create_dataset("rois_provenance", data=zarr_rois, object_codec=numcodecs.Pickle()) else: warnings.warn("The rois provenance is not serializable. The link will be lost on reload.") # Save imaging attributes if imaging_attributes is None: imaging_attributes = get_imaging_attributes(imaging) imaging_info = zarr_root.create_group("imaging_info") imaging_info.attrs["imaging_attributes"] = check_json(imaging_attributes) # Save rois data rois_group = zarr_root.create_group("rois") add_rois_to_zarr_group(rois, rois_group, saving_options=saving_options) # Extensions group zarr_root.create_group("extensions") zarr.consolidate_metadata(zarr_root.store) return cls.load_from_zarr(folder, imaging=imaging, backend_options=backend_options)
[docs] @classmethod def load_from_zarr(cls, folder, imaging=None, backend_options=None): from spikeinterface.core.loading import load as si_load from spikeinterface.core.zarrextractors import super_zarr_open backend_options = backend_options or {} storage_options = backend_options.get("storage_options", {}) zarr_root = super_zarr_open(str(folder), mode="r", storage_options=storage_options) # Load rois rois = ZarrRois(folder, zarr_group_name="rois", storage_options=storage_options) # Load imaging if possible if imaging is None: rec_field = zarr_root.get("imaging") if rec_field is not None: rec_dict = rec_field[0] try: imaging = si_load(rec_dict, base_folder=folder) except Exception: imaging = None # Imaging attributes imaging_attributes = zarr_root["imaging_info"].attrs["imaging_attributes"] analyzer = RoiAnalyzer( rois=rois, imaging=imaging, imaging_attributes=imaging_attributes, format="zarr", backend_options=backend_options, ) analyzer.folder = folder return analyzer
# ------------------------------------------------------------------ # Temporary imaging # ------------------------------------------------------------------
[docs] def set_temporary_imaging(self, imaging: BaseImaging, check_dtype: bool = True): """Set a temporary imaging (e.g. a cached/loaded copy) without persisting it. Parameters ---------- imaging : BaseImaging The imaging object. check_dtype : bool, default: True Whether to validate dtype match. """ assert self.imaging_attributes is not None matches, error = do_imaging_attributes_match(imaging, self.imaging_attributes, check_dtype=check_dtype) if not matches: raise ValueError(error) if self._imaging is not None: warnings.warn("RoiAnalyzer imaging is already set. Temporarily replacing it.") self._temporary_imaging = imaging
# ------------------------------------------------------------------ # ROI property helpers # ------------------------------------------------------------------
[docs] def set_roi_property( self, key: str, values: list | np.ndarray | tuple, ids: list | np.ndarray | tuple | None = None, missing_value: Any = None, save: bool = True, ) -> None: """Set a property on the internal ROIs copy, optionally persisting to disk. Parameters ---------- key : str Property name. values : array-like Property values. ids : array-like | None Subset of ROI ids to set. If None, set for all. missing_value : Any Fill value for ids not in ``ids``. save : bool, default: True Whether to persist (binary_folder / zarr only). """ self.rois.set_property(key, values, ids=ids, missing_value=missing_value) if not self.is_read_only() and save: if self.format == "binary_folder": assert isinstance(self.folder, Path) props_dir = self.folder / "rois" / "properties" props_dir.mkdir(exist_ok=True) np.save(props_dir / f"{key}.npy", self.rois.get_property(key)) elif self.format == "zarr": import zarr zarr_root = self._get_zarr_root(mode="r+") prop_values = self.rois.get_property(key) if prop_values.dtype.kind == "O": warnings.warn(f"Property '{key}' not saved (object dtype)") else: rois_group = zarr_root["rois"] if "properties" not in rois_group: rois_group.create_group("properties") props_group = rois_group["properties"] if key in props_group: props_group[key][:] = prop_values else: props_group.create_dataset(name=key, data=prop_values, compressor=None) zarr.consolidate_metadata(zarr_root.store)
[docs] def get_roi_property(self, key: str, ids=None) -> np.ndarray: """Get a property from the internal ROIs copy.""" return self.rois.get_property(key, ids=ids)
# ------------------------------------------------------------------ # Save / select / copy # ------------------------------------------------------------------ def _save_or_select( self, format="binary_folder", folder=None, roi_ids=None, backend_options=None, verbose=False, ) -> "RoiAnalyzer": """Internal method used by save_as(), copy(), select_rois(), remove_rois(). Parameters ---------- format : str Target backend. folder : str | Path | None Target folder (for non-memory). roi_ids : array-like | None Subset of roi ids to keep. None = keep all. backend_options : dict | None Backend-specific options. verbose : bool Whether to print progress. Returns ------- RoiAnalyzer """ if self.has_imaging(): imaging = self._imaging elif self.has_temporary_imaging(): imaging = self._temporary_imaging else: imaging = None # Get the original rois provenance (or fall back to internal copy) rois_provenance = self.get_rois_provenance() if rois_provenance is None: rois_provenance = self.rois # Copy any in-memory properties added after creation for key in self.rois.get_property_keys(): if key not in rois_provenance.get_property_keys(): rois_provenance.set_property(key, self.rois.get_property(key)) if roi_ids is not None: rois_provenance = rois_provenance.select_rois(roi_ids) backend_options = backend_options or {} if format == "memory": new_analyzer = RoiAnalyzer.create_memory(rois_provenance, imaging, self.imaging_attributes) elif format == "binary_folder": assert folder is not None, "folder must be provided for binary_folder" new_analyzer = RoiAnalyzer.create_binary_folder( Path(folder), rois_provenance, imaging, self.imaging_attributes, backend_options ) elif format == "zarr": assert folder is not None, "folder must be provided for zarr" folder = clean_zarr_folder_name(folder) new_analyzer = RoiAnalyzer.create_zarr( folder, rois_provenance, imaging, self.imaging_attributes, backend_options ) else: raise ValueError(f"Unsupported format: {format}") # Propagate extensions extensions_to_compute = _sort_extensions_by_dependency( {ext.extension_name: ext.params for ext in self.extensions.values()} ) for extension_name in extensions_to_compute: extension = self.extensions[extension_name] new_analyzer.extensions[extension_name] = extension.copy(new_analyzer, roi_ids=roi_ids) return new_analyzer
[docs] def save_as(self, format="memory", folder=None, backend_options=None) -> "RoiAnalyzer": """Save the analyzer to a different backend. Parameters ---------- format : "memory" | "binary_folder" | "zarr" Target backend. folder : str | Path | None Required for non-memory formats. backend_options : dict | None Backend-specific options. Returns ------- RoiAnalyzer """ if format == "zarr": folder = clean_zarr_folder_name(folder) return self._save_or_select(format=format, folder=folder, backend_options=backend_options)
[docs] def select_rois(self, roi_ids, format="memory", folder=None) -> "RoiAnalyzer": """Create a new RoiAnalyzer with a subset of ROIs. Extensions are filtered to the selected ROI ids. Parameters ---------- roi_ids : array-like ROI ids to keep. format : str Target backend. folder : str | Path | None Required for non-memory formats. Returns ------- RoiAnalyzer """ if format == "zarr": folder = clean_zarr_folder_name(folder) return self._save_or_select(format=format, folder=folder, roi_ids=roi_ids)
[docs] def remove_rois(self, remove_roi_ids, format="memory", folder=None) -> "RoiAnalyzer": """Create a new RoiAnalyzer with some ROIs removed. Parameters ---------- remove_roi_ids : array-like ROI ids to remove. format : str Target backend. folder : str | Path | None Required for non-memory formats. Returns ------- RoiAnalyzer """ roi_ids = self.roi_ids[~np.isin(self.roi_ids, remove_roi_ids)] if format == "zarr": folder = clean_zarr_folder_name(folder) return self._save_or_select(format=format, folder=folder, roi_ids=roi_ids)
[docs] def copy(self) -> "RoiAnalyzer": """Create an in-memory copy.""" return self._save_or_select(format="memory", folder=None)
# ------------------------------------------------------------------ # Properties / accessors # ------------------------------------------------------------------ @property def imaging(self) -> BaseImaging: if not self.has_imaging() and not self.has_temporary_imaging(): raise ValueError("RoiAnalyzer could not load the imaging") imaging = self._temporary_imaging or self._imaging assert imaging is not None return imaging @property def roi_ids(self) -> np.ndarray: return self.rois.roi_ids @property def sampling_frequency(self) -> float: assert self.imaging_attributes is not None return float(self.imaging_attributes["sampling_frequency"]) @property def shape(self) -> tuple: assert self.imaging_attributes is not None return tuple(self.imaging_attributes["shape"]) @property def num_planes(self) -> int: return self.shape[2]
[docs] def has_imaging(self) -> bool: return self._imaging is not None
[docs] def has_temporary_imaging(self) -> bool: return self._temporary_imaging is not None
[docs] def get_num_rois(self) -> int: return self.rois.get_num_rois()
[docs] def get_num_epochs(self) -> int: assert self.imaging_attributes is not None return self.imaging_attributes["num_epochs"]
[docs] def get_num_samples(self, epoch_index: int | None = None) -> int: if epoch_index is None: if self.get_num_epochs() == 1: epoch_index = 0 else: raise ValueError("epoch_index must be provided for multi-epoch data.") assert self.imaging_attributes is not None return self.imaging_attributes["num_samples"][epoch_index]
[docs] def get_total_samples(self) -> int: assert self.imaging_attributes is not None return sum(self.imaging_attributes["num_samples"])
[docs] def get_total_duration(self) -> float: if self.has_imaging() or self.has_temporary_imaging(): return self.imaging.get_total_duration() return self.get_total_samples() / self.sampling_frequency
[docs] def get_dtype(self): return np.dtype(self.imaging_attributes["dtype"])
[docs] def is_read_only(self) -> bool: if self.format == "memory": return False assert self.folder is not None if self.format == "binary_folder": return not os.access(self.folder, os.W_OK) else: if not is_path_remote(str(self.folder)): return not os.access(self.folder, os.W_OK) return False
[docs] def get_rois_provenance(self): """Get the original rois object if possible, otherwise None.""" from spikeinterface.core.loading import load as si_load if self.format == "memory": return None elif self.format == "binary_folder": assert isinstance(self.folder, Path) for ext in ("json", "pickle"): fname = self.folder / f"rois_provenance.{ext}" if fname.exists(): try: return si_load(fname, base_folder=self.folder) except Exception: pass return None elif self.format == "zarr": zarr_root = self._get_zarr_root(mode="r") if "rois_provenance" in zarr_root.keys(): try: rois_dict = zarr_root["rois_provenance"][0] return si_load(rois_dict, base_folder=self.folder) except Exception: pass return None
# ------------------------------------------------------------------ # Extension management # ------------------------------------------------------------------
[docs] def compute(self, input, save=True, extension_params=None, verbose=False, **kwargs) -> "AnalyzerExtension | None": """Compute one or several extensions. Parameters ---------- input : str | dict | list Extension name (str), dict of {name: params}, or list of names. save : bool, default: True Whether to persist computed extensions. extension_params : dict | None Per-extension params when ``input`` is a list. verbose : bool Print progress. **kwargs Passed to the extension's ``set_params`` (if str) or as job_kwargs. Returns ------- AnalyzerExtension | None The extension instance if input is a string, None otherwise. """ if isinstance(input, str): return self.compute_one_extension(extension_name=input, save=save, verbose=verbose, **kwargs) elif isinstance(input, dict): params_, job_kwargs = split_job_kwargs(kwargs) assert len(params_) == 0, f"Unexpected arguments: {set(params_)}" self.compute_several_extensions(extensions=input, save=save, verbose=verbose, **job_kwargs) elif isinstance(input, list): params_, job_kwargs = split_job_kwargs(kwargs) assert len(params_) == 0, f"Unexpected arguments: {set(params_)}" extensions: dict[str, dict[str, Any]] = {k: {} for k in input} if extension_params is not None: for name, params in extension_params.items(): assert name in input, f"Extension '{name}' not in input list" extensions[name] = params self.compute_several_extensions(extensions=extensions, save=save, verbose=verbose, **job_kwargs) else: raise ValueError("compute() expects a str, dict, or list") return None
[docs] def compute_one_extension(self, extension_name, save=True, verbose=False, **kwargs) -> "AnalyzerExtension": """Compute a single extension. Automatically deletes dependent extensions to keep data coherent. """ extension_class = get_extension_class(extension_name) # Delete children that depend on this extension for child in _get_children_dependencies(extension_name, self._get_extensions_params()): if self.has_extension(child): if verbose: print(f"Deleting extension: {child}") self.delete_extension(child) params, job_kwargs = split_job_kwargs(kwargs) # Check dependencies if extension_class.need_imaging: assert ( self.has_imaging() or self.has_temporary_imaging() ), f"Extension '{extension_name}' requires the imaging" for dep in extension_class.get_required_dependencies(**params): if "|" in dep: ok = any(self.get_extension(d) is not None for d in dep.split("|")) else: ok = self.get_extension(dep) is not None assert ok, f"Extension '{extension_name}' requires '{dep}' to be computed first" ext_instance = extension_class(self) ext_instance.set_params(save=save, **params) if extension_class.need_job_kwargs: ext_instance.run(save=save, verbose=verbose, **job_kwargs) else: ext_instance.run(save=save, verbose=verbose) self.extensions[extension_name] = ext_instance return ext_instance
[docs] def compute_several_extensions(self, extensions, save=True, verbose=False, **job_kwargs): """Compute several extensions respecting dependency order. Parameters ---------- extensions : dict Mapping of extension_name -> params dict. save : bool Whether to persist. verbose : bool Print progress. **job_kwargs Parallelization kwargs. """ # Validate dependencies ext_names = list(extensions.keys()) for name, params in extensions.items(): for dep in get_extension_class(name).get_required_dependencies(**params): if "|" in dep: ok = any(self.has_extension(d) or d in ext_names for d in dep.split("|")) else: ok = self.has_extension(dep) or dep in ext_names assert ok, f"Extension '{name}' requires '{dep}' to be computed first" sorted_exts = _sort_extensions_by_dependency(extensions) # Delete children of all extensions we're about to recompute all_params = {**self._get_extensions_params(), **extensions} for name in sorted_exts: for child in _get_children_dependencies(name, all_params): if verbose: print(f"Deleting extension: {child}") self.delete_extension(child) # Group: pipeline vs non-pipeline pipeline_exts = {} pre_pipeline_exts = {} post_pipeline_exts = {} for name, params in sorted_exts.items(): ext_class = get_extension_class(name) if ext_class.use_nodepipeline: pipeline_exts[name] = params elif any( get_extension_class(d).use_nodepipeline for d in ext_class.get_any_dependencies(**params) if d in sorted_exts ): post_pipeline_exts[name] = params else: pre_pipeline_exts[name] = params # Pre-pipeline for name, params in pre_pipeline_exts.items(): ext_class = get_extension_class(name) if ext_class.need_job_kwargs: self.compute_one_extension(name, save=save, verbose=verbose, **params, **job_kwargs) else: self.compute_one_extension(name, save=save, verbose=verbose, **params) # Pipeline extensions (run together via node pipeline) if len(pipeline_exts) > 0: from spikeinterface.core.node_pipeline import run_node_pipeline all_nodes = [] result_routage = [] instances = {} for name, params in pipeline_exts.items(): ext_class = get_extension_class(name) assert self.has_imaging() or self.has_temporary_imaging(), f"Extension '{name}' requires the imaging" for var in ext_class.nodepipeline_variables or []: result_routage.append((name, var)) inst = ext_class(self) inst.set_params(save=save, **params) instances[name] = inst all_nodes.extend(inst.get_pipeline_nodes()) job_name = "Compute: " + " + ".join(pipeline_exts.keys()) t0 = perf_counter() results = run_node_pipeline( self.imaging, all_nodes, job_kwargs=job_kwargs, job_name=job_name, gather_mode="memory", squeeze_output=False, verbose=verbose, check_for_peak_source=False, ) runtime_s = perf_counter() - t0 for r, result in enumerate(results): ext_name, var_name = result_routage[r] instances[ext_name].data[var_name] = result instances[ext_name].run_info["runtime_s"] = runtime_s instances[ext_name].run_info["run_completed"] = True for name, inst in instances.items(): self.extensions[name] = inst if save: inst.save() # Post-pipeline for name, params in post_pipeline_exts.items(): ext_class = get_extension_class(name) if ext_class.need_job_kwargs: self.compute_one_extension(name, save=save, verbose=verbose, **params, **job_kwargs) else: self.compute_one_extension(name, save=save, verbose=verbose, **params)
[docs] def get_saved_extension_names(self) -> list[str]: """Get extension names saved on disk (without loading data).""" saved = [] if self.format == "binary_folder": assert isinstance(self.folder, Path) ext_folder = self.folder / "extensions" if ext_folder.is_dir(): for d in ext_folder.iterdir(): if d.is_dir() and (d / "params.json").is_file(): saved.append(d.stem) elif self.format == "zarr": zarr_root = self._get_zarr_root(mode="r") if "extensions" in zarr_root.keys(): for name in zarr_root["extensions"].keys(): if "params" in zarr_root["extensions"][name].attrs.keys(): saved.append(name) return saved
[docs] def get_extension(self, extension_name: str) -> "AnalyzerExtension | None": """Get an extension, auto-loading from disk if needed. Returns None if not computed.""" if extension_name in self.extensions: return self.extensions[extension_name] elif self.format != "memory" and self.has_extension(extension_name): self.load_extension(extension_name) return self.extensions[extension_name] return None
[docs] def load_extension(self, extension_name: str) -> "AnalyzerExtension | None": """Load an extension from disk into memory.""" assert self.format != "memory", "load_extension() is for non-memory formats" extension_class = get_extension_class(extension_name) if extension_class is None: return None ext_instance = extension_class.load(self) self.extensions[extension_name] = ext_instance return ext_instance
[docs] def load_all_saved_extension(self): """Load all saved extensions into memory.""" for name in self.get_saved_extension_names(): self.load_extension(name)
[docs] def delete_extension(self, extension_name) -> None: """Delete an extension from memory and from disk.""" if self.format != "memory" and self.has_extension(extension_name): ext = self.load_extension(extension_name) if ext is not None: ext.delete() self.extensions.pop(extension_name, None)
def _get_extensions_params(self) -> dict[str, dict]: """Params of every computed extension, keyed by name.""" # saved-but-unloaded extensions fall back to default params rather than being read from disk params: dict[str, dict] = ( {name: {} for name in self.get_saved_extension_names()} if self.format != "memory" else {} ) params.update({name: ext.params for name, ext in self.extensions.items()}) return params
[docs] def get_loaded_extension_names(self) -> list[str]: """Return names of currently loaded extensions.""" return list(self.extensions.keys())
[docs] def has_extension(self, extension_name: str) -> bool: """Check if an extension exists (in memory or on disk).""" if extension_name in self.extensions: return True if self.format == "memory": return False return extension_name in self.get_saved_extension_names()
[docs] def get_computable_extensions(self) -> list[str]: """List all registered extension names.""" return get_available_analyzer_extensions()
[docs] def get_default_extension_params(self, extension_name: str) -> dict: """Get default params for an extension.""" return get_default_analyzer_extension_params(extension_name)
# --------------------------------------------------------------------------- # Extension dependency utilities # --------------------------------------------------------------------------- _possible_extensions: list[type["AnalyzerExtension"]] = [] def _get_children_dependencies(extension_name, extensions): """Recursively find the extensions that depend on ``extension_name``. ``extensions`` maps each computed extension name -> params. Dependencies are resolved with those params, so optional dependencies (e.g. ``fluorescence`` on ``neuropil``) are included, and an ``"a|b"`` dependency resolves to its first alternative present in ``extensions``. """ names = [] for child, params in extensions.items(): ext_class = get_extension_class(child) deps = ext_class.get_required_dependencies(**params) + ext_class.get_optional_dependencies(**params) parents = {next((d for d in dep.split("|") if d in extensions), None) for dep in deps} if child not in names and extension_name in parents: names.append(child) names.extend(n for n in _get_children_dependencies(child, extensions) if n not in names) return names def _sort_extensions_by_dependency(extensions: dict) -> dict: """Sort extensions so parents come before children.""" ext_list = list(extensions.keys()) params_list = list(extensions.values()) i = 0 while i < len(ext_list): name = ext_list[i] params = params_list[i] ext_class = get_extension_class(name) required = ext_class.get_required_dependencies(**params) optional = ext_class.get_optional_dependencies(**params) all_deps = list(chain.from_iterable(d.split("|") for d in required + optional)) did_nothing = True for dep in all_deps: if dep in ext_list[i:]: dep_idx = ext_list.index(dep) params_list.insert(i, params_list.pop(dep_idx)) ext_list.insert(i, ext_list.pop(dep_idx)) did_nothing = False if did_nothing: i += 1 return dict(zip(ext_list, params_list))
[docs] def register_result_extension(extension_class): """Register an AnalyzerExtension subclass so it can be discovered by name.""" assert issubclass(extension_class, AnalyzerExtension) assert extension_class.extension_name is not None, "extension_name must not be None" global _possible_extensions already = any(extension_class is ext for ext in _possible_extensions) if not already: assert all( extension_class.extension_name != ext.extension_name for ext in _possible_extensions ), f"Extension name '{extension_class.extension_name}' already registered" _possible_extensions.append(extension_class)
[docs] def get_extension_class(extension_name: str, auto_import: bool = True): """Get extension class by name, auto-importing if needed.""" global _possible_extensions ext_dict = {ext.extension_name: ext for ext in _possible_extensions} if extension_name not in ext_dict: if extension_name in _builtin_extensions: module = _builtin_extensions[extension_name] if auto_import: importlib.import_module(module) ext_dict = {ext.extension_name: ext for ext in _possible_extensions} else: raise ValueError(f"Extension '{extension_name}' not registered. Import '{module}' first.") else: warnings.warn(f"Extension '{extension_name}' is unknown.") return None return ext_dict.get(extension_name)
[docs] def get_available_analyzer_extensions() -> list[str]: """Get all built-in extension names.""" return list(_builtin_extensions.keys())
[docs] def get_default_analyzer_extension_params(extension_name: str) -> dict: """Get the default params for an extension.""" import inspect ext_class = get_extension_class(extension_name) sig = inspect.signature(ext_class._set_params) return {k: v.default for k, v in sig.parameters.items() if k != "self" and v.default != inspect.Parameter.empty}
# --------------------------------------------------------------------------- # AnalyzerExtension # ---------------------------------------------------------------------------
[docs] class AnalyzerExtension: """Base class for RoiAnalyzer extensions. Subclasses must set ``extension_name`` and implement: * ``_set_params(**params)`` — validate / clean params * ``_run(**kwargs)`` — populate ``self.data`` * ``_select_extension_data(roi_ids)`` — filter data to a subset * ``_get_data()`` — return the computed result Optionally (for pipeline extensions): * ``_get_pipeline_nodes()`` — if ``use_nodepipeline = True`` """ extension_name: str | None = None depend_on: list[str] = [] need_imaging = False use_nodepipeline = False nodepipeline_variables: list[str] | None = None need_job_kwargs = False need_backward_compatibility_on_load = False def __init__(self, roi_analyzer): self._roi_analyzer = weakref.ref(roi_analyzer) self.params = None self.run_info = self._default_run_info_dict() self.data = dict() def _default_run_info_dict(self): return dict(run_completed=False, runtime_s=None) # ------------------------------------------------------------------ # Abstract methods — subclasses must implement these # ------------------------------------------------------------------ def _run(self, **kwargs): raise NotImplementedError def _set_params(self, **params): raise NotImplementedError def _select_extension_data(self, roi_ids): raise NotImplementedError def _get_pipeline_nodes(self): raise NotImplementedError def _get_data(self): raise NotImplementedError def _handle_backward_compatibility_on_load(self): pass # ------------------------------------------------------------------ # Properties # ------------------------------------------------------------------ @property def roi_analyzer(self): analyzer = self._roi_analyzer() if analyzer is None: raise ValueError(f"Extension '{self.extension_name}' lost its RoiAnalyzer (garbage collected)") return analyzer @property def format(self): return self.roi_analyzer.format @property def folder(self): return self.roi_analyzer.folder # ------------------------------------------------------------------ # Dependency introspection # ------------------------------------------------------------------
[docs] @classmethod def get_required_dependencies(cls, **params): return cls.depend_on
[docs] @classmethod def get_optional_dependencies(cls, **params): return []
[docs] @classmethod def get_any_dependencies(cls, **params): required = cls.get_required_dependencies(**params) optional = cls.get_optional_dependencies(**params) return list(chain.from_iterable(d.split("|") for d in required + optional))
[docs] @classmethod def get_default_params(cls): return get_default_analyzer_extension_params(cls.extension_name)
# ------------------------------------------------------------------ # Load / save # ------------------------------------------------------------------
[docs] @classmethod def load(cls, roi_analyzer): ext = cls(roi_analyzer) ext.load_params() ext.load_run_info() if ext.run_info is not None and ext.run_info["run_completed"]: ext.load_data() if cls.need_backward_compatibility_on_load: ext._handle_backward_compatibility_on_load() if len(ext.data) > 0: return ext else: ext.load_data() if cls.need_backward_compatibility_on_load: ext._handle_backward_compatibility_on_load() if len(ext.data) > 0: return ext return None
[docs] def load_run_info(self): run_info = None if self.format == "binary_folder": path = self._get_binary_extension_folder() / "run_info.json" if path.is_file(): with open(path) as f: run_info = json.load(f) elif self.format == "zarr": group = self._get_zarr_extension_group(mode="r") run_info = group.attrs.get("run_info", None) if run_info is None: warnings.warn(f"No run_info for '{self.extension_name}'; extension should be re-computed.") self.run_info = run_info
[docs] def load_params(self): if self.format == "binary_folder": path = self._get_binary_extension_folder() / "params.json" assert path.is_file(), f"No params file for extension '{self.extension_name}'" with open(path) as f: self.params = json.load(f) elif self.format == "zarr": group = self._get_zarr_extension_group(mode="r") assert "params" in group.attrs, f"No params for extension '{self.extension_name}'" self.params = group.attrs["params"]
[docs] def load_data(self): try: import pandas as pd HAS_PANDAS = True except ImportError: HAS_PANDAS = False if self.format == "binary_folder": folder = self._get_binary_extension_folder() for fpath in folder.iterdir(): if fpath.name in ("params.json", "info.json", "run_info.json") or fpath.name.startswith("._"): continue name = fpath.stem if fpath.suffix == ".json": with fpath.open() as f: self.set_data(name, json.load(f)) elif fpath.suffix == ".npy": self.set_data(name, np.load(fpath)) elif fpath.suffix == ".csv" and HAS_PANDAS: df = pd.read_csv(fpath, index_col=0) roi_ids = self.roi_analyzer.roi_ids if df.shape[0] == roi_ids.size and df.index.dtype != roi_ids.dtype: df.index = df.index.astype(roi_ids.dtype) self.set_data(name, df) elif fpath.suffix == ".pkl": with fpath.open("rb") as f: self.set_data(name, pickle.load(f)) elif self.format == "zarr": group = self._get_zarr_extension_group(mode="r") for name in group.keys(): item = group[name] if "dict" in item.attrs: self.set_data(name, item[0]) elif "dataframe" in item.attrs: if HAS_PANDAS: index = item["index"] df = pd.DataFrame(index=index) for col in item.keys(): if col != "index": df[col] = item[col][:] self.set_data(name, df.convert_dtypes()) elif "object" in item.attrs: self.set_data(name, item[0]) else: self.set_data(name, np.array(item)) if len(self.data) == 0: warnings.warn(f"No data for '{self.extension_name}'; extension should be re-computed.")
[docs] def run(self, save=True, **kwargs): if save and not self.roi_analyzer.is_read_only(): self._save_params() self._save_importing_provenance() t0 = perf_counter() self._run(**kwargs) self.run_info["runtime_s"] = perf_counter() - t0 self.run_info["run_completed"] = True if save and not self.roi_analyzer.is_read_only(): self._save_run_info() self._save_data() if self.format == "zarr": import zarr zarr.consolidate_metadata(self.roi_analyzer._get_zarr_root().store)
[docs] def save(self): self._save_params() self._save_importing_provenance() self._save_run_info() self._save_data() if self.format == "zarr": import zarr zarr.consolidate_metadata(self.roi_analyzer._get_zarr_root().store)
[docs] def copy(self, new_roi_analyzer, roi_ids=None): """Copy this extension to a new RoiAnalyzer, optionally filtering to roi_ids.""" new_ext = self.__class__(new_roi_analyzer) new_ext.params = self.params.copy() if roi_ids is None: new_ext.data = self.data else: new_ext.data = self._select_extension_data(roi_ids) new_ext.run_info = copy(self.run_info) new_ext.save() return new_ext
[docs] def delete(self): """Delete extension from disk and clear in-memory data.""" self._delete_extension_folder() self.params = None self.run_info = self._default_run_info_dict() self.data = dict()
[docs] def reset(self): """Reset extension: clear data and recreate empty folder.""" self._reset_extension_folder() self.params = None self.run_info = self._default_run_info_dict() self.data = dict()
[docs] def set_params(self, save=True, **params): """Set parameters and optionally persist.""" if save: self._reset_extension_folder() params = self._set_params(**params) self.params = params if self.roi_analyzer.is_read_only(): return if save: self._save_params() self._save_importing_provenance()
[docs] def get_data(self, *args, **kwargs): if self.run_info is not None: assert self.run_info[ "run_completed" ], f"Extension '{self.extension_name}' must be run before retrieving data" assert len(self.data) > 0, "Extension has been run but no data found." return self._get_data(*args, **kwargs)
[docs] def set_data(self, name, value): self.data[name] = value
[docs] def get_pipeline_nodes(self): assert self.use_nodepipeline, "get_pipeline_nodes() requires use_nodepipeline=True" return self._get_pipeline_nodes()
# ------------------------------------------------------------------ # Folder / zarr helpers # ------------------------------------------------------------------ def _get_binary_extension_folder(self): return self.folder / "extensions" / self.extension_name def _get_zarr_extension_group(self, mode="r+"): zarr_root = self.roi_analyzer._get_zarr_root(mode=mode) return zarr_root["extensions"][self.extension_name] def _reset_extension_folder(self): if self.format == "binary_folder": folder = self._get_binary_extension_folder() if folder.is_dir(): shutil.rmtree(folder) folder.mkdir(exist_ok=False, parents=True) elif self.format == "zarr": import zarr zarr_root = self.roi_analyzer._get_zarr_root(mode="r+") zarr_root["extensions"].create_group(self.extension_name, overwrite=True) zarr.consolidate_metadata(zarr_root.store) def _delete_extension_folder(self): if self.format == "binary_folder": folder = self._get_binary_extension_folder() if folder.is_dir(): shutil.rmtree(folder) elif self.format == "zarr": import zarr zarr_root = self.roi_analyzer._get_zarr_root(mode="r+") if self.extension_name in zarr_root["extensions"]: del zarr_root["extensions"][self.extension_name] zarr.consolidate_metadata(zarr_root.store) def _save_params(self): params_to_save = self.params.copy() self._reset_extension_folder() if self.format == "binary_folder": folder = self._get_binary_extension_folder() folder.mkdir(exist_ok=True, parents=True) (folder / "params.json").write_text(json.dumps(check_json(params_to_save), indent=4), encoding="utf8") elif self.format == "zarr": group = self._get_zarr_extension_group(mode="r+") group.attrs["params"] = check_json(params_to_save) def _save_importing_provenance(self): info = retrieve_importing_provenance(self.__class__) if self.format == "binary_folder": folder = self._get_binary_extension_folder() folder.mkdir(exist_ok=True, parents=True) (folder / "info.json").write_text(json.dumps(info, indent=4), encoding="utf8") elif self.format == "zarr": group = self._get_zarr_extension_group(mode="r+") group.attrs["info"] = info def _save_run_info(self): if self.run_info is None: return run_info = self.run_info.copy() if self.format == "binary_folder": folder = self._get_binary_extension_folder() (folder / "run_info.json").write_text(json.dumps(run_info, indent=4), encoding="utf8") elif self.format == "zarr": group = self._get_zarr_extension_group(mode="r+") group.attrs["run_info"] = run_info def _save_data(self): if self.format == "memory": return if self.roi_analyzer.is_read_only(): raise ValueError(f"RoiAnalyzer is read-only; cannot save extension '{self.extension_name}'") try: import pandas as pd HAS_PANDAS = True except ImportError: HAS_PANDAS = False if self.format == "binary_folder": folder = self._get_binary_extension_folder() for name, data in self.data.items(): if isinstance(data, dict): with (folder / f"{name}.json").open("w") as f: json.dump(check_json(data), f) elif isinstance(data, np.ndarray): data_file = folder / f"{name}.npy" if isinstance(data, np.memmap) and data_file.exists(): pass # already on disk else: np.save(data_file, data) elif HAS_PANDAS and isinstance(data, pd.DataFrame): data.to_csv(folder / f"{name}.csv", index=True) else: try: with (folder / f"{name}.pkl").open("wb") as f: pickle.dump(data, f) except Exception: raise Exception(f"Could not save '{name}' as extension data") elif self.format == "zarr": import numcodecs from spikeinterface.core.zarrextractors import get_default_zarr_compressor saving_options = self.roi_analyzer._backend_options.get("saving_options", {}) group = self._get_zarr_extension_group(mode="r+") if "compressor" not in saving_options: saving_options["compressor"] = get_default_zarr_compressor() for name, data in self.data.items(): if name in group: del group[name] if isinstance(data, (dict, list)): group.create_dataset( name=name, data=np.array([check_json(data)], dtype=object), object_codec=numcodecs.JSON(), ) group[name].attrs["dict"] = True elif isinstance(data, np.ndarray): group.create_dataset(name=name, data=data, **saving_options) elif HAS_PANDAS and isinstance(data, pd.DataFrame): df_group = group.create_group(name) indices = data.index.to_numpy() if indices.dtype.kind == "O": indices = indices.astype(str) df_group.create_dataset(name="index", data=indices) for col in data.columns: col_data = data[col].to_numpy() if col_data.dtype.kind == "O": col_data = col_data.astype(str) df_group.create_dataset(name=col, data=col_data) df_group.attrs["dataframe"] = True else: # Wrap via an object array assignment rather than np.array([data], dtype=object): # the latter still invokes data.__array__ for array-like objects (e.g. sparse # arrays), which either densifies unexpectedly or raises outright. boxed = np.empty(1, dtype=object) boxed[0] = data try: group.create_dataset( name=name, data=boxed, object_codec=numcodecs.Pickle(), ) except Exception: raise Exception(f"Could not save '{name}' as extension data") group[name].attrs["object"] = True # ------------------------------------------------------------------ # Function factory for backwards-compatible compute_xxx() helpers # ------------------------------------------------------------------
[docs] @classmethod def function_factory(cls): class FuncWrapper: def __init__(self, extension_name): self.extension_name = extension_name def __call__(self, roi_analyzer, *args, **kwargs): if not isinstance(roi_analyzer, RoiAnalyzer): raise ValueError(f"compute_{self.extension_name}() needs a RoiAnalyzer instance") ext = roi_analyzer.compute(cls.extension_name, *args, **kwargs) return ext.get_data() func = FuncWrapper(cls.extension_name) func.__doc__ = cls.__doc__ return func
# --------------------------------------------------------------------------- # Built-in extensions registry # --------------------------------------------------------------------------- _builtin_extensions: dict[str, str] = { "fluorescence": "photon_mosaic.core", "df_over_f": "photon_mosaic.core", "deconvolution": "photon_mosaic.core", "neuropil": "photon_mosaic.core", "background": "photon_mosaic.core", }