"""
RoiAnalyzer: core class for pairing BaseRois and BaseImaging for analysis.
Adapted from SpikeInterface's SortingAnalyzer to work with photon-mosaic's
BaseRois and BaseImaging instead of BaseSorting and BaseRecording.
"""
import importlib
import json
import os
import pickle
import shutil
import warnings
import weakref
from copy import copy
from itertools import chain
from pathlib import Path
from time import perf_counter
from typing import Any, Literal
import numpy as np
from spikeinterface.core.core_tools import (
check_json,
clean_zarr_folder_name,
is_path_remote,
retrieve_importing_provenance,
)
from spikeinterface.core.job_tools import split_job_kwargs
import photon_mosaic
from .baseimaging import BaseImaging
from .baserois import BaseRois
from .binaryrois import BinaryFolderRois
from .imaging_tools import do_imaging_attributes_match, get_imaging_attributes
from .numpyimaging import NumpyRois
from .zarrrois import ZarrRois, add_rois_to_zarr_group
# ---------------------------------------------------------------------------
# High-level factory functions
# ---------------------------------------------------------------------------
[docs]
def create_roi_analyzer(
rois: BaseRois,
imaging: BaseImaging,
format: str = "memory",
folder=None,
overwrite: bool = False,
backend_options: dict | None = None,
) -> "RoiAnalyzer":
"""Create a RoiAnalyzer by pairing a BaseRois and a BaseImaging.
The RoiAnalyzer manages a collection of AnalyzerExtension instances for
post-processing steps such as trace extraction, dF/F, deconvolution, etc.
Parameters
----------
rois : BaseRois
The ROIs object.
imaging : BaseImaging
The imaging object.
format : "memory" | "binary_folder" | "zarr", default: "memory"
The storage backend. If "binary_folder" or "zarr", ``folder`` must be provided.
folder : str | Path | None, default: None
The folder where the analyzer is persisted (required for non-memory formats).
overwrite : bool, default: False
If True, overwrite the folder if it already exists.
backend_options : dict | None, default: None
Backend-specific options. May contain:
* storage_options: dict | None (fsspec storage options)
* saving_options: dict | None (e.g. compression for zarr)
Returns
-------
RoiAnalyzer
The created RoiAnalyzer object.
"""
if format != "memory" and not is_path_remote(folder):
folder = clean_zarr_folder_name(folder) if format == "zarr" else folder
if Path(folder).is_dir():
if overwrite:
shutil.rmtree(folder)
else:
raise ValueError(f"Folder {folder} already exists! Use overwrite=True to overwrite it.")
roi_analyzer = RoiAnalyzer.create(
rois,
imaging,
format=format, # type: ignore[arg-type]
folder=folder,
backend_options=backend_options,
)
return roi_analyzer
[docs]
def load_roi_analyzer(
folder, load_extensions: bool = True, format: str = "auto", backend_options: dict | None = None
) -> "RoiAnalyzer":
"""Load a RoiAnalyzer from disk.
Parameters
----------
folder : str or Path
The folder where the analyzer is stored.
load_extensions : bool, default: True
Whether to load all saved extensions.
format : "auto" | "binary_folder" | "zarr", default: "auto"
The format of the stored analyzer. "auto" will guess from the path.
backend_options : dict | None, default: None
Backend-specific options (e.g. storage_options for remote zarr).
Returns
-------
RoiAnalyzer
The loaded RoiAnalyzer.
"""
return RoiAnalyzer.load(folder, load_extensions=load_extensions, format=format, backend_options=backend_options)
# ---------------------------------------------------------------------------
# RoiAnalyzer
# ---------------------------------------------------------------------------
[docs]
class RoiAnalyzer:
"""Pair BaseRois and BaseImaging for extensible post-processing analysis.
Maintains a collection of computed AnalyzerExtension instances. Supports
three storage backends: ``"memory"``, ``"binary_folder"``, and ``"zarr"``.
Do not instantiate directly — use :func:`create_roi_analyzer` or
:meth:`RoiAnalyzer.create`.
"""
def __init__(
self,
rois: BaseRois,
imaging: BaseImaging | None = None,
imaging_attributes: dict | None = None,
format: str | None = None,
backend_options: dict | None = None,
):
# Fast init — validation is done in create / load
self.rois = rois
self._imaging = imaging
self.imaging_attributes = imaging_attributes
self.format = format
self.folder: str | Path | None = None
self._temporary_imaging: BaseImaging | None = None
self._backend_options = {} if backend_options is None else backend_options
# Extensions are not loaded at init
self.extensions: dict[str, AnalyzerExtension] = dict()
def __repr__(self) -> str:
cls_name = self.__class__.__name__
n_rois = self.get_num_rois()
n_epochs = self.get_num_epochs()
assert self.imaging_attributes is not None
shape = tuple(self.imaging_attributes["shape"])
txt = (
f"{cls_name}: {n_rois} ROIs - {shape[0]}x{shape[1]} ({shape[2]} planes) - {n_epochs} epochs - {self.format}"
)
if self.format != "memory":
if is_path_remote(str(self.folder)):
txt += " (remote)"
if self.has_imaging():
txt += " - has imaging"
if self.has_temporary_imaging():
txt += " - has temporary imaging"
ext_txt = f"Loaded {len(self.extensions)} extensions"
if len(self.extensions) > 0:
ext_txt += f": {', '.join(self.extensions.keys())}"
txt += "\n" + ext_txt
return txt
# ------------------------------------------------------------------
# Create / load
# ------------------------------------------------------------------
[docs]
@classmethod
def create(
cls,
rois: BaseRois,
imaging: BaseImaging,
format: Literal["memory", "binary_folder", "zarr"] = "memory",
folder=None,
backend_options: dict | None = None,
):
"""Create a new RoiAnalyzer.
Parameters
----------
rois : BaseRois
The ROIs object.
imaging : BaseImaging
The imaging object.
format : str
Storage backend.
folder : str | Path | None
Required for non-memory formats.
backend_options : dict | None
Backend-specific options.
Returns
-------
RoiAnalyzer
"""
assert imaging is not None, "An imaging object must be provided to create a RoiAnalyzer"
# Validate compatibility
if not np.isclose(rois.sampling_frequency, imaging.sampling_frequency, atol=0.1):
raise ValueError(
f"Sampling frequency mismatch: rois={rois.sampling_frequency}, " f"imaging={imaging.sampling_frequency}"
)
if rois.shape[:2] != imaging.shape[:2]:
raise ValueError(f"Spatial shape mismatch: rois={rois.shape[:2]}, imaging={imaging.shape[:2]}")
if rois.num_planes != imaging.num_planes:
raise ValueError(f"Number of planes mismatch: rois={rois.num_planes}, imaging={imaging.num_planes}")
if format == "memory":
return cls.create_memory(rois, imaging, imaging_attributes=None)
elif format == "binary_folder":
return cls.create_binary_folder(
folder, rois, imaging, imaging_attributes=None, backend_options=backend_options
)
elif format == "zarr":
assert folder is not None, "For format='zarr' folder must be provided"
if not is_path_remote(folder):
folder = clean_zarr_folder_name(folder)
return cls.create_zarr(folder, rois, imaging, imaging_attributes=None, backend_options=backend_options)
else:
raise ValueError(f"Unknown format: {format}")
[docs]
@classmethod
def load(cls, folder, imaging=None, load_extensions=True, format="auto", backend_options=None):
"""Load from folder or zarr.
The imaging can be provided if its location has changed.
"""
if format == "auto":
if str(folder).endswith(".zarr"):
format = "zarr"
else:
format = "binary_folder"
if format == "binary_folder":
analyzer = cls.load_from_binary_folder(folder, imaging=imaging, backend_options=backend_options)
elif format == "zarr":
analyzer = cls.load_from_zarr(folder, imaging=imaging, backend_options=backend_options)
else:
raise ValueError(f"Unknown format: {format}")
if not is_path_remote(str(folder)):
if load_extensions:
analyzer.load_all_saved_extension()
return analyzer
# ------------------------------------------------------------------
# Memory backend
# ------------------------------------------------------------------
[docs]
@classmethod
def create_memory(cls, rois, imaging, imaging_attributes):
if imaging_attributes is None:
assert imaging is not None
imaging_attributes = get_imaging_attributes(imaging)
else:
imaging_attributes = imaging_attributes.copy()
# Make an in-memory copy of rois for fast access. Pass image masks through as-is
# (dense ndarray or sparse array) -- wrapping in np.array() would raise for a sparse
# array (which refuses implicit densification via __array__) instead of copying it.
rois_copy = NumpyRois(
roi_image_masks=rois.get_roi_image_masks(),
sampling_frequency=rois.sampling_frequency,
roi_ids=np.array(rois.roi_ids),
)
rois.copy_metadata(rois_copy, only_main=False, ids=rois_copy.roi_ids)
analyzer = RoiAnalyzer(
rois=rois_copy,
imaging=imaging,
imaging_attributes=imaging_attributes,
format="memory",
)
return analyzer
# ------------------------------------------------------------------
# Binary folder backend
# ------------------------------------------------------------------
[docs]
@classmethod
def create_binary_folder(cls, folder, rois, imaging, imaging_attributes, backend_options):
folder = Path(folder)
if folder.is_dir():
raise ValueError(f"Folder already exists: {folder}")
folder.mkdir(parents=True)
# photon-mosaic info
info = dict(
version=photon_mosaic.__version__,
object="RoiAnalyzer",
)
with open(folder / "photon_mosaic_info.json", "w") as f:
json.dump(check_json(info), f, indent=4)
# Save a copy of the rois
rois.save(format="binary", folder=folder / "rois")
# Save imaging provenance
if imaging is not None:
if imaging.check_serializability("json"):
imaging.dump(folder / "imaging.json", relative_to=folder)
elif imaging.check_serializability("pickle"):
imaging.dump(folder / "imaging.pickle", relative_to=folder)
else:
warnings.warn("The imaging is not serializable. The imaging link will be lost on reload.")
# Save rois provenance
if rois.check_serializability("json"):
rois.dump(folder / "rois_provenance.json", relative_to=folder)
elif rois.check_serializability("pickle"):
rois.dump(folder / "rois_provenance.pickle", relative_to=folder)
else:
warnings.warn("The rois provenance is not serializable. The link will be lost on reload.")
# Imaging attributes
attr_folder = folder / "imaging_info"
attr_folder.mkdir()
if imaging_attributes is None:
imaging_attributes = get_imaging_attributes(imaging)
attr_file = attr_folder / "imaging_attributes.json"
attr_file.write_text(json.dumps(check_json(imaging_attributes), indent=4), encoding="utf8")
# Create extensions folder
(folder / "extensions").mkdir()
return cls.load_from_binary_folder(folder, imaging=imaging, backend_options=backend_options)
[docs]
@classmethod
def load_from_binary_folder(cls, folder, imaging=None, backend_options=None):
from spikeinterface.core.loading import load as si_load
folder = Path(folder)
assert folder.is_dir(), f"Folder does not exist: {folder}"
# Load rois
rois = BinaryFolderRois(folder_path=folder / "rois")
# Try to load the imaging if not provided
if imaging is None:
for ext in ("json", "pickle"):
fname = folder / f"imaging.{ext}"
if fname.exists():
try:
imaging = si_load(fname, base_folder=folder)
break
except Exception:
pass
# Imaging attributes
attr_file = folder / "imaging_info" / "imaging_attributes.json"
if not attr_file.exists():
raise ValueError(f"Not a valid RoiAnalyzer binary_folder: {folder}")
with open(attr_file, "r") as f:
imaging_attributes = json.load(f)
analyzer = RoiAnalyzer(
rois=rois,
imaging=imaging,
imaging_attributes=imaging_attributes,
format="binary_folder",
backend_options=backend_options,
)
analyzer.folder = folder
return analyzer
# ------------------------------------------------------------------
# Zarr backend
# ------------------------------------------------------------------
def _get_zarr_root(self, mode="r+"):
from spikeinterface.core.zarrextractors import super_zarr_open
storage_options = self._backend_options.get("storage_options", {})
return super_zarr_open(str(self.folder), mode=mode, storage_options=storage_options)
[docs]
@classmethod
def create_zarr(cls, folder, rois, imaging, imaging_attributes, backend_options):
import numcodecs
import zarr
backend_options = backend_options or {}
storage_options = backend_options.get("storage_options", {})
saving_options = backend_options.get("saving_options", {})
remote = is_path_remote(str(folder))
if not remote:
folder = clean_zarr_folder_name(folder)
if Path(folder).is_dir():
raise ValueError(f"Folder already exists: {folder}")
zarr_root = zarr.open(str(folder), mode="w", storage_options=storage_options)
# Info
info = dict(
version=photon_mosaic.__version__,
object="RoiAnalyzer",
)
zarr_root.attrs["photon_mosaic_info"] = check_json(info)
# Save imaging provenance
relative_to = folder if not remote else None
if imaging is not None:
rec_dict = imaging.to_dict(relative_to=relative_to, recursive=True)
if imaging.check_serializability("json"):
zarr_rec = np.array([check_json(rec_dict)], dtype=object)
zarr_root.create_dataset("imaging", data=zarr_rec, object_codec=numcodecs.JSON())
elif imaging.check_serializability("pickle"):
zarr_rec = np.array([rec_dict], dtype=object)
zarr_root.create_dataset("imaging", data=zarr_rec, object_codec=numcodecs.Pickle())
else:
warnings.warn("The imaging is not serializable. The link will be lost on reload.")
# Save rois provenance
rois_dict = rois.to_dict(relative_to=relative_to, recursive=True)
if rois.check_serializability("json"):
zarr_rois = np.array([check_json(rois_dict)], dtype=object)
zarr_root.create_dataset("rois_provenance", data=zarr_rois, object_codec=numcodecs.JSON())
elif rois.check_serializability("pickle"):
zarr_rois = np.array([rois_dict], dtype=object)
zarr_root.create_dataset("rois_provenance", data=zarr_rois, object_codec=numcodecs.Pickle())
else:
warnings.warn("The rois provenance is not serializable. The link will be lost on reload.")
# Save imaging attributes
if imaging_attributes is None:
imaging_attributes = get_imaging_attributes(imaging)
imaging_info = zarr_root.create_group("imaging_info")
imaging_info.attrs["imaging_attributes"] = check_json(imaging_attributes)
# Save rois data
rois_group = zarr_root.create_group("rois")
add_rois_to_zarr_group(rois, rois_group, saving_options=saving_options)
# Extensions group
zarr_root.create_group("extensions")
zarr.consolidate_metadata(zarr_root.store)
return cls.load_from_zarr(folder, imaging=imaging, backend_options=backend_options)
[docs]
@classmethod
def load_from_zarr(cls, folder, imaging=None, backend_options=None):
from spikeinterface.core.loading import load as si_load
from spikeinterface.core.zarrextractors import super_zarr_open
backend_options = backend_options or {}
storage_options = backend_options.get("storage_options", {})
zarr_root = super_zarr_open(str(folder), mode="r", storage_options=storage_options)
# Load rois
rois = ZarrRois(folder, zarr_group_name="rois", storage_options=storage_options)
# Load imaging if possible
if imaging is None:
rec_field = zarr_root.get("imaging")
if rec_field is not None:
rec_dict = rec_field[0]
try:
imaging = si_load(rec_dict, base_folder=folder)
except Exception:
imaging = None
# Imaging attributes
imaging_attributes = zarr_root["imaging_info"].attrs["imaging_attributes"]
analyzer = RoiAnalyzer(
rois=rois,
imaging=imaging,
imaging_attributes=imaging_attributes,
format="zarr",
backend_options=backend_options,
)
analyzer.folder = folder
return analyzer
# ------------------------------------------------------------------
# Temporary imaging
# ------------------------------------------------------------------
[docs]
def set_temporary_imaging(self, imaging: BaseImaging, check_dtype: bool = True):
"""Set a temporary imaging (e.g. a cached/loaded copy) without persisting it.
Parameters
----------
imaging : BaseImaging
The imaging object.
check_dtype : bool, default: True
Whether to validate dtype match.
"""
assert self.imaging_attributes is not None
matches, error = do_imaging_attributes_match(imaging, self.imaging_attributes, check_dtype=check_dtype)
if not matches:
raise ValueError(error)
if self._imaging is not None:
warnings.warn("RoiAnalyzer imaging is already set. Temporarily replacing it.")
self._temporary_imaging = imaging
# ------------------------------------------------------------------
# ROI property helpers
# ------------------------------------------------------------------
[docs]
def set_roi_property(
self,
key: str,
values: list | np.ndarray | tuple,
ids: list | np.ndarray | tuple | None = None,
missing_value: Any = None,
save: bool = True,
) -> None:
"""Set a property on the internal ROIs copy, optionally persisting to disk.
Parameters
----------
key : str
Property name.
values : array-like
Property values.
ids : array-like | None
Subset of ROI ids to set. If None, set for all.
missing_value : Any
Fill value for ids not in ``ids``.
save : bool, default: True
Whether to persist (binary_folder / zarr only).
"""
self.rois.set_property(key, values, ids=ids, missing_value=missing_value)
if not self.is_read_only() and save:
if self.format == "binary_folder":
assert isinstance(self.folder, Path)
props_dir = self.folder / "rois" / "properties"
props_dir.mkdir(exist_ok=True)
np.save(props_dir / f"{key}.npy", self.rois.get_property(key))
elif self.format == "zarr":
import zarr
zarr_root = self._get_zarr_root(mode="r+")
prop_values = self.rois.get_property(key)
if prop_values.dtype.kind == "O":
warnings.warn(f"Property '{key}' not saved (object dtype)")
else:
rois_group = zarr_root["rois"]
if "properties" not in rois_group:
rois_group.create_group("properties")
props_group = rois_group["properties"]
if key in props_group:
props_group[key][:] = prop_values
else:
props_group.create_dataset(name=key, data=prop_values, compressor=None)
zarr.consolidate_metadata(zarr_root.store)
[docs]
def get_roi_property(self, key: str, ids=None) -> np.ndarray:
"""Get a property from the internal ROIs copy."""
return self.rois.get_property(key, ids=ids)
# ------------------------------------------------------------------
# Save / select / copy
# ------------------------------------------------------------------
def _save_or_select(
self,
format="binary_folder",
folder=None,
roi_ids=None,
backend_options=None,
verbose=False,
) -> "RoiAnalyzer":
"""Internal method used by save_as(), copy(), select_rois(), remove_rois().
Parameters
----------
format : str
Target backend.
folder : str | Path | None
Target folder (for non-memory).
roi_ids : array-like | None
Subset of roi ids to keep. None = keep all.
backend_options : dict | None
Backend-specific options.
verbose : bool
Whether to print progress.
Returns
-------
RoiAnalyzer
"""
if self.has_imaging():
imaging = self._imaging
elif self.has_temporary_imaging():
imaging = self._temporary_imaging
else:
imaging = None
# Get the original rois provenance (or fall back to internal copy)
rois_provenance = self.get_rois_provenance()
if rois_provenance is None:
rois_provenance = self.rois
# Copy any in-memory properties added after creation
for key in self.rois.get_property_keys():
if key not in rois_provenance.get_property_keys():
rois_provenance.set_property(key, self.rois.get_property(key))
if roi_ids is not None:
rois_provenance = rois_provenance.select_rois(roi_ids)
backend_options = backend_options or {}
if format == "memory":
new_analyzer = RoiAnalyzer.create_memory(rois_provenance, imaging, self.imaging_attributes)
elif format == "binary_folder":
assert folder is not None, "folder must be provided for binary_folder"
new_analyzer = RoiAnalyzer.create_binary_folder(
Path(folder), rois_provenance, imaging, self.imaging_attributes, backend_options
)
elif format == "zarr":
assert folder is not None, "folder must be provided for zarr"
folder = clean_zarr_folder_name(folder)
new_analyzer = RoiAnalyzer.create_zarr(
folder, rois_provenance, imaging, self.imaging_attributes, backend_options
)
else:
raise ValueError(f"Unsupported format: {format}")
# Propagate extensions
extensions_to_compute = _sort_extensions_by_dependency(
{ext.extension_name: ext.params for ext in self.extensions.values()}
)
for extension_name in extensions_to_compute:
extension = self.extensions[extension_name]
new_analyzer.extensions[extension_name] = extension.copy(new_analyzer, roi_ids=roi_ids)
return new_analyzer
[docs]
def save_as(self, format="memory", folder=None, backend_options=None) -> "RoiAnalyzer":
"""Save the analyzer to a different backend.
Parameters
----------
format : "memory" | "binary_folder" | "zarr"
Target backend.
folder : str | Path | None
Required for non-memory formats.
backend_options : dict | None
Backend-specific options.
Returns
-------
RoiAnalyzer
"""
if format == "zarr":
folder = clean_zarr_folder_name(folder)
return self._save_or_select(format=format, folder=folder, backend_options=backend_options)
[docs]
def select_rois(self, roi_ids, format="memory", folder=None) -> "RoiAnalyzer":
"""Create a new RoiAnalyzer with a subset of ROIs.
Extensions are filtered to the selected ROI ids.
Parameters
----------
roi_ids : array-like
ROI ids to keep.
format : str
Target backend.
folder : str | Path | None
Required for non-memory formats.
Returns
-------
RoiAnalyzer
"""
if format == "zarr":
folder = clean_zarr_folder_name(folder)
return self._save_or_select(format=format, folder=folder, roi_ids=roi_ids)
[docs]
def remove_rois(self, remove_roi_ids, format="memory", folder=None) -> "RoiAnalyzer":
"""Create a new RoiAnalyzer with some ROIs removed.
Parameters
----------
remove_roi_ids : array-like
ROI ids to remove.
format : str
Target backend.
folder : str | Path | None
Required for non-memory formats.
Returns
-------
RoiAnalyzer
"""
roi_ids = self.roi_ids[~np.isin(self.roi_ids, remove_roi_ids)]
if format == "zarr":
folder = clean_zarr_folder_name(folder)
return self._save_or_select(format=format, folder=folder, roi_ids=roi_ids)
[docs]
def copy(self) -> "RoiAnalyzer":
"""Create an in-memory copy."""
return self._save_or_select(format="memory", folder=None)
# ------------------------------------------------------------------
# Properties / accessors
# ------------------------------------------------------------------
@property
def imaging(self) -> BaseImaging:
if not self.has_imaging() and not self.has_temporary_imaging():
raise ValueError("RoiAnalyzer could not load the imaging")
imaging = self._temporary_imaging or self._imaging
assert imaging is not None
return imaging
@property
def roi_ids(self) -> np.ndarray:
return self.rois.roi_ids
@property
def sampling_frequency(self) -> float:
assert self.imaging_attributes is not None
return float(self.imaging_attributes["sampling_frequency"])
@property
def shape(self) -> tuple:
assert self.imaging_attributes is not None
return tuple(self.imaging_attributes["shape"])
@property
def num_planes(self) -> int:
return self.shape[2]
[docs]
def has_imaging(self) -> bool:
return self._imaging is not None
[docs]
def has_temporary_imaging(self) -> bool:
return self._temporary_imaging is not None
[docs]
def get_num_rois(self) -> int:
return self.rois.get_num_rois()
[docs]
def get_num_epochs(self) -> int:
assert self.imaging_attributes is not None
return self.imaging_attributes["num_epochs"]
[docs]
def get_num_samples(self, epoch_index: int | None = None) -> int:
if epoch_index is None:
if self.get_num_epochs() == 1:
epoch_index = 0
else:
raise ValueError("epoch_index must be provided for multi-epoch data.")
assert self.imaging_attributes is not None
return self.imaging_attributes["num_samples"][epoch_index]
[docs]
def get_total_samples(self) -> int:
assert self.imaging_attributes is not None
return sum(self.imaging_attributes["num_samples"])
[docs]
def get_total_duration(self) -> float:
if self.has_imaging() or self.has_temporary_imaging():
return self.imaging.get_total_duration()
return self.get_total_samples() / self.sampling_frequency
[docs]
def get_dtype(self):
return np.dtype(self.imaging_attributes["dtype"])
[docs]
def is_read_only(self) -> bool:
if self.format == "memory":
return False
assert self.folder is not None
if self.format == "binary_folder":
return not os.access(self.folder, os.W_OK)
else:
if not is_path_remote(str(self.folder)):
return not os.access(self.folder, os.W_OK)
return False
[docs]
def get_rois_provenance(self):
"""Get the original rois object if possible, otherwise None."""
from spikeinterface.core.loading import load as si_load
if self.format == "memory":
return None
elif self.format == "binary_folder":
assert isinstance(self.folder, Path)
for ext in ("json", "pickle"):
fname = self.folder / f"rois_provenance.{ext}"
if fname.exists():
try:
return si_load(fname, base_folder=self.folder)
except Exception:
pass
return None
elif self.format == "zarr":
zarr_root = self._get_zarr_root(mode="r")
if "rois_provenance" in zarr_root.keys():
try:
rois_dict = zarr_root["rois_provenance"][0]
return si_load(rois_dict, base_folder=self.folder)
except Exception:
pass
return None
# ------------------------------------------------------------------
# Extension management
# ------------------------------------------------------------------
[docs]
def compute(self, input, save=True, extension_params=None, verbose=False, **kwargs) -> "AnalyzerExtension | None":
"""Compute one or several extensions.
Parameters
----------
input : str | dict | list
Extension name (str), dict of {name: params}, or list of names.
save : bool, default: True
Whether to persist computed extensions.
extension_params : dict | None
Per-extension params when ``input`` is a list.
verbose : bool
Print progress.
**kwargs
Passed to the extension's ``set_params`` (if str) or as job_kwargs.
Returns
-------
AnalyzerExtension | None
The extension instance if input is a string, None otherwise.
"""
if isinstance(input, str):
return self.compute_one_extension(extension_name=input, save=save, verbose=verbose, **kwargs)
elif isinstance(input, dict):
params_, job_kwargs = split_job_kwargs(kwargs)
assert len(params_) == 0, f"Unexpected arguments: {set(params_)}"
self.compute_several_extensions(extensions=input, save=save, verbose=verbose, **job_kwargs)
elif isinstance(input, list):
params_, job_kwargs = split_job_kwargs(kwargs)
assert len(params_) == 0, f"Unexpected arguments: {set(params_)}"
extensions: dict[str, dict[str, Any]] = {k: {} for k in input}
if extension_params is not None:
for name, params in extension_params.items():
assert name in input, f"Extension '{name}' not in input list"
extensions[name] = params
self.compute_several_extensions(extensions=extensions, save=save, verbose=verbose, **job_kwargs)
else:
raise ValueError("compute() expects a str, dict, or list")
return None
[docs]
def compute_one_extension(self, extension_name, save=True, verbose=False, **kwargs) -> "AnalyzerExtension":
"""Compute a single extension.
Automatically deletes dependent extensions to keep data coherent.
"""
extension_class = get_extension_class(extension_name)
# Delete children that depend on this extension
for child in _get_children_dependencies(extension_name, self._get_extensions_params()):
if self.has_extension(child):
if verbose:
print(f"Deleting extension: {child}")
self.delete_extension(child)
params, job_kwargs = split_job_kwargs(kwargs)
# Check dependencies
if extension_class.need_imaging:
assert (
self.has_imaging() or self.has_temporary_imaging()
), f"Extension '{extension_name}' requires the imaging"
for dep in extension_class.get_required_dependencies(**params):
if "|" in dep:
ok = any(self.get_extension(d) is not None for d in dep.split("|"))
else:
ok = self.get_extension(dep) is not None
assert ok, f"Extension '{extension_name}' requires '{dep}' to be computed first"
ext_instance = extension_class(self)
ext_instance.set_params(save=save, **params)
if extension_class.need_job_kwargs:
ext_instance.run(save=save, verbose=verbose, **job_kwargs)
else:
ext_instance.run(save=save, verbose=verbose)
self.extensions[extension_name] = ext_instance
return ext_instance
[docs]
def compute_several_extensions(self, extensions, save=True, verbose=False, **job_kwargs):
"""Compute several extensions respecting dependency order.
Parameters
----------
extensions : dict
Mapping of extension_name -> params dict.
save : bool
Whether to persist.
verbose : bool
Print progress.
**job_kwargs
Parallelization kwargs.
"""
# Validate dependencies
ext_names = list(extensions.keys())
for name, params in extensions.items():
for dep in get_extension_class(name).get_required_dependencies(**params):
if "|" in dep:
ok = any(self.has_extension(d) or d in ext_names for d in dep.split("|"))
else:
ok = self.has_extension(dep) or dep in ext_names
assert ok, f"Extension '{name}' requires '{dep}' to be computed first"
sorted_exts = _sort_extensions_by_dependency(extensions)
# Delete children of all extensions we're about to recompute
all_params = {**self._get_extensions_params(), **extensions}
for name in sorted_exts:
for child in _get_children_dependencies(name, all_params):
if verbose:
print(f"Deleting extension: {child}")
self.delete_extension(child)
# Group: pipeline vs non-pipeline
pipeline_exts = {}
pre_pipeline_exts = {}
post_pipeline_exts = {}
for name, params in sorted_exts.items():
ext_class = get_extension_class(name)
if ext_class.use_nodepipeline:
pipeline_exts[name] = params
elif any(
get_extension_class(d).use_nodepipeline
for d in ext_class.get_any_dependencies(**params)
if d in sorted_exts
):
post_pipeline_exts[name] = params
else:
pre_pipeline_exts[name] = params
# Pre-pipeline
for name, params in pre_pipeline_exts.items():
ext_class = get_extension_class(name)
if ext_class.need_job_kwargs:
self.compute_one_extension(name, save=save, verbose=verbose, **params, **job_kwargs)
else:
self.compute_one_extension(name, save=save, verbose=verbose, **params)
# Pipeline extensions (run together via node pipeline)
if len(pipeline_exts) > 0:
from spikeinterface.core.node_pipeline import run_node_pipeline
all_nodes = []
result_routage = []
instances = {}
for name, params in pipeline_exts.items():
ext_class = get_extension_class(name)
assert self.has_imaging() or self.has_temporary_imaging(), f"Extension '{name}' requires the imaging"
for var in ext_class.nodepipeline_variables or []:
result_routage.append((name, var))
inst = ext_class(self)
inst.set_params(save=save, **params)
instances[name] = inst
all_nodes.extend(inst.get_pipeline_nodes())
job_name = "Compute: " + " + ".join(pipeline_exts.keys())
t0 = perf_counter()
results = run_node_pipeline(
self.imaging,
all_nodes,
job_kwargs=job_kwargs,
job_name=job_name,
gather_mode="memory",
squeeze_output=False,
verbose=verbose,
check_for_peak_source=False,
)
runtime_s = perf_counter() - t0
for r, result in enumerate(results):
ext_name, var_name = result_routage[r]
instances[ext_name].data[var_name] = result
instances[ext_name].run_info["runtime_s"] = runtime_s
instances[ext_name].run_info["run_completed"] = True
for name, inst in instances.items():
self.extensions[name] = inst
if save:
inst.save()
# Post-pipeline
for name, params in post_pipeline_exts.items():
ext_class = get_extension_class(name)
if ext_class.need_job_kwargs:
self.compute_one_extension(name, save=save, verbose=verbose, **params, **job_kwargs)
else:
self.compute_one_extension(name, save=save, verbose=verbose, **params)
[docs]
def get_saved_extension_names(self) -> list[str]:
"""Get extension names saved on disk (without loading data)."""
saved = []
if self.format == "binary_folder":
assert isinstance(self.folder, Path)
ext_folder = self.folder / "extensions"
if ext_folder.is_dir():
for d in ext_folder.iterdir():
if d.is_dir() and (d / "params.json").is_file():
saved.append(d.stem)
elif self.format == "zarr":
zarr_root = self._get_zarr_root(mode="r")
if "extensions" in zarr_root.keys():
for name in zarr_root["extensions"].keys():
if "params" in zarr_root["extensions"][name].attrs.keys():
saved.append(name)
return saved
[docs]
def get_extension(self, extension_name: str) -> "AnalyzerExtension | None":
"""Get an extension, auto-loading from disk if needed. Returns None if not computed."""
if extension_name in self.extensions:
return self.extensions[extension_name]
elif self.format != "memory" and self.has_extension(extension_name):
self.load_extension(extension_name)
return self.extensions[extension_name]
return None
[docs]
def load_extension(self, extension_name: str) -> "AnalyzerExtension | None":
"""Load an extension from disk into memory."""
assert self.format != "memory", "load_extension() is for non-memory formats"
extension_class = get_extension_class(extension_name)
if extension_class is None:
return None
ext_instance = extension_class.load(self)
self.extensions[extension_name] = ext_instance
return ext_instance
[docs]
def load_all_saved_extension(self):
"""Load all saved extensions into memory."""
for name in self.get_saved_extension_names():
self.load_extension(name)
[docs]
def delete_extension(self, extension_name) -> None:
"""Delete an extension from memory and from disk."""
if self.format != "memory" and self.has_extension(extension_name):
ext = self.load_extension(extension_name)
if ext is not None:
ext.delete()
self.extensions.pop(extension_name, None)
def _get_extensions_params(self) -> dict[str, dict]:
"""Params of every computed extension, keyed by name."""
# saved-but-unloaded extensions fall back to default params rather than being read from disk
params: dict[str, dict] = (
{name: {} for name in self.get_saved_extension_names()} if self.format != "memory" else {}
)
params.update({name: ext.params for name, ext in self.extensions.items()})
return params
[docs]
def get_loaded_extension_names(self) -> list[str]:
"""Return names of currently loaded extensions."""
return list(self.extensions.keys())
[docs]
def has_extension(self, extension_name: str) -> bool:
"""Check if an extension exists (in memory or on disk)."""
if extension_name in self.extensions:
return True
if self.format == "memory":
return False
return extension_name in self.get_saved_extension_names()
[docs]
def get_computable_extensions(self) -> list[str]:
"""List all registered extension names."""
return get_available_analyzer_extensions()
[docs]
def get_default_extension_params(self, extension_name: str) -> dict:
"""Get default params for an extension."""
return get_default_analyzer_extension_params(extension_name)
# ---------------------------------------------------------------------------
# Extension dependency utilities
# ---------------------------------------------------------------------------
_possible_extensions: list[type["AnalyzerExtension"]] = []
def _get_children_dependencies(extension_name, extensions):
"""Recursively find the extensions that depend on ``extension_name``.
``extensions`` maps each computed extension name -> params. Dependencies are resolved with those
params, so optional dependencies (e.g. ``fluorescence`` on ``neuropil``) are included, and an
``"a|b"`` dependency resolves to its first alternative present in ``extensions``.
"""
names = []
for child, params in extensions.items():
ext_class = get_extension_class(child)
deps = ext_class.get_required_dependencies(**params) + ext_class.get_optional_dependencies(**params)
parents = {next((d for d in dep.split("|") if d in extensions), None) for dep in deps}
if child not in names and extension_name in parents:
names.append(child)
names.extend(n for n in _get_children_dependencies(child, extensions) if n not in names)
return names
def _sort_extensions_by_dependency(extensions: dict) -> dict:
"""Sort extensions so parents come before children."""
ext_list = list(extensions.keys())
params_list = list(extensions.values())
i = 0
while i < len(ext_list):
name = ext_list[i]
params = params_list[i]
ext_class = get_extension_class(name)
required = ext_class.get_required_dependencies(**params)
optional = ext_class.get_optional_dependencies(**params)
all_deps = list(chain.from_iterable(d.split("|") for d in required + optional))
did_nothing = True
for dep in all_deps:
if dep in ext_list[i:]:
dep_idx = ext_list.index(dep)
params_list.insert(i, params_list.pop(dep_idx))
ext_list.insert(i, ext_list.pop(dep_idx))
did_nothing = False
if did_nothing:
i += 1
return dict(zip(ext_list, params_list))
[docs]
def register_result_extension(extension_class):
"""Register an AnalyzerExtension subclass so it can be discovered by name."""
assert issubclass(extension_class, AnalyzerExtension)
assert extension_class.extension_name is not None, "extension_name must not be None"
global _possible_extensions
already = any(extension_class is ext for ext in _possible_extensions)
if not already:
assert all(
extension_class.extension_name != ext.extension_name for ext in _possible_extensions
), f"Extension name '{extension_class.extension_name}' already registered"
_possible_extensions.append(extension_class)
[docs]
def get_extension_class(extension_name: str, auto_import: bool = True):
"""Get extension class by name, auto-importing if needed."""
global _possible_extensions
ext_dict = {ext.extension_name: ext for ext in _possible_extensions}
if extension_name not in ext_dict:
if extension_name in _builtin_extensions:
module = _builtin_extensions[extension_name]
if auto_import:
importlib.import_module(module)
ext_dict = {ext.extension_name: ext for ext in _possible_extensions}
else:
raise ValueError(f"Extension '{extension_name}' not registered. Import '{module}' first.")
else:
warnings.warn(f"Extension '{extension_name}' is unknown.")
return None
return ext_dict.get(extension_name)
[docs]
def get_available_analyzer_extensions() -> list[str]:
"""Get all built-in extension names."""
return list(_builtin_extensions.keys())
[docs]
def get_default_analyzer_extension_params(extension_name: str) -> dict:
"""Get the default params for an extension."""
import inspect
ext_class = get_extension_class(extension_name)
sig = inspect.signature(ext_class._set_params)
return {k: v.default for k, v in sig.parameters.items() if k != "self" and v.default != inspect.Parameter.empty}
# ---------------------------------------------------------------------------
# AnalyzerExtension
# ---------------------------------------------------------------------------
[docs]
class AnalyzerExtension:
"""Base class for RoiAnalyzer extensions.
Subclasses must set ``extension_name`` and implement:
* ``_set_params(**params)`` — validate / clean params
* ``_run(**kwargs)`` — populate ``self.data``
* ``_select_extension_data(roi_ids)`` — filter data to a subset
* ``_get_data()`` — return the computed result
Optionally (for pipeline extensions):
* ``_get_pipeline_nodes()`` — if ``use_nodepipeline = True``
"""
extension_name: str | None = None
depend_on: list[str] = []
need_imaging = False
use_nodepipeline = False
nodepipeline_variables: list[str] | None = None
need_job_kwargs = False
need_backward_compatibility_on_load = False
def __init__(self, roi_analyzer):
self._roi_analyzer = weakref.ref(roi_analyzer)
self.params = None
self.run_info = self._default_run_info_dict()
self.data = dict()
def _default_run_info_dict(self):
return dict(run_completed=False, runtime_s=None)
# ------------------------------------------------------------------
# Abstract methods — subclasses must implement these
# ------------------------------------------------------------------
def _run(self, **kwargs):
raise NotImplementedError
def _set_params(self, **params):
raise NotImplementedError
def _select_extension_data(self, roi_ids):
raise NotImplementedError
def _get_pipeline_nodes(self):
raise NotImplementedError
def _get_data(self):
raise NotImplementedError
def _handle_backward_compatibility_on_load(self):
pass
# ------------------------------------------------------------------
# Properties
# ------------------------------------------------------------------
@property
def roi_analyzer(self):
analyzer = self._roi_analyzer()
if analyzer is None:
raise ValueError(f"Extension '{self.extension_name}' lost its RoiAnalyzer (garbage collected)")
return analyzer
@property
def format(self):
return self.roi_analyzer.format
@property
def folder(self):
return self.roi_analyzer.folder
# ------------------------------------------------------------------
# Dependency introspection
# ------------------------------------------------------------------
[docs]
@classmethod
def get_required_dependencies(cls, **params):
return cls.depend_on
[docs]
@classmethod
def get_optional_dependencies(cls, **params):
return []
[docs]
@classmethod
def get_any_dependencies(cls, **params):
required = cls.get_required_dependencies(**params)
optional = cls.get_optional_dependencies(**params)
return list(chain.from_iterable(d.split("|") for d in required + optional))
[docs]
@classmethod
def get_default_params(cls):
return get_default_analyzer_extension_params(cls.extension_name)
# ------------------------------------------------------------------
# Load / save
# ------------------------------------------------------------------
[docs]
@classmethod
def load(cls, roi_analyzer):
ext = cls(roi_analyzer)
ext.load_params()
ext.load_run_info()
if ext.run_info is not None and ext.run_info["run_completed"]:
ext.load_data()
if cls.need_backward_compatibility_on_load:
ext._handle_backward_compatibility_on_load()
if len(ext.data) > 0:
return ext
else:
ext.load_data()
if cls.need_backward_compatibility_on_load:
ext._handle_backward_compatibility_on_load()
if len(ext.data) > 0:
return ext
return None
[docs]
def load_run_info(self):
run_info = None
if self.format == "binary_folder":
path = self._get_binary_extension_folder() / "run_info.json"
if path.is_file():
with open(path) as f:
run_info = json.load(f)
elif self.format == "zarr":
group = self._get_zarr_extension_group(mode="r")
run_info = group.attrs.get("run_info", None)
if run_info is None:
warnings.warn(f"No run_info for '{self.extension_name}'; extension should be re-computed.")
self.run_info = run_info
[docs]
def load_params(self):
if self.format == "binary_folder":
path = self._get_binary_extension_folder() / "params.json"
assert path.is_file(), f"No params file for extension '{self.extension_name}'"
with open(path) as f:
self.params = json.load(f)
elif self.format == "zarr":
group = self._get_zarr_extension_group(mode="r")
assert "params" in group.attrs, f"No params for extension '{self.extension_name}'"
self.params = group.attrs["params"]
[docs]
def load_data(self):
try:
import pandas as pd
HAS_PANDAS = True
except ImportError:
HAS_PANDAS = False
if self.format == "binary_folder":
folder = self._get_binary_extension_folder()
for fpath in folder.iterdir():
if fpath.name in ("params.json", "info.json", "run_info.json") or fpath.name.startswith("._"):
continue
name = fpath.stem
if fpath.suffix == ".json":
with fpath.open() as f:
self.set_data(name, json.load(f))
elif fpath.suffix == ".npy":
self.set_data(name, np.load(fpath))
elif fpath.suffix == ".csv" and HAS_PANDAS:
df = pd.read_csv(fpath, index_col=0)
roi_ids = self.roi_analyzer.roi_ids
if df.shape[0] == roi_ids.size and df.index.dtype != roi_ids.dtype:
df.index = df.index.astype(roi_ids.dtype)
self.set_data(name, df)
elif fpath.suffix == ".pkl":
with fpath.open("rb") as f:
self.set_data(name, pickle.load(f))
elif self.format == "zarr":
group = self._get_zarr_extension_group(mode="r")
for name in group.keys():
item = group[name]
if "dict" in item.attrs:
self.set_data(name, item[0])
elif "dataframe" in item.attrs:
if HAS_PANDAS:
index = item["index"]
df = pd.DataFrame(index=index)
for col in item.keys():
if col != "index":
df[col] = item[col][:]
self.set_data(name, df.convert_dtypes())
elif "object" in item.attrs:
self.set_data(name, item[0])
else:
self.set_data(name, np.array(item))
if len(self.data) == 0:
warnings.warn(f"No data for '{self.extension_name}'; extension should be re-computed.")
[docs]
def run(self, save=True, **kwargs):
if save and not self.roi_analyzer.is_read_only():
self._save_params()
self._save_importing_provenance()
t0 = perf_counter()
self._run(**kwargs)
self.run_info["runtime_s"] = perf_counter() - t0
self.run_info["run_completed"] = True
if save and not self.roi_analyzer.is_read_only():
self._save_run_info()
self._save_data()
if self.format == "zarr":
import zarr
zarr.consolidate_metadata(self.roi_analyzer._get_zarr_root().store)
[docs]
def save(self):
self._save_params()
self._save_importing_provenance()
self._save_run_info()
self._save_data()
if self.format == "zarr":
import zarr
zarr.consolidate_metadata(self.roi_analyzer._get_zarr_root().store)
[docs]
def copy(self, new_roi_analyzer, roi_ids=None):
"""Copy this extension to a new RoiAnalyzer, optionally filtering to roi_ids."""
new_ext = self.__class__(new_roi_analyzer)
new_ext.params = self.params.copy()
if roi_ids is None:
new_ext.data = self.data
else:
new_ext.data = self._select_extension_data(roi_ids)
new_ext.run_info = copy(self.run_info)
new_ext.save()
return new_ext
[docs]
def delete(self):
"""Delete extension from disk and clear in-memory data."""
self._delete_extension_folder()
self.params = None
self.run_info = self._default_run_info_dict()
self.data = dict()
[docs]
def reset(self):
"""Reset extension: clear data and recreate empty folder."""
self._reset_extension_folder()
self.params = None
self.run_info = self._default_run_info_dict()
self.data = dict()
[docs]
def set_params(self, save=True, **params):
"""Set parameters and optionally persist."""
if save:
self._reset_extension_folder()
params = self._set_params(**params)
self.params = params
if self.roi_analyzer.is_read_only():
return
if save:
self._save_params()
self._save_importing_provenance()
[docs]
def get_data(self, *args, **kwargs):
if self.run_info is not None:
assert self.run_info[
"run_completed"
], f"Extension '{self.extension_name}' must be run before retrieving data"
assert len(self.data) > 0, "Extension has been run but no data found."
return self._get_data(*args, **kwargs)
[docs]
def set_data(self, name, value):
self.data[name] = value
[docs]
def get_pipeline_nodes(self):
assert self.use_nodepipeline, "get_pipeline_nodes() requires use_nodepipeline=True"
return self._get_pipeline_nodes()
# ------------------------------------------------------------------
# Folder / zarr helpers
# ------------------------------------------------------------------
def _get_binary_extension_folder(self):
return self.folder / "extensions" / self.extension_name
def _get_zarr_extension_group(self, mode="r+"):
zarr_root = self.roi_analyzer._get_zarr_root(mode=mode)
return zarr_root["extensions"][self.extension_name]
def _reset_extension_folder(self):
if self.format == "binary_folder":
folder = self._get_binary_extension_folder()
if folder.is_dir():
shutil.rmtree(folder)
folder.mkdir(exist_ok=False, parents=True)
elif self.format == "zarr":
import zarr
zarr_root = self.roi_analyzer._get_zarr_root(mode="r+")
zarr_root["extensions"].create_group(self.extension_name, overwrite=True)
zarr.consolidate_metadata(zarr_root.store)
def _delete_extension_folder(self):
if self.format == "binary_folder":
folder = self._get_binary_extension_folder()
if folder.is_dir():
shutil.rmtree(folder)
elif self.format == "zarr":
import zarr
zarr_root = self.roi_analyzer._get_zarr_root(mode="r+")
if self.extension_name in zarr_root["extensions"]:
del zarr_root["extensions"][self.extension_name]
zarr.consolidate_metadata(zarr_root.store)
def _save_params(self):
params_to_save = self.params.copy()
self._reset_extension_folder()
if self.format == "binary_folder":
folder = self._get_binary_extension_folder()
folder.mkdir(exist_ok=True, parents=True)
(folder / "params.json").write_text(json.dumps(check_json(params_to_save), indent=4), encoding="utf8")
elif self.format == "zarr":
group = self._get_zarr_extension_group(mode="r+")
group.attrs["params"] = check_json(params_to_save)
def _save_importing_provenance(self):
info = retrieve_importing_provenance(self.__class__)
if self.format == "binary_folder":
folder = self._get_binary_extension_folder()
folder.mkdir(exist_ok=True, parents=True)
(folder / "info.json").write_text(json.dumps(info, indent=4), encoding="utf8")
elif self.format == "zarr":
group = self._get_zarr_extension_group(mode="r+")
group.attrs["info"] = info
def _save_run_info(self):
if self.run_info is None:
return
run_info = self.run_info.copy()
if self.format == "binary_folder":
folder = self._get_binary_extension_folder()
(folder / "run_info.json").write_text(json.dumps(run_info, indent=4), encoding="utf8")
elif self.format == "zarr":
group = self._get_zarr_extension_group(mode="r+")
group.attrs["run_info"] = run_info
def _save_data(self):
if self.format == "memory":
return
if self.roi_analyzer.is_read_only():
raise ValueError(f"RoiAnalyzer is read-only; cannot save extension '{self.extension_name}'")
try:
import pandas as pd
HAS_PANDAS = True
except ImportError:
HAS_PANDAS = False
if self.format == "binary_folder":
folder = self._get_binary_extension_folder()
for name, data in self.data.items():
if isinstance(data, dict):
with (folder / f"{name}.json").open("w") as f:
json.dump(check_json(data), f)
elif isinstance(data, np.ndarray):
data_file = folder / f"{name}.npy"
if isinstance(data, np.memmap) and data_file.exists():
pass # already on disk
else:
np.save(data_file, data)
elif HAS_PANDAS and isinstance(data, pd.DataFrame):
data.to_csv(folder / f"{name}.csv", index=True)
else:
try:
with (folder / f"{name}.pkl").open("wb") as f:
pickle.dump(data, f)
except Exception:
raise Exception(f"Could not save '{name}' as extension data")
elif self.format == "zarr":
import numcodecs
from spikeinterface.core.zarrextractors import get_default_zarr_compressor
saving_options = self.roi_analyzer._backend_options.get("saving_options", {})
group = self._get_zarr_extension_group(mode="r+")
if "compressor" not in saving_options:
saving_options["compressor"] = get_default_zarr_compressor()
for name, data in self.data.items():
if name in group:
del group[name]
if isinstance(data, (dict, list)):
group.create_dataset(
name=name,
data=np.array([check_json(data)], dtype=object),
object_codec=numcodecs.JSON(),
)
group[name].attrs["dict"] = True
elif isinstance(data, np.ndarray):
group.create_dataset(name=name, data=data, **saving_options)
elif HAS_PANDAS and isinstance(data, pd.DataFrame):
df_group = group.create_group(name)
indices = data.index.to_numpy()
if indices.dtype.kind == "O":
indices = indices.astype(str)
df_group.create_dataset(name="index", data=indices)
for col in data.columns:
col_data = data[col].to_numpy()
if col_data.dtype.kind == "O":
col_data = col_data.astype(str)
df_group.create_dataset(name=col, data=col_data)
df_group.attrs["dataframe"] = True
else:
# Wrap via an object array assignment rather than np.array([data], dtype=object):
# the latter still invokes data.__array__ for array-like objects (e.g. sparse
# arrays), which either densifies unexpectedly or raises outright.
boxed = np.empty(1, dtype=object)
boxed[0] = data
try:
group.create_dataset(
name=name,
data=boxed,
object_codec=numcodecs.Pickle(),
)
except Exception:
raise Exception(f"Could not save '{name}' as extension data")
group[name].attrs["object"] = True
# ------------------------------------------------------------------
# Function factory for backwards-compatible compute_xxx() helpers
# ------------------------------------------------------------------
[docs]
@classmethod
def function_factory(cls):
class FuncWrapper:
def __init__(self, extension_name):
self.extension_name = extension_name
def __call__(self, roi_analyzer, *args, **kwargs):
if not isinstance(roi_analyzer, RoiAnalyzer):
raise ValueError(f"compute_{self.extension_name}() needs a RoiAnalyzer instance")
ext = roi_analyzer.compute(cls.extension_name, *args, **kwargs)
return ext.get_data()
func = FuncWrapper(cls.extension_name)
func.__doc__ = cls.__doc__
return func
# ---------------------------------------------------------------------------
# Built-in extensions registry
# ---------------------------------------------------------------------------
_builtin_extensions: dict[str, str] = {
"fluorescence": "photon_mosaic.core",
"df_over_f": "photon_mosaic.core",
"deconvolution": "photon_mosaic.core",
"neuropil": "photon_mosaic.core",
"background": "photon_mosaic.core",
}