import json
import mmap
import shutil
import warnings
from pathlib import Path
import numpy as np
from spikeinterface.core.core_tools import (
load_annotations_from_folder,
load_properties_from_folder,
save_annotations_to_folder,
save_properties_to_folder,
save_provenance_to_folder,
)
from .baseimaging import BaseImaging, BaseImagingEpoch
[docs]
class BinaryImaging(BaseImaging):
"""
ImagingExtractor for a binary format
Parameters
----------
file_paths : str or Path or list
Path to the binary file
sampling_frequency : float
The sampling frequency
shape : tuple(int, int) | tuple(int, int, int)
Image height, width, and optionally number of planes
dtype : str or dtype
The dtype of the binary file
t_starts : None or list of float, default: None
Times in seconds of the first sample for each epoch. If None, defaults to 0 for all epochs.
file_offset : int, default: 0
Number of bytes in the file to offset by during memmap instantiation.
file_timestamps_paths : str or Path or list, default: None
Path to the binary file containing timestamps for each segment. If None, timestamps are not loaded
Returns
-------
imaging : BinaryImaging
The imaging object
"""
def __init__(
self,
file_paths,
sampling_frequency,
shape,
dtype,
t_starts=None,
file_offset=0,
file_timestamps_paths: str | Path | list[str | Path] | None = None,
):
BaseImaging.__init__(self, sampling_frequency, shape, dtype)
if isinstance(file_paths, list):
# several epochs
file_path_list = [Path(p) for p in file_paths]
else:
# one epoch
file_path_list = [Path(file_paths)]
if t_starts is not None:
assert len(t_starts) == len(file_path_list), "t_starts must be a list of the same size as file_paths"
t_starts = [float(t_start) for t_start in t_starts]
dtype = np.dtype(dtype)
if file_timestamps_paths is None:
timestamps_path_list: list[str | Path] | None = None
elif isinstance(file_timestamps_paths, list):
timestamps_path_list = file_timestamps_paths
else:
timestamps_path_list = [file_timestamps_paths]
for i, file_path in enumerate(file_path_list):
if t_starts is None:
t_start = None
else:
t_start = t_starts[i]
if timestamps_path_list is None:
file_timestamps_path = None
else:
file_timestamps_path = timestamps_path_list[i]
imaging_epoch = BinaryImagingEpoch(
file_path, sampling_frequency, t_start, shape, dtype, file_offset, file_timestamps_path
)
self.add_epoch(imaging_epoch)
self._kwargs = {
"file_paths": [str(Path(e).absolute()) for e in file_path_list],
"sampling_frequency": sampling_frequency,
"t_starts": t_starts,
"shape": shape,
"dtype": dtype.str,
"file_offset": file_offset,
}
[docs]
def is_binary_compatible(self) -> bool:
return True
[docs]
def get_binary_description(self):
d = dict(
file_paths=self._kwargs["file_paths"],
dtype=np.dtype(self._kwargs["dtype"]),
shape=self._kwargs["shape"],
file_offset=self._kwargs["file_offset"],
)
return d
def __del__(self): # pragma: no cover
"""
Ensures that all epoch resources are properly cleaned up when this imaging extractor is deleted.
Closes any open file handles in the imaging epochs.
"""
# Close all imaging epochs
if hasattr(self, "epochs"):
for epoch in self.epochs:
# This will trigger the __del__ method of the BaseImagingEpoch
# which will close the file handle
del epoch
[docs]
class BinaryImagingEpoch(BaseImagingEpoch):
def __init__(self, file_path, sampling_frequency, t_start, shape, dtype, file_offset, file_timestamps_path):
BaseImagingEpoch.__init__(self, sampling_frequency=sampling_frequency, t_start=t_start)
self.shape = shape
self.dtype = np.dtype(dtype)
self.file_offset = file_offset
self.file_path = file_path
self.file = open(self.file_path, "rb")
self.bytes_per_sample = np.prod(shape) * self.dtype.itemsize
self.data_size_in_bytes = Path(file_path).stat().st_size - file_offset
self.num_samples = self.data_size_in_bytes // self.bytes_per_sample
if file_timestamps_path is not None:
self._time_vector = np.memmap(file_timestamps_path, dtype="float64", mode="r", shape=(self.num_samples,))
[docs]
def get_num_samples(self) -> int:
"""Returns the number of samples in this signal block
Returns:
SampleIndex : Number of samples in the signal block
"""
return self.num_samples
[docs]
def get_series(
self,
start_frame: int,
end_frame: int,
plane_indices: slice | np.ndarray | None = None,
) -> np.ndarray:
# Calculate byte offsets for start and end frames
start_byte = self.file_offset + start_frame * self.bytes_per_sample
end_byte = self.file_offset + end_frame * self.bytes_per_sample
# Calculate the length of the data chunk to load into memory
length = end_byte - start_byte
# The mmap offset must be a multiple of mmap.ALLOCATIONGRANULARITY
memmap_offset, start_offset = divmod(start_byte, mmap.ALLOCATIONGRANULARITY)
memmap_offset *= mmap.ALLOCATIONGRANULARITY
# Adjust the length so it includes the extra data from rounding down
# the memmap offset to a multiple of ALLOCATIONGRANULARITY
length += start_offset
# Create the mmap object
memmap_obj = mmap.mmap(
self.file.fileno(),
length=length,
access=mmap.ACCESS_READ,
offset=memmap_offset,
)
# Create a numpy array using the mmap object as the buffer
# Note that the shape must be recalculated based on the new data chunk
shape: tuple[int, int, int, int]
shape = (
(end_frame - start_frame),
self.shape[0],
self.shape[1],
self.shape[2],
# We could also read only the specific planes here
# if we implemented more complex memory mapping offsets
)
# Now the entire array should correspond to the data between start_frame and end_frame,
# so we can use it directly
series = np.ndarray(
shape=shape,
dtype=self.dtype,
buffer=memmap_obj,
offset=start_offset,
)
# Slice planes if needed
series = series[:, :, :, plane_indices] if plane_indices is not None else series
return series
def __del__(self): # pragma: no cover
# Ensure that the file handle is closed when the epoch is garbage-collected
try:
if hasattr(self, "file") and self.file and not self.file.closed:
self.file.close()
except Exception as e:
warnings.warn(f"Error closing file handle in BaseImagingEpoch: {e}")
pass
# For backward compatibility (old good time)
read_binary = BinaryImaging
[docs]
class BinaryFolderImaging(BinaryImaging):
"""
BinaryFolderImaging is an internal format used in photon-mosaic.
It is a BinaryImaging + metadata contained in a folder.
It is created with the function: `imaging.save(format="binary", folder="/myfolder")`
Parameters
----------
folder_path : str or Path
Returns
-------
imaging : BinaryFolderImaging
The imaging object
"""
def __init__(self, folder_path):
from spikeinterface.core.core_tools import make_paths_absolute
folder_path = Path(folder_path)
with open(folder_path / "binary.json", "r") as f:
d = json.load(f)
if not d["class"].endswith(".BinaryImaging"):
raise ValueError("This folder is not a binary photon-mosaic folder")
assert d["relative_paths"]
d = make_paths_absolute(d, folder_path)
BinaryImaging.__init__(self, **d["kwargs"])
# Load properties and annotations
load_properties_from_folder(folder_path / "properties", self)
load_annotations_from_folder(folder_path, self)
self._kwargs = dict(folder_path=str(Path(folder_path).absolute()))
self._bin_kwargs = d["kwargs"]
[docs]
def is_binary_compatible(self) -> bool:
return True
[docs]
def get_binary_description(self):
d = dict(
file_paths=self._bin_kwargs["file_paths"],
dtype=np.dtype(self._bin_kwargs["dtype"]),
shape=self._bin_kwargs["shape"],
file_offset=self._bin_kwargs["file_offset"],
)
return d
[docs]
@staticmethod
def write_imaging(
imaging: BaseImaging,
folder_path: str | Path,
verbose: bool = False,
overwrite: bool = False,
dtype=None,
**job_kwargs,
):
"""Write imaging data to a folder in binary format.
Each epoch is written to its own `.raw` file.
Parameters
----------
imaging : BaseImaging
Imaging object to write.
folder_path : str | Path
Destination folder where binary files are saved.
verbose : bool, default: False
If ``True``, enables verbose output during writing.
overwrite : bool, default: False
If ``True``, removes an existing destination folder before writing.
dtype : dtype, optional
Data type used to store trace data. If ``None``, uses
``imaging.get_dtype()``.
**job_kwargs
Additional keyword arguments forwarded to
:func:`spikeinterface.core.time_series_tools.write_binary`.
Returns
-------
Path
Path to the written `BinaryFolderImaging`.
Notes
-----
Implemented as a static method so it can be called by
:meth:`BaseImaging.save` without instantiating :class:`BinaryImaging`.
This is its only intended use.
"""
from spikeinterface.core.time_series_tools import write_binary
folder_path = Path(folder_path)
if folder_path.is_dir():
if not overwrite:
raise FileExistsError(f"Folder {folder_path} already exists. Use overwrite=True to overwrite it.")
else:
shutil.rmtree(folder_path)
folder_path.mkdir(exist_ok=False, parents=True)
file_paths = [folder_path / f"traces_cached_seg{i}.raw" for i in range(imaging.get_num_epochs())]
if dtype is None:
dtype = imaging.get_dtype()
# Check if there are any time vectors
t_starts = imaging.get_segment_t_starts()
if imaging.has_any_time_vector():
file_timestamps_paths: list[str | Path] | None = [
folder_path / f"times_cached_seg{i}.raw" for i in range(imaging.get_num_epochs())
]
else:
file_timestamps_paths = None
write_binary(
imaging,
file_paths=file_paths,
file_timestamps_paths=file_timestamps_paths,
dtype=dtype,
verbose=verbose,
**job_kwargs,
)
save_provenance_to_folder(folder_path, imaging, relative_to=folder_path)
save_properties_to_folder(folder_path / "properties", imaging)
save_annotations_to_folder(folder_path, imaging)
# This is created so it can be saved as json because the `BinaryFolderRecording` requires it loading
# See the __init__
binary_imaging = BinaryImaging(
file_paths=file_paths,
file_timestamps_paths=file_timestamps_paths,
shape=imaging.shape,
sampling_frequency=imaging.get_sampling_frequency(),
dtype=dtype,
t_starts=t_starts,
file_offset=0,
)
binary_imaging.dump(folder_path / "binary.json", relative_to=folder_path)
# Create the si_folder file to make the load() easier until version 0.105.0
# All properties, annotations, and probe information are already saved in the folder,
# so we don't need to include them in the si_folder.json
cached = BinaryFolderImaging(folder_path=folder_path)
si_folder_path = folder_path / "si_folder.json"
cached.dump_to_json(
file_path=si_folder_path,
relative_to=folder_path,
include_properties=False,
include_annotations=False,
include_extra_metadata=False,
)
return cached
read_binary_folder = BinaryFolderImaging