Implement ChunkExecutor

doc/api.rst

@@ -73,7 +73,7 @@ Low-level
 .. automodule:: spikeinterface.core
     :noindex:
 
-    .. autoclass:: ChunkRecordingExecutor
+    .. autoclass:: ChunkExecutor
 
 
 Back-compatibility with ``WaveformExtractor`` (version > 0.100.0)

src/spikeinterface/comparison/multicomparisons.py

@@ -222,7 +222,7 @@ def __init__(
 
         for segment_index in range(multisortingcomparison._num_segments):
             sorting_segment = AgreementSortingSegment(multisortingcomparison._spiketrains[segment_index])
-            self.add_sorting_segment(sorting_segment)
+            self.add_segment(sorting_segment)
 
         self._kwargs = dict(
             sampling_frequency=sampling_frequency,

src/spikeinterface/core/__init__.py

@@ -95,12 +95,12 @@
     get_best_job_kwargs,
     ensure_n_jobs,
     ensure_chunk_size,
-    ChunkRecordingExecutor,
+    ChunkExecutor,
     split_job_kwargs,
     fix_job_kwargs,
 )
+from .chunkable_tools import write_binary, write_memory
 from .recording_tools import (
-    write_binary_recording,
     write_to_h5_dataset_format,
     get_random_data_chunks,
     get_channel_distances,

src/spikeinterface/core/analyzer_extension_core.py

@@ -9,6 +9,8 @@
   * ComputeNoiseLevels which is very convenient to have
 """
 
+from __future__ import annotations
+
 import warnings
 import numpy as np
 from collections import namedtuple

src/spikeinterface/core/base.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+from abc import ABC, abstractmethod
 from pathlib import Path
 import shutil
 from typing import Any, Iterable, List, Optional, Sequence, Union
@@ -85,6 +86,9 @@ def __init__(self, main_ids: Sequence) -> None:
                 self._main_ids.dtype.kind in "uiSU"
             ), f"Main IDs can only be integers (signed/unsigned) or strings, not {self._main_ids.dtype}"
 
+        # segments
+        self._segments: List[BaseSegment] = []
+
         # dict at object level
         self._annotations = {}
 
@@ -140,9 +144,16 @@ def name(self, value):
             # we remove the annotation if it exists
             _ = self._annotations.pop("name", None)
 
+    @property
+    def segments(self) -> list:
+        return self._segments
+
+    def add_segment(self, segment: BaseSegment) -> None:
+        self._segments.append(segment)
+        segment.set_parent_extractor(self)
+
     def get_num_segments(self) -> int:
-        # This is implemented in BaseRecording or BaseSorting
-        raise NotImplementedError
+        return len(self._segments)
 
     def get_parent(self) -> Optional[BaseExtractor]:
         """Returns parent object if it exists, otherwise None"""
@@ -234,13 +245,6 @@ def set_annotation(self, annotation_key: str, value: Any, overwrite=False) -> No
             else:
                 raise ValueError(f"{annotation_key} is already an annotation key. Use 'overwrite=True' to overwrite it")
 
-    def get_preferred_mp_context(self):
-        """
-        Get the preferred context for multiprocessing.
-        If None, the context is set by the multiprocessing package.
-        """
-        return self._preferred_mp_context
-
     def get_annotation(self, key: str, copy: bool = True) -> Any:
         """
         Get a annotation.
@@ -429,8 +433,9 @@ def copy_metadata(
 
         other.extra_requirements.extend(self.extra_requirements)
 
-        if self._preferred_mp_context is not None:
-            other._preferred_mp_context = self._preferred_mp_context
+        # call all extra copy metadata if it exists (e.g., with chunkable mixin)
+        if hasattr(self, "_extra_copy_metadata"):
+            self._extra_copy_metadata(other, only_main=only_main, ids=ids, skip_properties=skip_properties)
 
     def to_dict(
         self,