Integrate the saturation detector

Author: samuelgarcia

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 25.12.0
+    rev: 26.1.0
     hooks:
     -   id: black
         files: ^src/

doc/modules/core.rst

@@ -157,11 +157,36 @@ with 10 units:
     # times are not set, the samples are divided by the sampling frequency
 
 
-Internally, any sorting object can construct 2 internal caches:
-  1. a list (per segment) of dict (per unit) of numpy.array. This cache is useful when accessing spike trains on a unit
-     per unit basis across segments.
-  2. a unique numpy.array with structured dtype aka "spikes vector". This is useful for processing by small chunks of
-     time, like for extracting amplitudes from a recording.
+Efficiency and caching when fetching spiketrains
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By design a Sorting object is lazy so that the constructor is fast: no data are fetched during init.
+Sortings that read data (the extractors) or modify data (select unit, remove spikes) behave
+the same: they are lazy. The spiketrain data are only fetched when the user requests them, e.g.
+using `sorting.get_unit_spike_train()`. This can lead to very slow access depending on how the
+underlying data is organized, but with the advantage that the `Sorting` object is agnostic of
+the underlying data organization.
+
+To fix the slow access problem, we can create different internal representations of the spiketrain data and
+automatically cache them. Then the next time `sorting.get_unit_spike_train()` is called, the cached
+data is instantaneously returned. Further, we can create and cache different reorganizations of
+the spiketrain, which are optimally organized for specific types of calculation.
+
+Computations involving combined recording-sorting information, such as fetching recording chunks and
+spiketrain chunks to accumulate waveforms, are often quickest when spikes are time-ordered. For
+this use case, we  use an internal representation called the `spike_vector`. This is a unique buffer:
+a numpy.array with dtype `[("sample_index", "int64"), ("unit_index", "int64"), ("segment_index", "int64")]`.
+
+For computations which are done unit-by-unit, like computing isi-violations per unit, it is better that
+spikes from a single unit are concurrent in memory. For these other cases, we can re-order the
+`spike_vector` in different ways:
+  * order by unit, then segment, then sample
+  * order by segment, then unit, then sample
+
+This is done using `sorting.to_reordered_spike_vector()`. The first time a reordering is done, the
+reordered spiketrain is cached in memory by default. Users should rarely have to worry about these
+details, but developers should keep memory layout in mind when implementing new features.
+
 
 SortingAnalyzer
 ---------------

src/spikeinterface/benchmark/benchmark_base.py

@@ -65,6 +65,10 @@ def _check_cases(cls, cases, levels=None, reference=None):
                 assert isinstance(key, tuple), f"Case key {key} for cases is not homogeneous"
                 num_levels = len(reference)
                 assert len(key) == num_levels, f"Case key {key} for cases is not homogeneous, tuple negth differ"
+                if levels is None:
+                    levels = [f"level{i}" for i in range(num_levels)]
+                else:
+                    levels = list(levels)
             else:
                 raise ValueError("Keys for cases must str or tuple")
         return levels

src/spikeinterface/benchmark/benchmark_matching.py

@@ -12,7 +12,7 @@
 
 import numpy as np
 from .benchmark_base import Benchmark, BenchmarkStudy, MixinStudyUnitCount
-from spikeinterface.core.basesorting import minimum_spike_dtype
+from spikeinterface.core.base import minimum_spike_dtype
 
 
 class MatchingBenchmark(Benchmark):

src/spikeinterface/benchmark/benchmark_peak_detection.py

@@ -13,7 +13,7 @@
 import numpy as np
 from spikeinterface.core.job_tools import fix_job_kwargs, split_job_kwargs
 from .benchmark_base import Benchmark, BenchmarkStudy
-from spikeinterface.core.basesorting import minimum_spike_dtype
+from spikeinterface.core.base import minimum_spike_dtype
 from spikeinterface.core.sortinganalyzer import create_sorting_analyzer
 from .benchmark_plot_tools import fit_sigmoid, sigmoid
 

src/spikeinterface/benchmark/benchmark_sorter.py

@@ -8,7 +8,6 @@
 from spikeinterface.sorters import run_sorter
 from spikeinterface.comparison import compare_sorter_to_ground_truth
 
-
 # TODO later integrate CollisionGTComparison optionally in this class.
 
 

src/spikeinterface/benchmark/benchmark_sorter_without_gt.py

@@ -10,7 +10,6 @@
 
 from spikeinterface.benchmark import analyse_residual
 
-
 # TODO later integrate CollisionGTComparison optionally in this class.
 
 

src/spikeinterface/benchmark/tests/test_residual_analysis.py

@@ -7,7 +7,6 @@
 
 from spikeinterface.benchmark import analyse_residual
 
-
 job_kwargs = dict(n_jobs=-1, progress_bar=True)
 
 

src/spikeinterface/comparison/tests/test_multisortingcomparison.py

@@ -9,7 +9,6 @@
 from spikeinterface.extractors import NumpySorting
 from spikeinterface.comparison import compare_multiple_sorters, MultiSortingComparison
 
-
 ON_GITHUB = bool(os.getenv("GITHUB_ACTIONS"))
 
 

src/spikeinterface/comparison/tests/test_templatecomparison.py

@@ -5,7 +5,6 @@
 from spikeinterface.core import create_sorting_analyzer, generate_ground_truth_recording
 from spikeinterface.comparison import compare_templates, compare_multiple_templates
 
-
 # def setup_module():
 #     if test_dir.is_dir():
 #         shutil.rmtree(test_dir)