Merge branch 'master' into merge_ap_lfp_neuropix

Author: DradeAW

.github/workflows/full-test.yml

@@ -154,7 +154,7 @@ jobs:
           python ./.github/build_job_summary.py report_core.txt >> $GITHUB_STEP_SUMMARY  
           rm report_core.txt
       - name: Test extractors
-        if: ${{ steps.modules-changed.outputs.EXTRACTORS_CHANGED == 'true' }}
+        if: ${{ steps.modules-changed.outputs.EXTRACTORS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
         run: |
           source ~/test_env/bin/activate
           pytest -m extractors -vv -ra --durations=0 --durations-min=0.001 | tee report_extractors.txt; test ${PIPESTATUS[0]} -eq 0 || exit 1

pyproject.toml

@@ -25,7 +25,6 @@ dependencies = [
     "threadpoolctl",
     "tqdm",
     "probeinterface>=0.2.16",
-    "psutil",
 ]
 
 [build-system]

spikeinterface/core/baserecording.py

@@ -43,8 +43,9 @@ def __repr__(self):
         nchan = self.get_num_channels()
         sf_khz = self.get_sampling_frequency() / 1000.
         duration = self.get_total_duration()
+        dtype = self.get_dtype()
         memory_size = self.get_memory_size()
-        txt = f"{clsname}: {nchan} channels - {nseg} segments - {sf_khz:0.1f}kHz - {duration:0.3f}s - {memory_size}"
+        txt = f"{clsname}: {nchan} channels - {nseg} segments - {sf_khz:0.1f}kHz - {duration:0.3f}s - {dtype} type - {memory_size}"
         if 'file_paths' in self._kwargs:
             txt += '\n  file_paths: {}'.format(self._kwargs['file_paths'])
         if 'file_path' in self._kwargs:

spikeinterface/core/core_tools.py

@@ -793,8 +793,10 @@ def recursive_path_modifier(d, func, target='path', copy=True):
     else:
         for k, v in d.items():
             if target in k:
-                # paths can be str or list of str
-                if isinstance(v, str):
+                # paths can be str or list of str or None
+                if v is None:
+                    continue
+                if isinstance(v, (str, Path)):
                     dc[k] =func(v)
                 elif isinstance(v, list):
                     dc[k] = [func(e) for e in v]
@@ -815,13 +817,32 @@ def recursive_key_finder(d, key):
 
 def convert_bytes_to_str(byte_value:int ) -> str:
     """
-    Converts a number of bytes to a value in either KiB, MiB, GiB, or TiB.
+    Convert a number of bytes to a human-readable string with an appropriate unit.
 
-    Args:
-        byte_value (int): The number of bytes to convert.
+    This function converts a given number of bytes into a human-readable string
+    representing the value in either bytes (B), kibibytes (KiB), mebibytes (MiB),
+    gibibytes (GiB), or tebibytes (TiB). The function uses the IEC binary prefixes
+    (1 KiB = 1024 B, 1 MiB = 1024 KiB, etc.) to determine the appropriate unit.
 
-    Returns:
-        str: The converted value with the appropriate unit (KiB, MiB, GiB, or TiB).
+    Parameters
+    ----------
+    byte_value : int
+        The number of bytes to convert.
+
+    Returns
+    -------
+    str
+        The converted value as a formatted string with two decimal places,
+        followed by a space and the appropriate unit (B, KiB, MiB, GiB, or TiB).
+
+    Examples
+    --------
+    >>> convert_bytes_to_str(1024)
+    '1.00 KiB'
+    >>> convert_bytes_to_str(1048576)
+    '1.00 MiB'
+    >>> convert_bytes_to_str(45056)
+    '43.99 KiB'
     """
     suffixes = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
     i = 0

spikeinterface/core/recording_tools.py

@@ -1,8 +1,15 @@
 import numpy as np
 
 
-def get_random_data_chunks(recording, return_scaled=False, num_chunks_per_segment=20, 
-                           chunk_size=10000, concatenated=True, seed=0, margin_frames=0):
+def get_random_data_chunks(
+    recording,
+    return_scaled=False,
+    num_chunks_per_segment=20,
+    chunk_size=10000,
+    concatenated=True,
+    seed=0,
+    margin_frames=0,
+):
     """
     Exctract random chunks across segments
 
@@ -31,22 +38,30 @@ def get_random_data_chunks(recording, return_scaled=False, num_chunks_per_segmen
     # Should be done by changing kwargs with total_num_chunks=XXX and total_duration=YYYY
     # And randomize the number of chunk per segment weighted by segment duration
 
-    # check chunk size    
+    # check chunk size
     for segment_index in range(recording.get_num_segments()):
-        assert chunk_size < recording.get_num_samples(segment_index), (f"chunk_size is greater than the number "
-                                                                        f"of samples for segment index {segment_index}. "
-                                                                        f"Use a smaller chunk_size!")
+        assert chunk_size < recording.get_num_samples(segment_index), (
+            f"chunk_size is greater than the number "
+            f"of samples for segment index {segment_index}. "
+            f"Use a smaller chunk_size!"
+        )
 
     chunk_list = []
     for segment_index in range(recording.get_num_segments()):
         length = recording.get_num_frames(segment_index)
-        
-        random_starts = np.random.RandomState(seed=seed).randint(margin_frames, length - chunk_size - margin_frames, size=num_chunks_per_segment)
+
+        random_starts = np.random.RandomState(seed=seed).randint(
+            margin_frames,
+            length - chunk_size - margin_frames,
+            size=num_chunks_per_segment,
+        )
         for start_frame in random_starts:
-            chunk = recording.get_traces(start_frame=start_frame,
-                                         end_frame=start_frame + chunk_size,
-                                         segment_index=segment_index,
-                                         return_scaled=return_scaled)
+            chunk = recording.get_traces(
+                start_frame=start_frame,
+                end_frame=start_frame + chunk_size,
+                segment_index=segment_index,
+                return_scaled=return_scaled,
+            )
             chunk_list.append(chunk)
     if concatenated:
         return np.concatenate(chunk_list, axis=0)
@@ -59,7 +74,9 @@ def get_channel_distances(recording):
     Distance between channel pairs
     """
     locations = recording.get_channel_locations()
-    channel_distances = np.linalg.norm(locations[:, np.newaxis] - locations[np.newaxis, :], axis=2)
+    channel_distances = np.linalg.norm(
+        locations[:, np.newaxis] - locations[np.newaxis, :], axis=2
+    )
 
     return channel_distances
 
@@ -95,40 +112,59 @@ def get_closest_channels(recording, channel_ids=None, num_channels=None):
     for i in range(locations.shape[0]):
         distances = np.linalg.norm(locations[i, :] - locations, axis=1)
         order = np.argsort(distances)
-        closest_channels_inds.append(order[1:num_channels + 1])
-        dists.append(distances[order][1:num_channels + 1])
+        closest_channels_inds.append(order[1 : num_channels + 1])
+        dists.append(distances[order][1 : num_channels + 1])
 
     return np.array(closest_channels_inds), np.array(dists)
 
 
 def get_noise_levels(recording, return_scaled=True, **random_chunk_kwargs):
     """
     Estimate noise for each channel using MAD methods.
-    
+
     Internally it sample some chunk across segment.
     And then, it use MAD estimator (more robust than STD)
-    
+
     """
-    random_chunks = get_random_data_chunks(recording, return_scaled=return_scaled, **random_chunk_kwargs)
+    random_chunks = get_random_data_chunks(
+        recording, return_scaled=return_scaled, **random_chunk_kwargs
+    )
     med = np.median(random_chunks, axis=0, keepdims=True)
     # hard-coded so that core doesn't depend on scipy
-    noise_levels = np.median(np.abs(random_chunks - med), axis=0) / 0.6744897501960817
+    noise_levels = (
+        np.median(np.abs(random_chunks - med), axis=0) / 0.6744897501960817
+    )
     return noise_levels
 
 
-def get_chunk_with_margin(rec_segment, start_frame, end_frame,
-                          channel_indices, margin, add_zeros=False, 
-                          window_on_margin=False, dtype=None):
+def get_chunk_with_margin(
+    rec_segment,
+    start_frame,
+    end_frame,
+    channel_indices,
+    margin,
+    add_zeros=False,
+    add_reflect_padding=False,
+    window_on_margin=False,
+    dtype=None,
+):
     """
     Helper to get chunk with margin
+
+    The margin is extracted from the recording when possible. If
+    at the edge of the recording, no margin is used unless one
+    of `add_zeros` or `add_reflect_padding` is True. In the first
+    case zero padding is used, in the second case np.pad is called
+    with mod="reflect".
     """
     length = rec_segment.get_num_samples()
 
     if channel_indices is None:
         channel_indices = slice(None)
 
-    if not add_zeros:
-        assert not window_on_margin, 'window_on_margin can be used only for add_zeros=True'
+    if not (add_zeros or add_reflect_padding):
+        if window_on_margin and not add_zeros:
+            raise ValueError("window_on_margin requires add_zeros=True")
         if start_frame is None:
             left_margin = 0
             start_frame = 0
@@ -144,10 +180,14 @@ def get_chunk_with_margin(rec_segment, start_frame, end_frame,
             right_margin = length - end_frame
         else:
             right_margin = margin
-        traces_chunk = rec_segment.get_traces(start_frame - left_margin, end_frame + right_margin, channel_indices)
+        traces_chunk = rec_segment.get_traces(
+            start_frame - left_margin,
+            end_frame + right_margin,
+            channel_indices,
+        )
 
     else:
-        # add_zeros=True
+        # either add_zeros or reflect_padding
         assert start_frame is not None
         assert end_frame is not None
         chunk_size = end_frame - start_frame
@@ -167,41 +207,66 @@ def get_chunk_with_margin(rec_segment, start_frame, end_frame,
             end_frame2 = end_frame + margin
             right_pad = 0
 
-        traces_chunk = rec_segment.get_traces(start_frame2, end_frame2, channel_indices)
-        
-        
-        if dtype is not None or window_on_margin or left_pad > 0 or right_pad > 0:
+        traces_chunk = rec_segment.get_traces(
+            start_frame2, end_frame2, channel_indices
+        )
+
+        if (
+            dtype is not None
+            or window_on_margin
+            or left_pad > 0
+            or right_pad > 0
+        ):
             need_copy = True
         else:
             need_copy = False
 
+        left_margin = margin
+        right_margin = margin
+
         if need_copy:
             if dtype is None:
                 dtype = traces_chunk.dtype
-            traces_chunk2 = np.zeros((full_size, traces_chunk.shape[1]), dtype=dtype)
-            i0 = left_pad
-            i1 = left_pad + traces_chunk.shape[0]
-            traces_chunk2[i0: i1, :] = traces_chunk
+
             left_margin = margin
             if end_frame < (length + margin):
                 right_margin = margin
             else:
                 right_margin = end_frame + margin - length
-            if window_on_margin:
-                # apply inplace taper  on border
-                taper = (1 - np.cos(np.arange(margin) / margin * np.pi)) / 2
-                taper = taper[:, np.newaxis]
-                traces_chunk2[:margin] *= taper
-                traces_chunk2[-margin:] *= taper[::-1]
-            traces_chunk = traces_chunk2
-        else:
-            left_margin = margin
-            right_margin = margin
+
+            if add_zeros:
+                traces_chunk2 = np.zeros(
+                    (full_size, traces_chunk.shape[1]), dtype=dtype
+                )
+                i0 = left_pad
+                i1 = left_pad + traces_chunk.shape[0]
+                traces_chunk2[i0:i1, :] = traces_chunk
+                if window_on_margin:
+                    # apply inplace taper on border
+                    taper = (
+                        1 - np.cos(np.arange(margin) / margin * np.pi)
+                    ) / 2
+                    taper = taper[:, np.newaxis]
+                    traces_chunk2[:margin] *= taper
+                    traces_chunk2[-margin:] *= taper[::-1]
+                traces_chunk = traces_chunk2
+            elif add_reflect_padding:
+                # in this case, we don't want to taper
+                traces_chunk = np.pad(
+                    traces_chunk.astype(dtype),
+                    [(left_pad, right_pad), (0, 0)],
+                    mode="reflect",
+                )
+            else:
+                # we need a copy to change the dtype
+                traces_chunk = np.asarray(traces_chunk, dtype=dtype)
 
     return traces_chunk, left_margin, right_margin
 
 
-def order_channels_by_depth(recording, channel_ids=None, dimensions=('x', 'y')):
+def order_channels_by_depth(
+    recording, channel_ids=None, dimensions=("x", "y")
+):
     """
     Order channels by depth, by first ordering the x-axis, and then the y-axis.
 
@@ -213,7 +278,7 @@ def order_channels_by_depth(recording, channel_ids=None, dimensions=('x', 'y')):
         If given, a subset of channels to order locations for
     dimensions : str or tuple
         If str, it needs to be 'x', 'y', 'z'.
-        If tuple, it sorts the locations in two dimensions using lexsort. 
+        If tuple, it sorts the locations in two dimensions using lexsort.
         This approach is recommended since there is less ambiguity, by default ('x', 'y')
 
     Returns
@@ -229,18 +294,20 @@ def order_channels_by_depth(recording, channel_ids=None, dimensions=('x', 'y')):
     locations = locations[channel_inds, :]
 
     if isinstance(dimensions, str):
-        dim = ['x', 'y', 'z'].index(dimensions)
+        dim = ["x", "y", "z"].index(dimensions)
         assert dim < ndim, "Invalid dimensions!"
-        order_f = np.argsort(locations[:, dim])
+        order_f = np.argsort(locations[:, dim], kind="stable")
     else:
-        assert isinstance(dimensions, tuple), "dimensions can be a str or a tuple"
+        assert isinstance(
+            dimensions, tuple
+        ), "dimensions can be a str or a tuple"
         locations_to_sort = ()
         for dim in dimensions:
-            dim = ['x', 'y', 'z'].index(dim)
+            dim = ["x", "y", "z"].index(dim)
             assert dim < ndim, "Invalid dimensions!"
-            locations_to_sort += (locations[:, dim], )
+            locations_to_sort += (locations[:, dim],)
         order_f = np.lexsort(locations_to_sort)
-    order_r = np.argsort(order_f)
+    order_r = np.argsort(order_f, kind="stable")
 
     return order_f, order_r
 
@@ -253,21 +320,27 @@ def check_probe_do_not_overlap(probes):
     for i in range(len(probes)):
         probe_i = probes[i]
         # check that all positions in probe_j are outside probe_i boundaries
-        x_bounds_i = [np.min(probe_i.contact_positions[:, 0]),
-                      np.max(probe_i.contact_positions[:, 0])]
-        y_bounds_i = [np.min(probe_i.contact_positions[:, 1]),
-                      np.max(probe_i.contact_positions[:, 1])]
+        x_bounds_i = [
+            np.min(probe_i.contact_positions[:, 0]),
+            np.max(probe_i.contact_positions[:, 0]),
+        ]
+        y_bounds_i = [
+            np.min(probe_i.contact_positions[:, 1]),
+            np.max(probe_i.contact_positions[:, 1]),
+        ]
 
         for j in range(i + 1, len(probes)):
             probe_j = probes[j]
 
-            if np.any(np.array([x_bounds_i[0] < cp[0] < x_bounds_i[1] and
-                                y_bounds_i[0] < cp[1] < y_bounds_i[1]
-                                for cp in probe_j.contact_positions])):
+            if np.any(
+                np.array(
+                    [
+                        x_bounds_i[0] < cp[0] < x_bounds_i[1]
+                        and y_bounds_i[0] < cp[1] < y_bounds_i[1]
+                        for cp in probe_j.contact_positions
+                    ]
+                )
+            ):
                 raise Exception(
-                    "Probes are overlapping! Retrieve locations of single probes separately")
-
-
-
-
-
+                    "Probes are overlapping! Retrieve locations of single probes separately"
+                )