Skip to content

Commit c9fc8e1

Browse files
authored
Merge pull request #3062 from samuelgarcia/dredge_lfp
Dredge lfp and dredge ap
2 parents ad3e924 + abb6a7c commit c9fc8e1

39 files changed

+3822
-1880
lines changed

doc/api.rst

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -407,12 +407,6 @@ Peak Detection
407407

408408
.. autofunction:: detect_peaks
409409

410-
Motion Correction
411-
~~~~~~~~~~~~~~~~~
412-
.. automodule:: spikeinterface.sortingcomponents.motion_interpolation
413-
414-
.. autoclass:: InterpolateMotionRecording
415-
416410
Clustering
417411
~~~~~~~~~~
418412
.. automodule:: spikeinterface.sortingcomponents.clustering
@@ -424,3 +418,15 @@ Template Matching
424418
.. automodule:: spikeinterface.sortingcomponents.matching
425419

426420
.. autofunction:: find_spikes_from_templates
421+
422+
Motion Correction
423+
~~~~~~~~~~~~~~~~~
424+
.. automodule:: spikeinterface.sortingcomponents.motion
425+
426+
.. autoclass:: Motion
427+
.. autofunction:: estimate_motion
428+
.. autofunction:: interpolate_motion
429+
.. autofunction:: correct_motion_on_peaks
430+
.. autofunction:: interpolate_motion_on_traces
431+
.. autofunction:: clean_motion_vector
432+
.. autoclass:: InterpolateMotionRecording

doc/how_to/benchmark_with_hybrid_recordings.rst

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ order to smoothly inject spikes into the recording.
2424
import spikeinterface.generation as sgen
2525
import spikeinterface.widgets as sw
2626
27-
from spikeinterface.sortingcomponents.motion_estimation import estimate_motion
27+
from spikeinterface.sortingcomponents.motion import estimate_motion
2828
2929
import numpy as np
3030
import matplotlib.pyplot as plt
@@ -1202,63 +1202,63 @@ drifts when injecting hybrid spikes.
12021202
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12031203
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12041204
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1205-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1205+
1. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12061206
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12071207
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12081208
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1209-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1209+
2. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12101210
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12111211
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12121212
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1213-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1213+
3. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12141214
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12151215
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12161216
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1217-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1217+
4. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12181218
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12191219
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12201220
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1221-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1221+
5. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12221222
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12231223
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12241224
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1225-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1225+
6. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12261226
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12271227
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12281228
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1229-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1229+
7. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12301230
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12311231
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12321232
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1233-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1233+
8. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12341234
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12351235
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12361236
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1237-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1237+
9. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12381238
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12391239
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12401240
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1241-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1241+
10. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12421242
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12431243
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12441244
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1245-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1245+
11. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12461246
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12471247
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12481248
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1249-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1249+
12. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12501250
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12511251
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12521252
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1253-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1253+
13. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12541254
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12551255
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12561256
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1257-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1257+
14. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12581258
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12591259
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12601260
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385
1261-
0. 0. 0.07692308 0.07692308 0.15384615 0.15384615
1261+
15. 0. 0.07692308 0.07692308 0.15384615 0.15384615
12621262
0.23076923 0.23076923 0.30769231 0.30769231 0.38461538 0.38461538
12631263
0.46153846 0.46153846 0.53846154 0.53846154 0.61538462 0.61538462
12641264
0.69230769 0.69230769 0.76923077 0.76923077 0.84615385 0.84615385]</details></ul></details>

doc/how_to/drift_with_lfp.rst

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
Estimate drift using the LFP traces
2+
===================================
3+
4+
Drift is a well known issue for long shank probes. Some datasets, especially from primates and humans,
5+
can experience very fast motion due to breathing and heart beats. In these cases, the standard motion
6+
estimation methods that use detected spikes as a basis for motion inference will fail, because there
7+
are not enough spikes to "follow" such fast drifts.
8+
9+
Charlie Windolf and colleagues from the Paninski Lab at Columbia have developed a method to estimate
10+
the motion using the LFP signal: **DREDge**. (more details about the method in the paper
11+
`DREDge: robust motion correction for high-density extracellular recordings across species <https://doi.org/10.1101/2023.10.24.563768>`_).
12+
13+
This method is particularly suited for the open dataset recorded at Massachusetts General Hospital by Angelique Paulk and colleagues in humans (more details in the [paper](https://doi.org/10.1038/s41593-021-00997-0)). The dataset can be dowloaed from [datadryad](https://datadryad.org/stash/dataset/doi:10.5061/dryad.d2547d840) and it contains recordings on human patients with a Neuropixels probe, some of which with very high and fast motion on the probe, which prevents accurate spike sorting without a proper and adequate motion correction
14+
15+
The **DREDge** method has two options: **dredge_lfp** and **dredge_ap**, which have both been ported inside `SpikeInterface`.
16+
17+
Here we will demonstrate the **dredge_lfp** method to estimate the fast and high drift on this recording.
18+
19+
For each patient, the dataset contains two streams:
20+
21+
* a highpass "action potential" (AP), sampled at 30kHz
22+
* a lowpass "local field" (LF) sampled at 2.5kHz
23+
24+
For this demonstration, we will use the LF stream.
25+
26+
.. code:: ipython3
27+
28+
%matplotlib inline
29+
%load_ext autoreload
30+
%autoreload 2
31+
32+
.. code:: ipython3
33+
34+
from pathlib import Path
35+
import matplotlib.pyplot as plt
36+
37+
import spikeinterface.full as si
38+
from spikeinterface.sortingcomponents.motion import estimate_motion
39+
40+
.. code:: ipython3
41+
42+
# the dataset has been locally downloaded
43+
base_folder = Path("/mnt/data/sam/DataSpikeSorting/")
44+
np_data_drift = base_folder / 'human_neuropixel/Pt02/'
45+
46+
Read the spikeglx file
47+
~~~~~~~~~~~~~~~~~~~~~~
48+
49+
.. code:: ipython3
50+
51+
raw_rec = si.read_spikeglx(np_data_drift)
52+
print(raw_rec)
53+
54+
55+
.. parsed-literal::
56+
57+
SpikeGLXRecordingExtractor: 384 channels - 2.5kHz - 1 segments - 2,183,292 samples
58+
873.32s (14.56 minutes) - int16 dtype - 1.56 GiB
59+
60+
61+
Preprocessing
62+
~~~~~~~~~~~~~
63+
64+
Contrary to the **dredge_ap** approach, which needs detected peaks and peak locations, the **dredge_lfp**
65+
method is estimating the motion directly on traces.
66+
Importantly, the method requires some additional pre-processing steps:
67+
* ``bandpass_filter``: to "focus" the signal on a particular band
68+
* ``phase_shift``: to compensate for the sampling misalignement
69+
* ``resample``: to further reduce the sampling fequency of the signal and speed up the computation. The sampling frequency of the estimated motion will be the same as the resampling frequency. Here we choose 250Hz, which corresponds to a sampling interval of 4ms.
70+
* ``directional_derivative``: this optional step applies a second order derivative in the spatial dimension to enhance edges on the traces.
71+
This is not a general rules and need to be tested case by case.
72+
* ``average_across_direction``: Neuropixels 1.0 probes have two contacts per depth. This steps averages them to obtain a unique virtual signal along the probe depth ("y" in ``spikeinterface``).
73+
74+
After appying this preprocessing chain, the motion can be estimated almost by eyes ont the traces plotted with the map mode.
75+
76+
.. code:: ipython3
77+
78+
lfprec = si.bandpass_filter(
79+
raw_rec,
80+
freq_min=0.5,
81+
freq_max=250,
82+
83+
margin_ms=1500.,
84+
filter_order=3,
85+
dtype="float32",
86+
add_reflect_padding=True,
87+
)
88+
lfprec = si.phase_shift(lfprec)
89+
lfprec = si.resample(lfprec, resample_rate=250, margin_ms=1000)
90+
91+
lfprec = si.directional_derivative(lfprec, order=2, edge_order=1)
92+
lfprec = si.average_across_direction(lfprec)
93+
94+
print(lfprec)
95+
96+
97+
.. parsed-literal::
98+
99+
AverageAcrossDirectionRecording: 192 channels - 0.2kHz - 1 segments - 218,329 samples
100+
873.32s (14.56 minutes) - float32 dtype - 159.91 MiB
101+
102+
103+
.. code:: ipython3
104+
105+
%matplotlib inline
106+
si.plot_traces(lfprec, backend="matplotlib", mode="map", clim=(-0.05, 0.05), time_range=(400, 420))
107+
108+
109+
110+
.. image:: drift_with_lfp_files/drift_with_lfp_8_1.png
111+
112+
113+
Run the method
114+
~~~~~~~~~~~~~~
115+
116+
``estimate_motion()`` is the generic function to estimate motion with multiple
117+
methods in ``spikeinterface``.
118+
119+
This function returns a ``Motion`` object and we can notice that the interval is exactly
120+
the same as downsampled signal.
121+
122+
Here we use ``rigid=True``, which means that we have one unqiue signal to
123+
describe the motion across the entire probe depth.
124+
125+
.. code:: ipython3
126+
127+
motion = estimate_motion(lfprec, method='dredge_lfp', rigid=True, progress_bar=True)
128+
motion
129+
130+
131+
.. parsed-literal::
132+
133+
Online chunks [10.0s each]: 0%| | 0/87 [00:00<?, ?it/s]
134+
135+
136+
.. parsed-literal::
137+
138+
Motion rigid - interval 0.004s - 1 segments
139+
140+
141+
142+
Plot the drift
143+
~~~~~~~~~~~~~~
144+
145+
When plotting the drift, we can notice a very fast drift which corresponds to the heart rate.
146+
The slower oscillations can be attributed to the breathing signal.
147+
148+
We can appreciate how the estimated motion signal matches the processed LFP traces plotted above.
149+
150+
.. code:: ipython3
151+
152+
fig, ax = plt.subplots()
153+
si.plot_motion(motion, mode='line', ax=ax)
154+
ax.set_xlim(400, 420)
155+
ax.set_ylim(800, 1300)
156+
157+
158+
.. parsed-literal::
159+
160+
(800.0, 1300.0)
161+
162+
163+
.. image:: drift_with_lfp_files/drift_with_lfp_12_1.png
47 KB
Loading
191 KB
Loading

doc/how_to/handle_drift.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ to display the results.
245245

246246
.. code:: ipython
247247
248-
from spikeinterface.sortingcomponents.motion_interpolation import correct_motion_on_peaks
248+
from spikeinterface.sortingcomponents.motion import correct_motion_on_peaks
249249
250250
for preset in some_presets:
251251
folder = base_folder / 'motion_folder_dataset1' / preset

doc/how_to/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ Guides on how to solve specific, short problems in SpikeInterface. Learn how to.
1414
process_by_channel_group
1515
load_your_data_into_sorting
1616
benchmark_with_hybrid_recordings
17+
drift_with_lfp

doc/modules/motion_correction.rst

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,7 @@ The high-level :py:func:`~spikeinterface.preprocessing.correct_motion()` is inte
151151
from spikeinterface.sortingcomponents.peak_detection import detect_peaks
152152
from spikeinterface.sortingcomponents.peak_selection import select_peaks
153153
from spikeinterface.sortingcomponents.peak_localization import localize_peaks
154-
from spikeinterface.sortingcomponents.motion_estimation import estimate_motion
155-
from spikeinterface.sortingcomponents.motion_interpolation import interpolate_motion
154+
from spikeinterface.sortingcomponents.motion import estimate_motion, interpolate_motion
156155
157156
job_kwargs = dict(chunk_duration="1s", n_jobs=20, progress_bar=True)
158157
# Step 1 : activity profile

doc/modules/sortingcomponents.rst

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -190,10 +190,10 @@ Here is an example with non-rigid motion estimation:
190190
peak_locations = localize_peaks(recording=recording, peaks=peaks, ...) # as above
191191
192192
193-
from spikeinterface.sortingcomponents.motion_estimation import estimate_motion
193+
from spikeinterface.sortingcomponents.motion import estimate_motion
194194
motion, temporal_bins, spatial_bins,
195195
extra_check = estimate_motion(recording=recording, peaks=peaks, peak_locations=peak_locations,
196-
direction='y', bin_duration_s=10., bin_um=10., margin_um=0.,
196+
direction='y', bin_s=10., bin_um=10., margin_um=0.,
197197
method='decentralized_registration',
198198
rigid=False, win_shape='gaussian', win_step_um=50., win_sigma_um=150.,
199199
progress_bar=True, verbose=True)
@@ -206,7 +206,7 @@ Motion interpolation
206206

207207
The estimated motion can be used to interpolate traces, in other words, for drift correction.
208208
One possible way is to make an interpolation sample-by-sample to compensate for the motion.
209-
The :py:class:`~spikeinterface.sortingcomponents.motion_interpolation.InterpolateMotionRecording` is a preprocessing
209+
The :py:class:`~spikeinterface.sortingcomponents.motion.InterpolateMotionRecording` is a preprocessing
210210
step doing this. This preprocessing is *lazy*, so that interpolation is done on-the-fly. However, the class needs the
211211
"motion vector" as input, which requires a relatively long computation (peak detection, localization and motion
212212
estimation).
@@ -216,7 +216,7 @@ Here is a short example that depends on the output of "Motion interpolation":
216216

217217
.. code-block:: python
218218
219-
from spikeinterface.sortingcomponents.motion_interpolation import InterpolateMotionRecording
219+
from spikeinterface.sortingcomponents.motion import InterpolateMotionRecording
220220
221221
recording_corrected = InterpolateMotionRecording(recording=recording_with_drift, motion=motion, temporal_bins=temporal_bins, spatial_bins=spatial_bins
222222
spatial_interpolation_method='kriging',

examples/how_to/benchmark_with_hybrid_recordings.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
import spikeinterface.generation as sgen
3333
import spikeinterface.widgets as sw
3434

35-
from spikeinterface.sortingcomponents.motion_estimation import estimate_motion
35+
from spikeinterface.sortingcomponents.motion import estimate_motion
3636

3737
import numpy as np
3838
import matplotlib.pyplot as plt

0 commit comments

Comments
 (0)