Merge branch 'SpikeInterface:main' into drift_docs_to_sphinx_gallery

Author: JoeZiminski

.github/import_test.py

@@ -18,7 +18,7 @@
 
 n_samples = 10
 # Note that the symbols at the end are for centering the table
-markdown_output = f"## \n\n| Imported Module ({n_samples=}) | Importing Time (seconds) | Standard Deviation (seconds) |\n| :--: | :--------------: | :------------------: |\n"
+markdown_output = f"## \n\n| Imported Module ({n_samples=}) | Importing Time (seconds) | Standard Deviation (seconds) | Times List (seconds) |\n| :--: | :--------------: | :------------------: | :-------------: |\n"
 
 exceptions = []
 
@@ -45,10 +45,17 @@
         time_taken = float(result.stdout.strip())
         time_taken_list.append(time_taken)
 
+    # for time in time_taken_list:
+        # Uncomment once exporting import is fixed
+        # if time > 2.5:
+        #     exceptions.append(f"Importing {import_statement} took too long: {time:.2f} seconds")
+        #     break
+
     if time_taken_list:
         avg_time_taken = sum(time_taken_list) / len(time_taken_list)
         std_dev_time_taken = math.sqrt(sum((x - avg_time_taken) ** 2 for x in time_taken_list) / len(time_taken_list))
-        markdown_output += f"| `{import_statement}` | {avg_time_taken:.2f} | {std_dev_time_taken:.2f} |\n"
+        times_list_str = ", ".join(f"{time:.2f}" for time in time_taken_list)
+        markdown_output += f"| `{import_statement}` | {avg_time_taken:.2f} | {std_dev_time_taken:.2f} | {times_list_str} |\n"
 
 if exceptions:
     raise Exception("\n".join(exceptions))

.github/workflows/core-test.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           git config --global user.email "[email protected]"
@@ -31,7 +31,7 @@ jobs:
           pip install -e .[test_core]
       - name: Test core with pytest
         run: |
-          pytest -vv -ra --durations=0 --durations-min=0.001 src/spikeinterface/core | tee report.txt; test ${PIPESTATUS[0]} -eq 0 || exit 1
+          pytest -m "core" -vv -ra --durations=0 --durations-min=0.001 | tee report.txt; test $? -eq 0 || exit 1
         shell: bash  # Necessary for pipeline to work on windows
       - name: Build test summary
         run: |

.github/workflows/test_imports.yml

@@ -22,13 +22,13 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install Spikeinterface with only core dependencies
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "CI Almighty"
           python -m pip install -U pip  # Official recommended way
-          pip install -e .  # This should install core only
+          pip install .  # This should install core only
       - name: Profile Imports
         run: |
           echo "## OS: ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY
@@ -38,8 +38,7 @@ jobs:
         shell: bash  # Necessary for pipeline to work on windows
       - name: Install in full mode
         run: |
-          python -m pip install -U pip  # Official recommended way
-          pip install -e .[full]
+          pip install .[full]
       - name: Profile Imports with full
         run: |
           # Add a header to separate the two profiles

conftest.py

@@ -25,31 +25,27 @@ def pytest_sessionstart(session):
     for mark_name in mark_names:
         (pytest.global_test_folder / mark_name).mkdir()
 
-
 def pytest_collection_modifyitems(config, items):
     """
     This function marks (in the pytest sense) the tests according to their name and file_path location
     Marking them in turn allows the tests to be run by using the pytest -m marker_name option.
     """
 
-
-    # python 3.4/3.5 compat: rootdir = pathlib.Path(str(config.rootdir))
     rootdir = Path(config.rootdir)
-
+    modules_location = rootdir / "src" / "spikeinterface"
     for item in items:
-        rel_path = Path(item.fspath).relative_to(rootdir)
-        if "sorters" in str(rel_path):
-            if "/internal/" in str(rel_path):
+        rel_path = Path(item.fspath).relative_to(modules_location)
+        module = rel_path.parts[0]
+        if module == "sorters":
+            if "internal" in rel_path.parts:
                 item.add_marker("sorters_internal")
-            elif "/external/" in str(rel_path):
+            elif "external" in rel_path.parts:
                 item.add_marker("sorters_external")
             else:
                 item.add_marker("sorters")
         else:
-            for mark_name in mark_names:
-                if f"/{mark_name}/" in str(rel_path):
-                    mark = getattr(pytest.mark, mark_name)
-                    item.add_marker(mark)
+            item.add_marker(module)
+
 
 
 def pytest_sessionfinish(session, exitstatus):

doc/conf.py

@@ -67,6 +67,7 @@
     'numpydoc',
     'sphinx.ext.autosectionlabel',
     'sphinx_design',
+    'sphinxcontrib.jquery',
     "sphinx.ext.intersphinx",
     "sphinx.ext.extlinks",
     "IPython.sphinxext.ipython_directive",

doc/development/development.rst

@@ -152,7 +152,7 @@ for providing parameters, however is a little different. The project prefers the
 
 .. code-block:: bash
 
-    parameter_name: type, default: default_value
+    parameter_name : type, default: default_value
 
 
 This allows users to quickly understand the type of data that should be input into a function as well as whether a default is supplied. A full example would be:
@@ -165,21 +165,22 @@ This allows users to quickly understand the type of data that should be input in
 
         Parameters
         ----------
-        param_a: dict
+        param_a : dict
             A dictionary containing the data
-        param_b: int, default: 5
+        param_b : int, default: 5
             A scaling factor to be applied to the data
-        param_c: "mean" | "median", default: "mean"
+        param_c : "mean" | "median", default: "mean"
             What to calculate on the data
 
         Returns
         -------
-        great_data: dict
+        great_data : dict
             A dictionary of the processed data
         """
 
 
-Note that in this example we demonstrate two other docstring conventions followed by SpikeInterface. First, that all string arguments should be presented
+There should be a space between each parameter and the colon following it. This is neccessary for using the `numpydoc validator <https://numpydoc.readthedocs.io/en/latest/validation.html>`_.
+In the above example we demonstrate two other docstring conventions followed by SpikeInterface. First, that all string arguments should be presented
 with double quotes. This is the same stylistic convention followed by Black and enforced by the pre-commit for the repo. Second, when a parameter is a
 string with a limited number of values (e.g. :code:`mean` and :code:`median`), rather than give the type a value of :code:`str`, please list the possible strings
 so that the user knows what the options are.

doc/index.rst

@@ -58,6 +58,7 @@ SpikeInterface is made of several modules to deal with different aspects of the
     development/development
     whatisnew
     authors
+    references
 
 
 Other resources

doc/references.rst

@@ -0,0 +1,129 @@
+How to Cite
+===========
+
+If you like SpikeInterface, please star us on `Github <https://github.com/SpikeInterface/spikeinterface>`_!
+*giving us a star gives a measure of the level of use and interest, which goes a long way to getting funding*
+
+Please cite SpikeInterface in your papers with our eLife paper: [Buccino]_
+
+SpikeInterface stands on the shoulders of giants!
+Each method in SpikeInterface draws on (or directly runs) independently-created methods.
+Please try to reference the individual works that are important for your analysis pipeline.
+If you notice a missing reference, please let us know by `submitting an issue <https://github.com/SpikeInterface/spikeinterface/issues/new>`_ on Github.
+
+Preprocessing Module
+--------------------
+If you use one of the following preprocessing methods, please cite the appropriate source:
+
+- :code:`phase_shift` or :code:`highpass_spatial_filter` [IBL]_
+- :code:`detect_bad_channels(method='coherence+psd')` [IBL]_
+- :code:`common_reference` [Rolston]_
+
+Motion Correction
+^^^^^^^^^^^^^^^^^
+If you use the :code:`correct_motion` method in the preprocessing module, please cite [Garcia]_
+as well as the references that correspond to the :code:`preset` you used:
+
+- :code:`nonrigid_accurate` [Windolf]_ [Varol]_
+- :code:`nonrigid_fast_and_accurate` [Windolf]_ [Varol]_ [Pachitariu]_
+- :code:`rigid_fast` *no additional citation needed*
+- :code:`kilosort_like` [Pachitariu]_
+
+Sorters Module
+--------------
+If you use one of the following spike sorting algorithms (i.e. you use the :code:`run_sorter()` method,
+please include the appropriate citation for the :code:`sorter_name` parameter you use:
+*Note: unless otherwise stated, the reference given is to be used for all versions of the sorter*
+
+- :code:`combinato` [Niediek]_
+- :code:`hdsort` [Diggelmann]_
+- :code:`herdingspikes` [Muthmann]_ [Hilgen]_
+- :code:`kilosort`  [Pachitariu]_
+- :code:`mountainsort` [Chung]_
+- :code:`spykingcircus` [Yger]_
+- :code:`wavclus` [Chaure]_
+- :code:`yass` [Lee]_
+
+Qualitymetrics Module
+---------------------
+If you use the :code:`qualitymetrics` module, i.e. you use the :code:`analyzer.compute()`
+or :code:`compute_quality_metrics()` methods, please include the citations for the :code:`metric_names` that were particularly
+important for your research:
+
+- :code:`amplitude_cutoff` or  :code:`isi_violation` [Hill]_
+- :code:`amplitude_median` or :code:`sliding_rp_violation` [IBL]_
+- :code:`drift` [Siegle]_
+- :code:`rp_violation` [Llobet]_
+- :code:`sd_ratio` [Pouzat]_
+- :code:`snr` [Lemon]_ [Jackson]_
+- :code:`synchrony` [Grun]_
+
+If you use the :code:`qualitymetrics.pca_metrics` module, i.e. you use the
+:code:`compute_pc_metrics()` method, please include the citations for the :code:`metric_names` that were particularly
+important for your research:
+
+- :code:`d_prime` [Hill]_
+- :code:`isolation_distance` or :code:`l_ratio` [Schmitzer-Torbert]_
+- :code:`nearest_neighbor` or :code:`nn_isolation` or :code:`nn_noise_overlap` [Chung]_ [Siegle]_
+- :code:`silhouette`  [Rousseeuw]_ [Hruschka]_
+
+Curation Module
+---------------
+If you use the :code:`get_potential_auto_merge` method from the curation module, please cite [Llobet]_
+
+References
+----------
+
+.. [Buccino] `SpikeInterface, a unified framework for spike sorting. 2020. <https://pubmed.ncbi.nlm.nih.gov/33170122/>`_
+
+.. [Buzsaki] `The Log-Dynamic Brain: How Skewed Distributions Affect Network Operations. 2014. <https://pubmed.ncbi.nlm.nih.gov/24569488/>`_
+
+.. [Chaure] `A novel and fully automatic spike-sorting implementation with variable number of features. 2018. <https://pubmed.ncbi.nlm.nih.gov/29995603/>`_
+
+.. [Chung] `A Fully Automated Approach to Spike Sorting. 2017. <https://pubmed.ncbi.nlm.nih.gov/28910621/>`_
+
+.. [Diggelmann] `Automatic spike sorting for high-density microelectrode arrays. 2018. <https://pubmed.ncbi.nlm.nih.gov/30207864/>`_
+
+.. [Garcia] `A Modular Implementation to Handle and Benchmark Drift Correction for High-Density Extracellular Recordings. 2024. <https://pubmed.ncbi.nlm.nih.gov/38238082/>`_
+
+.. [Grun] `Impact of higher-order correlations on coincidence distributions of massively parallel data. 2007. <https://www.researchgate.net/publication/225145104_Impact_of_Higher-Order_Correlations_on_Coincidence_Distributions_of_Massively_Parallel_Data>`_
+
+.. [Harris] `Temporal interaction between single spikes and complex spike bursts in hippocampal pyramidal cells. 2001. <https://pubmed.ncbi.nlm.nih.gov/11604145/>`_
+
+.. [Hilgen] `Unsupervised Spike Sorting for Large-Scale, High-Density Multielectrode Arrays. 2017. <https://pubmed.ncbi.nlm.nih.gov/28273464/>`_
+
+.. [Hill] `Quality Metrics to Accompany Spike Sorting of Extracellular Signals. 2011. <https://pubmed.ncbi.nlm.nih.gov/21677152/>`_
+
+.. [Hruschka] `Evolutionary algorithms for clustering gene-expression data. 2004. <https://www.researchgate.net/publication/220765683_Evolutionary_Algorithms_for_Clustering_Gene-Expression_Data>`_
+
+.. [IBL] `Spike sorting pipeline for the International Brain Laboratory. 2022. <https://figshare.com/articles/online_resource/Spike_sorting_pipeline_for_the_International_Brain_Laboratory/19705522/3>`_
+
+.. [Jackson] Quantitative assessment of extracellular multichannel recording quality using measures of cluster separation. Society of Neuroscience Abstract. 2005.
+
+.. [Lee] `YASS: Yet another spike sorter. 2017. <https://www.biorxiv.org/content/10.1101/151928v1>`_
+
+.. [Lemon] Methods for neuronal recording in conscious animals. IBRO Handbook Series. 1984.
+
+.. [Llobet] `Automatic post-processing and merging of multiple spike-sorting analyses with Lussac. 2022. <https://www.biorxiv.org/content/10.1101/2022.02.08.479192v1>`_
+
+.. [Muthmann] `Spike Detection for Large Neural Populations Using High Density Multielectrode Arrays. 2015. <https://pubmed.ncbi.nlm.nih.gov/26733859/>`_
+
+.. [Niediek] `Reliable Analysis of Single-Unit Recordings from the Human Brain under Noisy Conditions: Tracking Neurons over Hours. 2016. <https://pubmed.ncbi.nlm.nih.gov/27930664/>`_
+
+.. [Pachitariu] `Spike sorting with Kilosort4. 2024. <https://pubmed.ncbi.nlm.nih.gov/38589517/>`_
+
+.. [Pouzat] `Using noise signature to optimize spike-sorting and to assess neuronal classification quality. 2002. <https://pubmed.ncbi.nlm.nih.gov/12535763/>`_
+
+.. [Rolston] `Common median referencing for improved action potential detection with multielectrode arrays. 2009. <https://pubmed.ncbi.nlm.nih.gov/19964004/>`_
+
+.. [Rousseeuw] `Silhouettes: A graphical aid to the interpretation and validation of cluster analysis. 1987. <https://www.sciencedirect.com/science/article/pii/0377042787901257>`_
+
+.. [Schmitzer-Torbert] `Neuronal Activity in the Rodent Dorsal Striatum in Sequential Navigation: Separation of Spatial and Reward Responses on the Multiple T Task. 2004. <https://pubmed.ncbi.nlm.nih.gov/14736863/>`_
+
+.. [Siegle] `Survey of Spiking in the Mouse Visual System Reveals Functional Hierarchy. 2021. <https://pubmed.ncbi.nlm.nih.gov/33473216/>`_
+
+.. [Varol] `Decentralized Motion Inference and Registration of Neuropixel Data. 2021. <https://ieeexplore.ieee.org/document/9414145>`_
+
+.. [Windolf] `Robust Online Multiband Drift Estimation in Electrophysiology Data. 2022. <https://www.biorxiv.org/content/10.1101/2022.12.04.519043v2>`_
+
+.. [Yger] `A spike sorting toolbox for up to thousands of electrodes validated with ground truth recordings in vitro and in vivo. 2018. <https://pubmed.ncbi.nlm.nih.gov/29557782/>`_

pyproject.toml

@@ -91,11 +91,11 @@ full = [
     "h5py",
     "pandas",
     "xarray",
-    "scipy<1.13",
+    "scipy",
     "scikit-learn",
     "networkx",
     "distinctipy",
-    "matplotlib",
+    "matplotlib>=3.6", # matplotlib.colormaps
     "cuda-python; platform_system != 'Darwin'",
     "numba",
 ]
@@ -159,8 +159,8 @@ test = [
 ]
 
 docs = [
-    "Sphinx==5.1.1",
-    "sphinx_rtd_theme==1.0.0",
+    "Sphinx",
+    "sphinx_rtd_theme",
     "sphinx-gallery",
     "sphinx-design",
     "numpydoc",
@@ -173,6 +173,7 @@ docs = [
     "hdbscan>=0.8.33",   # For sorters spykingcircus2 + tridesclous
     "numba", # For many postprocessing functions
     "xarray", # For use of SortingAnalyzer zarr format
+    "networkx",
     # for release we need pypi, so this needs to be commented
     "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",  # We always build from the latest version
     "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",  # We always build from the latest version

src/spikeinterface/comparison/collision.py

@@ -14,7 +14,7 @@ class CollisionGTComparison(GroundTruthComparison):
     This class needs maintenance and need a bit of refactoring.
 
 
-    collision_lag: float
+    collision_lag : float
         Collision lag in ms.
 
     """