Feat/plotly6 (#338)

* Parametrize test_utils.py on is_figure

* 🔍 remove dtype parsing as orjon>3.10 supports float16 #118

* 💪 refactor: streamline JupyterDash integration and remove unused persistent inline logic

* 💨 move construct_update_data_patch method into the FigureResampler class

* 🐐 refactor: enhance test utilities and add support for Plotly>=6 data handling

* 🙏 enhance serialization tests for plotly>6

* 📝 remove debug print statement and enhance type handling for hf_x

* 🔒 update dependency versions in pyproject.toml to Support plotly 6 #334

* 🔍 drop python3.7 CI workflow and upgrade upload-artifact action

* 🙏 fix pickling of figurewidget resampler

* 🙏 fix tests

* 💨 migration of code towards new upload artifact

* 💪 enhance CI workflow to improve test result uploads and add retention settings

* 🕳️ fix: ensure correct dtype handling for aggregated x indices in PlotlyAggregatorParser

* ⬆️ chore: update dependency constraints for pandas and pyarrow in pyproject.toml

* 🙈 fix linting

* 🔍 fix: correct spelling in streamlit_app.py comments and update dash-extensions and pyarrow versions in requirements.txt

* ⬆️ chore: update ipywidgets version constraint to allow for newer versions

* 🚧 test: set random seed for reproducibility in test_wrap_aggregate

* 🙈 chore: update ipywidgets version constraint for serialization support

* 🙈

* 🔍 ci: conditionally skip tests on Python 3.12 for Ubuntu (as it keeps hanging in github actions)

* 🔍 ci: exclude Python 3.12 on Ubuntu from test matrix to prevent hangs

* 🖊️ review code

* 🧹 cleanup comments

---------

Co-authored-by: Maxim Ivanov <[email protected]>
Co-authored-by: jeroen <[email protected]>

Author: 3 people

.github/workflows/test.yml

@@ -19,19 +19,15 @@ on:
 
 jobs:
   test:
-
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ['windows-latest', 'macOS-latest', 'ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
-        exclude:  # Python < 3.8 is not supported on Apple Silicon ARM64
-          - os: macOS-latest
-            python-version: '3.7'
-        include:  # So run on older version on Intel CPU
-          - os: macOS-13
-            python-version: '3.7'
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+        exclude:
+          - os: ubuntu-latest
+            python-version: '3.12'
     defaults:
       run:
         shell: bash
@@ -76,12 +72,15 @@ jobs:
       run: |
         poetry run pytest --cov=plotly_resampler --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov-report=xml tests
     - name: Upload pytest test results
-      uses: actions/upload-artifact@v3
+       # Use always() to always run this step to publish test results when there are test failures
+      if: ${{ always() && hashFiles('junit/test-results-${{ matrix.python-version }}.xml') != '' }}
+      uses: actions/upload-artifact@v4
       with:
-        name: pytest-results-${{ matrix.python-version }}
+        name: pytest-results-${{ matrix.python-version }}-${{ matrix.os }}-${{ github.run_number }}
         path: junit/test-results-${{ matrix.python-version }}.xml
-        # Use always() to always run this step to publish test results when there are test failures
-      if: ${{ always() }}
+        overwrite: true
+        retention-days: 7
+        compression-level: 5
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3

examples/other_apps/streamlit_app.py

@@ -16,7 +16,7 @@
 
 __author__ = "Jeroen Van Der Donckt"
 
-# Explicitely set pio.templates in order to have colored traces in the streamlit app!
+# Explicitly set pio.templates in order to have colored traces in the streamlit app!
 # -> https://discuss.streamlit.io/t/streamlit-overrides-colours-of-plotly-chart/34943/5
 import plotly.io as pio
 

examples/requirements.txt

@@ -1,9 +1,9 @@
 pyfunctional>=1.4.3
 dash-bootstrap-components>=1.2.0
-dash-extensions==1.0.1  # fixated on this version as more recent versions do not work
+dash-extensions==1.0.20  # fixated on this version as more recent versions do not work
 ipywidgets>=7.7.0
 memory-profiler>=0.60.0
 line-profiler>=3.5.1
-pyarrow>=6.0.0
+pyarrow>=17.0.0
 kaleido>=0.2.1
 flask-cors>=3.0.10

plotly_resampler/aggregation/plotly_aggregator_parser.py

@@ -190,7 +190,7 @@ def aggregate(
                 agg_x = (
                     start_idx
                     + hf_trace_data["x"].start
-                    + indices * hf_trace_data["x"].step
+                    + indices.astype(hf_trace_data["x"].dtype) * hf_trace_data["x"].step
                 )
             else:
                 agg_x = hf_x[indices]

plotly_resampler/figure_resampler/figure_resampler.py

@@ -13,7 +13,7 @@
 import os
 import warnings
 from pathlib import Path
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Union
 
 import dash
 import plotly.graph_objects as go
@@ -26,15 +26,9 @@
     MinMaxLTTB,
 )
 from .figure_resampler_interface import AbstractFigureAggregator
+from .jupyter_dash_persistent_inline_output import JupyterDashPersistentInlineOutput
 from .utils import is_figure, is_fr
 
-try:
-    from .jupyter_dash_persistent_inline_output import JupyterDashPersistentInlineOutput
-
-    _jupyter_dash_installed = True
-except ImportError:
-    _jupyter_dash_installed = False
-
 # Default arguments for the Figure overview
 ASSETS_FOLDER = Path(__file__).parent.joinpath("assets").absolute().__str__()
 _DEFAULT_OVERVIEW_LAYOUT_KWARGS = {
@@ -242,7 +236,6 @@ def __init__(
         self._host: str | None = None
         # Certain functions will be different when using persistent inline
         # (namely `show_dash` and `stop_callback`)
-        self._is_persistent_inline = False
 
     def _get_subplot_rows_and_cols_from_grid(self) -> Tuple[int, int]:
         """Get the number of rows and columns of the figure's grid.
@@ -535,7 +528,9 @@ def show_dash(
                 constructor via the ``show_dash_kwargs`` argument.
 
         """
-        available_modes = ["external", "inline", "inline_persistent", "jupyterlab"]
+        available_modes = list(dash._jupyter.JupyterDisplayMode.__args__) + [
+            "inline_persistent"
+        ]
         assert (
             mode is None or mode in available_modes
         ), f"mode must be one of {available_modes}"
@@ -576,25 +571,6 @@ def show_dash(
             init_dash_kwargs["external_scripts"] = ["https://cdn.jsdelivr.net/npm/lodash/lodash.min.js" ]
             # fmt: on
 
-        if mode == "inline_persistent":
-            mode = "inline"
-            if _jupyter_dash_installed:
-                # Inline persistent mode: we display a static image of the figure when the
-                # app is not reachable
-                # Note: this is the "inline" behavior of JupyterDashInlinePersistentOutput
-                app = JupyterDashPersistentInlineOutput("local_app", **init_dash_kwargs)
-                self._is_persistent_inline = True
-            else:
-                # If Jupyter Dash is not installed, inline persistent won't work and hence
-                # we default to normal inline mode with a normal Dash app
-                app = dash.Dash("local_app", **init_dash_kwargs)
-                warnings.warn(
-                    "'jupyter_dash' is not installed. The persistent inline mode will not work. Defaulting to standard inline mode."
-                )
-        else:
-            # jupyter dash uses a normal Dash app as figure
-            app = dash.Dash("local_app", **init_dash_kwargs)
-
         # fmt: off
         div = dash.html.Div(
             children=[
@@ -620,18 +596,19 @@ def show_dash(
                     **graph_properties,
                 ),
             ]
-        app.layout = div
 
+        # Create the app, populate the layout and register the resample callback
+        app = dash.Dash("local_app", **init_dash_kwargs)
+        app.layout = div
         self.register_update_graph_callback(
             app,
             "resample-figure",
             "overview-figure" if self._create_overview else None,
         )
 
-        height_param = "height" if self._is_persistent_inline else "jupyter_height"
-
         # 2. Run the app
-        if mode == "inline" and height_param not in kwargs:
+        height_param = "height" if mode == "inline_persistent" else "jupyter_height"
+        if "inline" in mode and height_param not in kwargs:
             # If app height is not specified -> re-use figure height for inline dash app
             #  Note: default layout height is 450 (whereas default app height is 650)
             #  See: https://plotly.com/python/reference/layout/#layout-height
@@ -646,9 +623,11 @@ def show_dash(
         self._host = kwargs.get("host", "127.0.0.1")
         self._port = kwargs.get("port", "8050")
 
-        # function signature is slightly different for the Dash and JupyterDash implementations
-        if self._is_persistent_inline:
-            app.run(mode=mode, **kwargs)
+        # function signatures are slightly different for the (Jupyter)Dash and the
+        # JupyterDashInlinePersistent implementations
+        if mode == "inline_persistent":
+            jpi = JupyterDashPersistentInlineOutput(self)
+            jpi.run_app(app=app, **kwargs)
         else:
             app.run(jupyter_mode=mode, **kwargs)
 
@@ -665,18 +644,10 @@ def stop_server(self, warn: bool = True):
             This only works if the dash-app was started with [`show_dash`][figure_resampler.figure_resampler.FigureResampler.show_dash].
         """
         if self._app is not None:
-            servers_dict = (
-                self._app._server_threads
-                if self._is_persistent_inline
-                else dash.jupyter_dash._servers
-            )
+            servers_dict = dash.jupyter_dash._servers
             old_server = servers_dict.get((self._host, self._port))
             if old_server:
-                if self._is_persistent_inline:
-                    old_server.kill()
-                    old_server.join()
-                else:
-                    old_server.shutdown()
+                old_server.shutdown()
             del servers_dict[(self._host, self._port)]
         elif warn:
             warnings.warn(
@@ -685,6 +656,47 @@ def stop_server(self, warn: bool = True):
                 + "\t- the dash-server wasn't started with 'show_dash'"
             )
 
+    def construct_update_data_patch(
+        self, relayout_data: dict
+    ) -> Union[dash.Patch, dash.no_update]:
+        """Construct the Patch of the to-be-updated front-end data, based on the layout
+        change.
+
+        Attention
+        ---------
+        This method is tightly coupled with Dash app callbacks. It takes the front-end
+        figure its ``relayoutData`` as input and returns the ``dash.Patch`` which needs
+        to be sent to the ``figure`` property for the corresponding ``dcc.Graph``.
+
+        Parameters
+        ----------
+        relayout_data: dict
+            A dict containing the ``relayoutData`` (i.e., the changed layout data) of
+            the corresponding front-end graph.
+
+        Returns
+        -------
+        dash.Patch:
+            The Patch object containing the figure updates which needs to be sent to
+            the front-end.
+
+        """
+        update_data = self._construct_update_data(relayout_data)
+        if not isinstance(update_data, list) or len(update_data) <= 1:
+            return dash.no_update
+
+        patched_figure = dash.Patch()  # create patch
+        for trace in update_data[1:]:  # skip first item as it contains the relayout
+            trace_index = trace.pop("index")  # the index of the corresponding trace
+            # All the other items are the trace properties which needs to be updated
+            for k, v in trace.items():
+                # NOTE: we need to use the `patched_figure` as a dict, and not
+                # `patched_figure.data` as the latter will replace **all** the
+                # data for the corresponding trace, and we just want to update the
+                # specific trace its properties.
+                patched_figure["data"][trace_index][k] = v
+        return patched_figure
+
     def register_update_graph_callback(
         self,
         app: dash.Dash,

plotly_resampler/figure_resampler/figure_resampler_interface.py

@@ -371,10 +371,8 @@ def _check_update_trace_data(
         )
 
         # -------------------- Set the hf_trace_data_props -------------------
-        # Parse the data types to an orjson compatible format
-        # NOTE: this can be removed once orjson supports f16
-        trace["x"] = self._parse_dtype_orjson(agg_x)
-        trace["y"] = self._parse_dtype_orjson(agg_y)
+        trace["x"] = agg_x
+        trace["y"] = agg_y
         trace["name"] = self._parse_trace_name(
             hf_trace_data, end_idx - start_idx, agg_x
         )
@@ -578,7 +576,7 @@ def _parse_get_trace_props(
             A namedtuple which serves as a datacontainer.
 
         """
-        hf_x: np.ndarray = (
+        hf_x: np.ndarray | pd.Index = (
             # fmt: off
             (np.asarray(trace["x"]) if trace["x"] is not None else None)
             if hasattr(trace, "x") and hf_x is None
@@ -593,6 +591,8 @@ def _parse_get_trace_props(
             else np.asarray(hf_x)
             # fmt: on
         )
+        if pd.core.dtypes.common.is_datetime64_any_dtype(hf_x):
+            hf_x = pd.Index(hf_x)
 
         hf_y = (
             trace["y"]
@@ -1037,7 +1037,7 @@ def add_trace(
                     [trace], **self._add_trace_to_add_traces_kwargs(trace_kwargs)
                 )
 
-        return super(self._figure_class, self).add_traces(
+        return super().add_traces(
             [trace], **self._add_trace_to_add_traces_kwargs(trace_kwargs)
         )
 
@@ -1169,7 +1169,7 @@ def add_traces(
             assert trace is not None
             data[i] = trace
 
-        return super(self._figure_class, self).add_traces(data, **traces_kwargs)
+        return super().add_traces(data, **traces_kwargs)
 
     def _clear_figure(self):
         """Clear the current figure object its data and layout."""
@@ -1259,47 +1259,6 @@ def _parse_relayout(self, relayout_dict: dict) -> dict:
             extra_layout_updates[f"{axis}.autorange"] = None
         return extra_layout_updates
 
-    def construct_update_data_patch(
-        self, relayout_data: dict
-    ) -> Union[dash.Patch, dash.no_update]:
-        """Construct the Patch of the to-be-updated front-end data, based on the layout
-        change.
-
-        Attention
-        ---------
-        This method is tightly coupled with Dash app callbacks. It takes the front-end
-        figure its ``relayoutData`` as input and returns the ``dash.Patch`` which needs
-        to be sent to the ``figure`` property for the corresponding ``dcc.Graph``.
-
-        Parameters
-        ----------
-        relayout_data: dict
-            A dict containing the ``relayoutData`` (i.e., the changed layout data) of
-            the corresponding front-end graph.
-
-        Returns
-        -------
-        dash.Patch:
-            The Patch object containing the figure updates which needs to be sent to
-            the front-end.
-
-        """
-        update_data = self._construct_update_data(relayout_data)
-        if not isinstance(update_data, list) or len(update_data) <= 1:
-            return dash.no_update
-
-        patched_figure = dash.Patch()  # create patch
-        for trace in update_data[1:]:  # skip first item as it contains the relayout
-            trace_index = trace.pop("index")  # the index of the corresponding trace
-            # All the other items are the trace properties which needs to be updated
-            for k, v in trace.items():
-                # NOTE: we need to use the `patched_figure` as a dict, and not
-                # `patched_figure.data` as the latter will replace **all** the
-                # data for the corresponding trace, and we just want to update the
-                # specific trace its properties.
-                patched_figure["data"][trace_index][k] = v
-        return patched_figure
-
     def _construct_update_data(
         self,
         relayout_data: dict,
@@ -1400,17 +1359,6 @@ def _construct_update_data(
             layout_traces_list.append(trace_reduced)
         return layout_traces_list
 
-    @staticmethod
-    def _parse_dtype_orjson(series: np.ndarray) -> np.ndarray:
-        """Verify the orjson compatibility of the series and convert it if needed."""
-        # NOTE:
-        #    * float16 and float128 aren't supported with latest orjson versions (3.8.1)
-        #    * this method assumes that the it will not get a float128 series
-        # -> this method can be removed if orjson supports float16
-        if series.dtype == np.float16:
-            return series.astype(np.float32)
-        return series
-
     @staticmethod
     def _re_matches(regex: re.Pattern, strings: Iterable[str]) -> List[str]:
         """Returns all the items in ``strings`` which regex.match(es) ``regex``."""

plotly_resampler/figure_resampler/figurewidget_resampler.py

@@ -326,3 +326,8 @@ def reload_data(self):
             )
             # TODO: when we know which traces have changed we can use
             # a new -> `update_xaxis_str` argument.
+
+    def __reduce__(self):
+        # Needed for pickling
+        # Specifically set the class name, as the metaclass is not easily picklable
+        return FigureWidgetResampler, *list(super().__reduce__())[1:]