Fix for [BUG] Error handling timezones #305 (#318)

* fix: check if update_data contains update before batch_update

* add test + avoid same error when verbose=True

* 🧹 create _hf_data_container if correct trace type

* 🙏 python 3.7 not supported on Apple Silicon

* remove WIP

* 🖊️ more verbose asserts

* 🖊️ more verbose asserts

* 🙏 more sleep time

* 🙏

* 🙌

* 🤔 fix for [BUG] Error handling timezones #305

* 🙈 linting

* 💨 Refactor timezone handling in PlotlyAggregatorParser

* Update minmax operator image

Signed-off-by: Emmanuel Ferdman <[email protected]>

* Drop duplicate sentence

* Feat/plotly6 (#338)

* Parametrize test_utils.py on is_figure

* 🔍 remove dtype parsing as orjon>3.10 supports float16 #118

* 💪 refactor: streamline JupyterDash integration and remove unused persistent inline logic

* 💨 move construct_update_data_patch method into the FigureResampler class

* 🐐 refactor: enhance test utilities and add support for Plotly>=6 data handling

* 🙏 enhance serialization tests for plotly>6

* 📝 remove debug print statement and enhance type handling for hf_x

* 🔒 update dependency versions in pyproject.toml to Support plotly 6 #334

* 🔍 drop python3.7 CI workflow and upgrade upload-artifact action

* 🙏 fix pickling of figurewidget resampler

* 🙏 fix tests

* 💨 migration of code towards new upload artifact

* 💪 enhance CI workflow to improve test result uploads and add retention settings

* 🕳️ fix: ensure correct dtype handling for aggregated x indices in PlotlyAggregatorParser

* ⬆️ chore: update dependency constraints for pandas and pyarrow in pyproject.toml

* 🙈 fix linting

* 🔍 fix: correct spelling in streamlit_app.py comments and update dash-extensions and pyarrow versions in requirements.txt

* ⬆️ chore: update ipywidgets version constraint to allow for newer versions

* 🚧 test: set random seed for reproducibility in test_wrap_aggregate

* 🙈 chore: update ipywidgets version constraint for serialization support

* 🙈

* 🔍 ci: conditionally skip tests on Python 3.12 for Ubuntu (as it keeps hanging in github actions)

* 🔍 ci: exclude Python 3.12 on Ubuntu from test matrix to prevent hangs

* 🖊️ review code

* 🧹 cleanup comments

---------

Co-authored-by: Maxim Ivanov <[email protected]>
Co-authored-by: jeroen <[email protected]>

* 🤔 fix for [BUG] Error handling timezones #305

* 🙈 linting

* 💨 Refactor timezone handling in PlotlyAggregatorParser

* 📌 bug: Fix timezone handling for DST in PlotlyAggregatorParser and update tests

---------

Signed-off-by: Emmanuel Ferdman <[email protected]>
Co-authored-by: jvdd <[email protected]>
Co-authored-by: Jeroen Van Der Donckt <[email protected]>
Co-authored-by: Emmanuel Ferdman <[email protected]>
Co-authored-by: Maxim Ivanov <[email protected]>

Author: 4 people

plotly_resampler/aggregation/plotly_aggregator_parser.py

@@ -38,7 +38,11 @@ def to_same_tz(
             return None
         elif reference_tz is not None:
             if ts.tz is not None:
-                assert ts.tz.__str__() == reference_tz.__str__()
+                # compare if these two have the same timezone / offset
+                try:
+                    assert ts.tz.__str__() == reference_tz.__str__()
+                except AssertionError:
+                    assert ts.utcoffset() == reference_tz.utcoffset(ts.tz_convert(None))
                 return ts
             else:  # localize -> time remains the same
                 return ts.tz_localize(reference_tz)
@@ -78,7 +82,15 @@ def get_start_end_indices(hf_trace_data, axis_type, start, end) -> Tuple[int, in
             # convert start & end to the same timezone
             if isinstance(hf_trace_data["x"], pd.DatetimeIndex):
                 tz = hf_trace_data["x"].tz
-                assert start.tz == end.tz
+                try:
+                    assert start.tz.__str__() == end.tz.__str__()
+                except (TypeError, AssertionError):
+                    # This fix is needed for DST (when the timezone is not fixed)
+                    assert start.tz_localize(None) == start.tz_convert(tz).tz_localize(
+                        None
+                    )
+                    assert end.tz_localize(None) == end.tz_convert(tz).tz_localize(None)
+
                 start = PlotlyAggregatorParser.to_same_tz(start, tz)
                 end = PlotlyAggregatorParser.to_same_tz(end, tz)
 

tests/test_figure_resampler.py

@@ -754,6 +754,74 @@ def test_tz_xaxis_range():
     assert len(out[2]["x"]) == 2000
 
 
+def test_compare_tz_with_fixed_offset():
+    # related: https://github.com/predict-idlab/plotly-resampler/issues/305
+    fig = FigureResampler()
+
+    x = pd.date_range("2024-04-01T00:00:00", "2025-01-01T00:00:00", freq="h")
+    x = x.tz_localize("Asia/Taipei")
+    y = np.random.randn(len(x))
+
+    fig.add_trace(
+        go.Scattergl(x=x, y=y, name="demo", mode="lines+markers"),
+        max_n_samples=int(len(x) * 0.2),
+    )
+
+    relayout_data = {
+        "xaxis.range[0]": "2024-04-27T08:00:00+08:00",
+        "xaxis.range[1]": "2024-05-04T17:15:39.491031+08:00",
+    }
+
+    fig.construct_update_data_patch(relayout_data)
+
+
+def test_compare_tz_with_fixed_offset_2():
+    # related: https://github.com/predict-idlab/plotly-resampler/issues/305
+    fig = FigureResampler()
+
+    x = pd.date_range("2024-04-01T00:00:00", "2025-01-01T00:00:00", freq="h")
+    x = x.tz_localize("UTC")
+    x = x.tz_convert("Canada/Pacific")
+    y = np.random.randn(len(x))
+
+    fig.add_trace(
+        go.Scattergl(x=x, y=y, name="demo", mode="lines+markers"),
+        max_n_samples=int(len(x) * 0.2),
+    )
+
+    relayout_data = {
+        "xaxis.range[0]": pd.Timestamp("2024-03-01T00:00:00").tz_localize(
+            "Canada/Pacific"
+        ),
+        "xaxis.range[1]": pd.Timestamp("2024-03-31T00:00:00").tz_localize(
+            "Canada/Pacific"
+        ),
+    }
+
+    fig.construct_update_data_patch(relayout_data)
+
+
+def test_relayout_tz_DST():
+    # related: https://github.com/predict-idlab/plotly-resampler/issues/305
+    fig = FigureResampler()
+
+    x = pd.date_range(
+        "2024-09-27 17:00:00", "2024-12-11 16:00:00", tz="US/Pacific", freq="1h"
+    )
+    y = np.random.randn(len(x))
+    fig.add_trace(
+        go.Scattergl(x=x, y=y, name="demo", mode="lines+markers"),
+        max_n_samples=int(len(x) * 0.2),
+    )
+
+    relayout_data = {
+        "xaxis.range[0]": "2024-09-27T17:00:00-07:00",
+        "xaxis.range[1]": "2024-12-12T15:59:00-08:00",
+    }
+
+    fig.construct_update_data_patch(relayout_data)
+
+
 def test_datetime_hf_x_no_index():
     df = pd.DataFrame(
         {"timestamp": pd.date_range("2020-01-01", "2020-01-02", freq="1s")}
@@ -789,8 +857,8 @@ def test_multiple_timezones_in_single_x_index__datetimes_and_timestamps():
     # TODO: can be improved with pytest parametrize
     y = np.arange(20)
 
-    index1 = pd.date_range("2018-01-01", periods=10, freq="H", tz="US/Eastern")
-    index2 = pd.date_range("2018-01-02", periods=10, freq="H", tz="Asia/Dubai")
+    index1 = pd.date_range("2018-01-01", periods=10, freq="h", tz="US/Eastern")
+    index2 = pd.date_range("2018-01-02", periods=10, freq="h", tz="Asia/Dubai")
     index_timestamps = index1.append(index2)
     assert all(isinstance(x, pd.Timestamp) for x in index_timestamps)
     index1_datetimes = pd.Index([x.to_pydatetime() for x in index1])
@@ -1009,7 +1077,7 @@ def test_time_tz_slicing_different_timestamp():
     cs = [
         dr,
         dr.tz_localize(None).tz_localize("Europe/Amsterdam"),
-        dr.tz_convert("Europe/Brussels"),
+        dr.tz_convert("Europe/Lisbon"),
         dr.tz_convert("Australia/Perth"),
         dr.tz_convert("Australia/Canberra"),
     ]
@@ -1027,6 +1095,25 @@ def test_time_tz_slicing_different_timestamp():
                 hf_data_dict, hf_data_dict["axis_type"], t_start, t_stop
             )
 
+    # THESE have the same timezone offset -> no AssertionError should be raised
+    cs = [
+        dr.tz_localize(None).tz_localize("Europe/Amsterdam"),
+        dr.tz_convert("Europe/Brussels"),
+        dr.tz_convert("Europe/Oslo"),
+        dr.tz_convert("Europe/Paris"),
+        dr.tz_convert("Europe/Rome"),
+    ]
+
+    for i, s in enumerate(cs):
+        t_start, t_stop = sorted(s.iloc[np.random.randint(0, n, 2)].index)
+        t_start = t_start.tz_convert(cs[(i + 1) % len(cs)].index.tz)
+        t_stop = t_stop.tz_convert(cs[(i + 1) % len(cs)].index.tz)
+
+        hf_data_dict = construct_hf_data_dict(s.index, s.values)
+        start_idx, end_idx = PlotlyAggregatorParser.get_start_end_indices(
+            hf_data_dict, hf_data_dict["axis_type"], t_start, t_stop
+        )
+
 
 def test_different_tz_no_tz_series_slicing():
     n = 60 * 60 * 24 * 3
@@ -1092,7 +1179,7 @@ def test_multiple_tz_no_tz_series_slicing():
 
         # Now the assumption cannot be made that s has the same time-zone as the
         # timestamps -> AssertionError will be raised.
-        with pytest.raises(AssertionError):
+        with pytest.raises((TypeError, AssertionError)):
             hf_data_dict = construct_hf_data_dict(s.tz_localize(None).index, s.values)
             PlotlyAggregatorParser.get_start_end_indices(
                 hf_data_dict, hf_data_dict["axis_type"], t_start, t_stop