Fix earlier attempt to fix dateconf DerivedLayers, this time with a test to prove it's right (#743)

* Fix earlier attempt to fix dateconf DerivedLayers, this time with a test to prove it

Author: dmannarino

app/routes/datasets/queries.py

@@ -726,8 +726,12 @@ def _get_date_conf_derived_layers(
     # TODO should these somehow be in the metadata or creation options instead of hardcoded?
     # 16435 is number of days from 1970-01-01 to 2015-01-01, and can be used to convert
     # our encoding of days since 2015 to a number that can be used generally for datetimes
-    decode_expression = "(A + 16435).astype('datetime64[D]').astype(str)"
-    encode_expression = "(A.astype('datetime64[D]') - 16435).astype(uint16)"
+    decode_expression = (
+        "(A.astype('timedelta64[D]') + np.datetime64('2015-01-01', 'D')).astype(str)"
+    )
+    encode_expression = (
+        "(np.datetime64(A, 'D') - np.datetime64('2015-01-01', 'D')).astype(uint16)"
+    )
     conf_encoding = RasterTable(
         default_meaning="not_detected",
         rows=[

tests_v2/fixtures/otf_payload/otf_payload.py

@@ -13,8 +13,8 @@
         "name": "my_first_dataset__date",
         "no_data": 0,
         "raster_table": None,
-        "decode_expression": "(A + 16435).astype('datetime64[D]').astype(str)",
-        "encode_expression": "(A.astype('datetime64[D]') - 16435).astype(uint16)",
+        "decode_expression": "(A.astype('timedelta64[D]') + np.datetime64('2015-01-01', 'D')).astype(str)",
+        "encode_expression": "(np.datetime64(A, 'D') - np.datetime64('2015-01-01', 'D')).astype(uint16)",
         "source_layer": "my_first_dataset__date_conf",
         "calc": "A % 10000",
     },

tests_v2/unit/app/routes/datasets/test_query.py

@@ -1,5 +1,5 @@
 import re
-from typing import Tuple
+from typing import List, Tuple
 from unittest.mock import Mock, patch
 from urllib.parse import parse_qsl, urlparse
 from uuid import UUID
@@ -22,6 +22,7 @@
 from app.routes.datasets.queries import (
     _get_data_environment,
     _get_data_environment_sql,
+    _get_date_conf_derived_layers,
     _query_dataset_json,
     _query_raster,
     _query_raster_lambda,
@@ -856,3 +857,23 @@ async def test_get_data_environment_sql():
         },
     ],
 }
+
+
+def test__get_date_conf_derived_layers_encode_decode_roundtrip():
+    """The old encode_expression failed because datetime64(A) without unit
+    specifier breaks with modern NumPy when A is a string."""
+    import numpy as np
+    from numpy import uint16
+
+    original_date = A = "2023-01-15"
+
+    layers: List[DerivedLayer] = _get_date_conf_derived_layers("foo", 0)
+
+    for layer in layers:
+        if layer.encode_expression:
+            encoded = eval(
+                layer.encode_expression, {"np": np, "uint16": uint16, "A": A}
+            )
+
+            decoded = eval(layer.decode_expression, {"np": np, "A": encoded})
+            assert decoded == original_date