Sets asc order on event request and start_date is now - 30 days

Author: Mark Kasaboski

packages/flare/bin/cron_job_ingest_events.py

@@ -10,6 +10,8 @@
 
 from datetime import date
 from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
 from typing import Any
 from typing import Iterator
 from typing import Optional
@@ -90,14 +92,14 @@ def main(
     create_collection(kvstore=kvstore)
 
     # To avoid cron jobs from doing the same work at the same time, exit new cron jobs if a cron job is already doing work
-    last_fetched_timestamp = get_last_fetched(kvstore=kvstore)
-    if last_fetched_timestamp and last_fetched_timestamp > (
-        datetime.now() - CRON_JOB_THRESHOLD_SINCE_LAST_FETCH
-    ):
-        logger.info(
-            f"Fetched events less than {int(CRON_JOB_THRESHOLD_SINCE_LAST_FETCH.seconds / 60)} minutes ago, exiting"
-        )
-        return
+    # last_fetched_timestamp = get_last_fetched(kvstore=kvstore)
+    # if last_fetched_timestamp and last_fetched_timestamp > (
+    #     datetime.now() - CRON_JOB_THRESHOLD_SINCE_LAST_FETCH
+    # ):
+    #     logger.info(
+    #         f"Fetched events less than {int(CRON_JOB_THRESHOLD_SINCE_LAST_FETCH.seconds / 60)} minutes ago, exiting"
+    #     )
+    #     return
 
     api_key = get_api_key(storage_passwords=storage_passwords)
     tenant_id = get_tenant_id(storage_passwords=storage_passwords)
@@ -210,7 +212,7 @@ def get_start_date(kvstore: KVStoreCollections) -> Optional[date]:
     )
     if start_date:
         try:
-            return date.fromisoformat(start_date)
+            return datetime.fromisoformat(start_date)
         except Exception:
             pass
     return None
@@ -255,7 +257,7 @@ def save_last_ingested_tenant_id(kvstore: KVStoreCollections, tenant_id: int) ->
         save_collection_value(
             kvstore=kvstore,
             key=CollectionKeys.START_DATE.value,
-            value=date.today().isoformat(),
+            value=(datetime.now(timezone.utc) - timedelta(days=30)).isoformat(),
         )
 
     save_collection_value(

packages/flare/bin/flare.py

@@ -8,7 +8,9 @@
 import requests
 import time
 
-from datetime import date
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
 from logger import Logger
 from typing import Any
 from typing import Dict
@@ -55,7 +57,7 @@ def fetch_feed_events(
         self,
         *,
         next: Optional[str] = None,
-        start_date: Optional[date] = None,
+        start_date: Optional[datetime] = None,
         ingest_full_event_data: bool,
         severities: list[str],
         source_types: list[str],
@@ -81,17 +83,18 @@ def _fetch_event_feed_metadata(
         self,
         *,
         next: Optional[str] = None,
-        start_date: Optional[date] = None,
+        start_date: Optional[datetime] = None,
         severities: list[str],
         source_types: list[str],
     ) -> Iterator[requests.Response]:
         data: Dict[str, Any] = {
             "from": next if next else None,
+            "order": "asc",
             "filters": {
                 "materialized_at": {
                     "gte": start_date.isoformat()
                     if start_date
-                    else date.today().isoformat()
+                    else (datetime.now(timezone.utc) - timedelta(days=30)).isoformat()
                 },
             },
         }

packages/flare/tests/bin/test_ingest_events.py

@@ -6,8 +6,8 @@
 from conftest import FakeKVStoreCollections
 from conftest import FakeLogger
 from conftest import FakeStoragePasswords
-from datetime import date
 from datetime import datetime
+from datetime import timezone
 from freezegun import freeze_time
 
 
@@ -101,11 +101,11 @@ def test_get_start_date_expect_none(kvstore: FakeKVStoreCollections) -> None:
 
 @pytest.mark.parametrize(
     "kvstore",
-    [[(CollectionKeys.START_DATE.value, "2000-01-01")]],
+    [[(CollectionKeys.START_DATE.value, "2000-01-01T00:00:00+00:00")]],
     indirect=True,
 )
 def test_get_start_date_expect_date(kvstore: FakeKVStoreCollections) -> None:
-    assert get_start_date(kvstore) == date(2000, 1, 1)
+    assert get_start_date(kvstore) == datetime(2000, 1, 1, 0, 0, tzinfo=timezone.utc)
 
 
 def test_get_last_ingested_tenant_id_expect_none(
@@ -125,7 +125,7 @@ def test_get_last_ingested_tenant_id_expect_integer(
     assert get_last_ingested_tenant_id(kvstore=kvstore) == 11111
 
 
-@freeze_time("2000-01-01")
+@freeze_time("2000-01-01T00:00:00+00:00")
 def test_save_last_ingested_tenant_id_expect_new_tenant_id_and_new_start_date(
     kvstore: FakeKVStoreCollections,
 ) -> None:
@@ -135,7 +135,7 @@ def test_save_last_ingested_tenant_id_expect_new_tenant_id_and_new_start_date(
 
     save_last_ingested_tenant_id(kvstore=kvstore, tenant_id=11111)
     assert kvstore[KV_COLLECTION_NAME].data.query() == [
-        {"_key": CollectionKeys.START_DATE.value, "value": "2000-01-01"},
+        {"_key": CollectionKeys.START_DATE.value, "value": "1999-12-02T00:00:00+00:00"},
         {"_key": CollectionKeys.LAST_INGESTED_TENANT_ID.value, "value": 11111},
     ]
 
@@ -144,23 +144,23 @@ def test_save_last_ingested_tenant_id_expect_new_tenant_id_and_new_start_date(
     "kvstore",
     [
         [
-            (CollectionKeys.START_DATE.value, "1999-12-12"),
+            (CollectionKeys.START_DATE.value, "1999-12-12T00:00:00+00:00"),
             (CollectionKeys.LAST_INGESTED_TENANT_ID.value, 11111),
         ]
     ],
     indirect=True,
 )
-@freeze_time("2000-01-01")
+@freeze_time("2000-01-01T00:00:00+00:00")
 def test_save_last_ingested_tenant_id_expect_updated_tenant_id_and_updated_start_date(
     kvstore: FakeKVStoreCollections,
 ) -> None:
     assert kvstore[KV_COLLECTION_NAME].data.query() == [
-        {"_key": CollectionKeys.START_DATE.value, "value": "1999-12-12"},
+        {"_key": CollectionKeys.START_DATE.value, "value": "1999-12-12T00:00:00+00:00"},
         {"_key": CollectionKeys.LAST_INGESTED_TENANT_ID.value, "value": 11111},
     ]
     save_last_ingested_tenant_id(kvstore=kvstore, tenant_id=22222)
     assert kvstore[KV_COLLECTION_NAME].data.query() == [
-        {"_key": CollectionKeys.START_DATE.value, "value": "2000-01-01"},
+        {"_key": CollectionKeys.START_DATE.value, "value": "1999-12-02T00:00:00+00:00"},
         {"_key": CollectionKeys.LAST_INGESTED_TENANT_ID.value, "value": 22222},
     ]
 
@@ -311,6 +311,6 @@ def test_main_expect_normal_run(
         flare_api_cls=FakeFlareAPI,
     )
     assert logger.messages == [
-        "INFO: Fetching tenant_id=11111, next=None, start_date=FakeDate(2000, 1, 1)",
+        "INFO: Fetching tenant_id=11111, next=None, start_date=FakeDatetime(1999, 12, 2, 0, 0, tzinfo=datetime.timezone.utc)",
         "INFO: Fetched 2 events",
     ]