KeyError: activity - adds retry for full event data else raises

Author: Mark Kasaboski

packages/flare/bin/cron_job_ingest_events.py

@@ -122,7 +122,6 @@ def main(
         flare_api_cls=flare_api_cls,
     ):
         save_last_fetched(kvstore=kvstore)
-
         save_next(kvstore=kvstore, tenant_id=tenant_id, next=next_token)
 
         print(json.dumps(event), flush=True)

packages/flare/bin/flare.py

@@ -73,10 +73,25 @@ def fetch_feed_events(
             next_token = event_feed["next"]
             for event in event_feed["items"]:
                 if ingest_full_event_data:
-                    event = self._fetch_full_event_from_uid(
-                        uid=event["metadata"]["uid"]
-                    )
-                    time.sleep(1)
+                    uid = event["metadata"]["uid"]
+                    retries = 3
+                    for attempt in range(retries):
+                        try:
+                            event = self._fetch_full_event_from_uid(uid=uid)
+                            break
+                        except Exception as e:
+                            self.logger.error(
+                                f"Failed to fetch full event data for {uid}: {e}"
+                            )
+                            if attempt < retries - 1:
+                                time.sleep(1) # Don't hit rate limit
+                            else:
+                                self.logger.error(
+                                    f"Failed to fetch full event data for UID {uid} after {retries} attempts."
+                                )
+                                raise e
+
+                    time.sleep(1)  # Don't hit rate limit
                 yield (event, next_token)
 
     def _fetch_event_feed_metadata(
@@ -116,9 +131,8 @@ def _fetch_event_feed_metadata(
 
     def _fetch_full_event_from_uid(self, *, uid: str) -> dict:
         event_response = self.flare_client.get(url=f"/firework/v2/activities/{uid}")
-        event = event_response.json()["activity"]
-        self.logger.debug(event)
-        return event
+        event_response.raise_for_status()
+        return event_response.json()["activity"]
 
     def fetch_api_key_validation(self) -> requests.Response:
         return self.flare_client.get(