twitter_tags: adding tags_frequent_extractions parameter

Author: c200bzh

source-twitter-fetcher/source_twitter_fetcher/source.py

@@ -38,6 +38,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         if "start_time" in config:
             tags_kwargs["start_time"] = datetime.strptime(config['start_time'], DATE_FORMAT)
 
+        # Add tags_frequent_extractions if provided in config
+        if "tags_frequent_extractions" in config:
+            tags_kwargs["tags_frequent_extractions"] = config["tags_frequent_extractions"]
+            
         tags = TagsStream(**tags_kwargs)
 
         tweet_metrics = TweetMetrics(

source-twitter-fetcher/source_twitter_fetcher/spec.yaml

@@ -50,4 +50,9 @@ connectionSpecification:
       description: "List of Twitter handles to monitor (e.g., ['@IFT', '@Airbyte'])"
       items:
         type: string
-      minItems: 1
+      minItems: 1 
+    tags_frequent_extractions:
+      type: boolean
+      title: "Tags Frequent Extractions"
+      description: "If true, defaults start_time to 1 hour 15 minutes before current time for more frequent extractions. If false, defaults to 5 days before current time."
+      default: false

source-twitter-fetcher/source_twitter_fetcher/tags_stream.py

@@ -12,15 +12,19 @@ class TagsStream(HttpStream):
     url_base = "https://api.x.com/2/"
     primary_key = "id"
 
-    def __init__(self, start_time: str = None, account_id: str = None, tags: List[str] = None, **kwargs):
+    def __init__(self, start_time: str = None, account_id: str = None, tags: List[str] = None, tags_frequent_extractions: bool = False, **kwargs):
         super().__init__(**kwargs)
 
-        # If start_time is provided, parse it; otherwise default to 5 days ago
+        # If start_time is provided, parse it; otherwise default based on tags_frequent_extractions
         if start_time:
             self.start_time = start_time if isinstance(start_time, datetime) else datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%SZ")
         else:
-            # Default to 5 days before current time
-            self.start_time = datetime.utcnow() - timedelta(days=5)
+            if tags_frequent_extractions:
+                # Default to 1 hour 15 minutes before current time
+                self.start_time = datetime.utcnow() - timedelta(hours=1, minutes=15)
+            else:
+                # Default to 5 days before current time
+                self.start_time = datetime.utcnow() - timedelta(days=5)
 
         self.account_id = account_id
         self.tags = tags or []