called distinct() on the fetched Lobsters stories to avoid duplicates

Author: somegenericdev

src/dtos.py

@@ -8,6 +8,8 @@ def __init__(self, _title, _url, _hacker_news_url, _last_seen, _posted_date):
 
     def __eq__(self, other):
         return self.url == other.url and self.posted_date == other.posted_date
+    def __hash__(self):
+        return hash((self.url, self.posted_date))
 
 class RateLimitException(Exception):
     pass

src/scrapers.py

@@ -81,7 +81,6 @@ def cleanup_old_stories(self):
     def get_old_stories(self):
         try:
             all_stories = list(Story.select().where(Story.type == StoryType.Hackernews.value))
-            print(all_stories)
             return seq(all_stories).map(lambda s: self.entity_to_dto(s)).to_list()
         except Exception as e:
             return []
@@ -177,7 +176,6 @@ def cleanup_old_stories(self):
     def get_old_stories(self):
         try:
             all_stories = list(Story.select().where(Story.type == StoryType.Lobsters.value))
-            print(all_stories)
             return seq(all_stories).map(lambda s: self.entity_to_dto(s)).to_list()
         except Exception as e:
             return []
@@ -206,7 +204,7 @@ def scrape(self):
             old_stories = self.get_old_stories()
             rc_file = self.read_rc_file()
             queries = json.loads(rc_file)["queries"]
-            all_current_stories = seq.range(1, 30).flat_map(lambda p: self.process_page(p)).to_list()
+            all_current_stories = seq.range(1, 30).flat_map(lambda p: self.process_page(p)).distinct().to_list()
             self.update_last_seen_dates(all_current_stories)
             filtered_stories = seq(all_current_stories).filter(lambda x: seq(queries).map(lambda q: x.title.lower().find(q.lower()) != -1).any()).to_list()
             diff_stories = seq(filtered_stories).filter(lambda x: not seq(old_stories).filter(lambda y: y == x).any()).to_list()