fix: Add error logging for course ID fetching and log course counts found during scraping

Author: techtanic

base.py

@@ -1109,6 +1109,11 @@ def get_course_id(self):
             except requests.exceptions.ConnectionError:
                 r = None
                 continue
+            except Exception as e:
+                logger.error(f"Error fetching course ID: {e}")
+                logger.error(f"Course URL: {url}")
+                r = None
+                continue
 
         if r is None:
             logger.error("Failed to fetch course ID after 3 attempts")
@@ -1122,7 +1127,6 @@ def get_course_id(self):
         course_id = soup.find("body").get("data-clp-course-id", "invalid")
         if course_id == "invalid":
             self.course.is_valid = False
-            logger.error(soup.prettify(encoding="utf-8"))
             self.course.error = "Course ID not found: Report to developer"
             return
 

cli.py

@@ -185,6 +185,8 @@ def create_scraping_thread(site: str):
             time.sleep(0.1)
 
         udemy.progress.update(task_id, completed=getattr(scraper, f"{code_name}_length"))
+        logger.debug(f"Courses Found {code_name}: {len(getattr(scraper, f'{code_name}_data'))}")
+        
         if getattr(scraper, f"{code_name}_error"):
             raise Exception(f"Error in: {site}")
     except Exception:

gui.py

@@ -66,7 +66,7 @@ def create_scraping_thread(site: str):
             )
 
             time.sleep(0.1)
-
+        logger.info(f"Courses Found {code_name}: {len(getattr(scraper, f'{code_name}_data'))}")
         if getattr(scraper, f"{code_name}_error"):
             raise Exception(f"Error in: {site}")
     except Exception: