chore: improved url validation

Author: PeriniM

scrapegraph-py/scrapegraph_py/async_client.py

@@ -18,6 +18,35 @@
 
 
 class AsyncClient:
+    @classmethod
+    def from_env(
+        cls,
+        verify_ssl: bool = True,
+        timeout: float = 120,
+        max_retries: int = 3,
+        retry_delay: float = 1.0,
+    ):
+        """Initialize AsyncClient using API key from environment variable.
+
+        Args:
+            verify_ssl: Whether to verify SSL certificates
+            timeout: Request timeout in seconds
+            max_retries: Maximum number of retry attempts
+            retry_delay: Delay between retries in seconds
+        """
+        from os import getenv
+
+        api_key = getenv("SGAI_API_KEY")
+        if not api_key:
+            raise ValueError("SGAI_API_KEY environment variable not set")
+        return cls(
+            api_key=api_key,
+            verify_ssl=verify_ssl,
+            timeout=timeout,
+            max_retries=max_retries,
+            retry_delay=retry_delay,
+        )
+
     def __init__(
         self,
         api_key: str,
@@ -54,34 +83,6 @@ def __init__(
 
         logger.info("✅ AsyncClient initialized successfully")
 
-    @classmethod
-    def from_env(
-        cls,
-        verify_ssl: bool = True,
-        timeout: float = 120,
-        max_retries: int = 3,
-        retry_delay: float = 1.0,
-    ):
-        """Initialize AsyncClient using API key from environment variable.
-
-        Args:
-            verify_ssl: Whether to verify SSL certificates
-            timeout: Request timeout in seconds
-            max_retries: Maximum number of retry attempts
-            retry_delay: Delay between retries in seconds
-        """
-        from os import getenv
-        api_key = getenv("SGAI_API_KEY")
-        if not api_key:
-            raise ValueError("SGAI_API_KEY environment variable not set")
-        return cls(
-            api_key=api_key,
-            verify_ssl=verify_ssl,
-            timeout=timeout,
-            max_retries=max_retries,
-            retry_delay=retry_delay,
-        )
-
     async def _make_request(self, method: str, url: str, **kwargs) -> Any:
         """Make HTTP request with retry logic."""
         for attempt in range(self.max_retries):

scrapegraph-py/scrapegraph_py/models/smartscraper.py

@@ -3,7 +3,6 @@
 from typing import Optional, Type
 from uuid import UUID
 
-import validators
 from pydantic import BaseModel, Field, model_validator
 
 
@@ -25,9 +24,13 @@ def validate_user_prompt(self) -> "SmartScraperRequest":
 
     @model_validator(mode="after")
     def validate_url(self) -> "SmartScraperRequest":
-        url = self.website_url
-        if not validators.url(url):
-            raise ValueError(f"Invalid URL: {url}")
+        if self.website_url is None or not self.website_url.strip():
+            raise ValueError("Website URL cannot be empty")
+        if not (
+            self.website_url.startswith("http://")
+            or self.website_url.startswith("https://")
+        ):
+            raise ValueError("Invalid URL")
         return self
 
     def model_dump(self, *args, **kwargs) -> dict: