fix: send a User-Agent header on every API request (#103)

Both clients now send `scrapegraph-py/<version>` instead of httpx's default User-Agent, on the pooled client and on the one-off absolute-URL requests, sync and async.

Author: VinciGit00

src/scrapegraph_py/async_client.py

@@ -11,6 +11,7 @@
 import httpx
 from pydantic import BaseModel, TypeAdapter
 
+from .client import USER_AGENT
 from .env import env
 from .schemas import (
     ApiResult,
@@ -262,7 +263,7 @@ def __init__(self, *, api_key: str | None = None):
         self._http = httpx.AsyncClient(
             base_url=env.base_url,
             timeout=env.timeout,
-            headers={"SGAI-APIKEY": self._api_key},
+            headers={"SGAI-APIKEY": self._api_key, "User-Agent": USER_AGENT},
         )
 
         self.crawl = AsyncCrawlResource(self)
@@ -292,7 +293,7 @@ async def _request[T](
                         url,
                         json=json_body,
                         params=params,
-                        headers={"SGAI-APIKEY": self._api_key},
+                        headers={"SGAI-APIKEY": self._api_key, "User-Agent": USER_AGENT},
                     )
             else:
                 resp = await self._http.request(method, path, json=json_body, params=params)

src/scrapegraph_py/client.py

@@ -6,6 +6,7 @@
 import sys
 import time
 from datetime import datetime
+from importlib import metadata
 from typing import Literal
 
 import httpx
@@ -44,6 +45,7 @@
 )
 
 _SERVER_TIMING_RE = re.compile(r"dur=(\d+(?:\.\d+)?)")
+USER_AGENT = f"scrapegraph-py/{metadata.version('scrapegraph-py')}"
 
 
 def _debug(label: str, data: object = None) -> None:
@@ -262,7 +264,7 @@ def __init__(self, *, api_key: str | None = None):
         self._http = httpx.Client(
             base_url=env.base_url,
             timeout=env.timeout,
-            headers={"SGAI-APIKEY": self._api_key},
+            headers={"SGAI-APIKEY": self._api_key, "User-Agent": USER_AGENT},
         )
 
         self.crawl = CrawlResource(self)
@@ -291,7 +293,7 @@ def _request[T](
                     url,
                     json=json_body,
                     params=params,
-                    headers={"SGAI-APIKEY": self._api_key},
+                    headers={"SGAI-APIKEY": self._api_key, "User-Agent": USER_AGENT},
                     timeout=env.timeout,
                 )
             else:

tests/test_client.py

@@ -4,6 +4,7 @@
 import pytest
 
 from scrapegraph_py import (
+    AsyncScrapeGraphAI,
     FetchConfig,
     HtmlFormatConfig,
     ImagesFormatConfig,
@@ -14,6 +15,7 @@
     ScrapeGraphAI,
     ScreenshotFormatConfig,
 )
+from scrapegraph_py.client import USER_AGENT
 
 API_KEY = "test-sgai-key"
 BASE_URL = "https://api.scrapegraphai.com/v2"
@@ -483,6 +485,15 @@ def test_explicit_api_key(self):
         sgai = ScrapeGraphAI(api_key="explicit-key")
         assert sgai._api_key == "explicit-key"
 
+    def test_sends_user_agent(self):
+        sgai = ScrapeGraphAI(api_key=API_KEY)
+        assert sgai._http.headers["User-Agent"] == USER_AGENT
+        assert USER_AGENT.startswith("scrapegraph-py/")
+
+    def test_async_sends_user_agent(self):
+        sgai = AsyncScrapeGraphAI(api_key=API_KEY)
+        assert sgai._http.headers["User-Agent"] == USER_AGENT
+
 
 class TestCamelCaseSerialization:
     def test_snake_to_camel(self):