Merge pull request #22 from ScrapingAnt/feature/add-custom-headers-support

feature/add-custom-headers-support: done

Author: megabotan

README.md

@@ -50,6 +50,7 @@ https://docs.scrapingant.com/request-response-format#available-parameters
 | --- | --- | --- |
 | url | <code>string</code> |  |
 | cookies | <code>List[Cookie]</code> | None |
+| headers | <code>List[Dict[str, str]]</code> | None |
 | js_snippet | <code>string</code> | None |
 | proxy_type | <code>ProxyType</code> | datacenter | 
 | proxy_country | <code>str</code> | None | 
@@ -171,6 +172,29 @@ else:
     print(f'Successfully parsed data: {parsed_data}')
 ```
 
+### Sending custom headers
+
+```python3
+from scrapingant_client import ScrapingAntClient
+
+client = ScrapingAntClient(token='<YOUR-SCRAPINGANT-API-TOKEN>')
+
+result = client.general_request(
+    'https://httpbin.org/headers', 
+    headers={
+        'test-header': 'test-value'
+    }
+)
+print(result.content)
+
+# Http basic auth example
+result = client.general_request(
+    'https://jigsaw.w3.org/HTTP/Basic/', 
+    headers={'Authorization': 'Basic Z3Vlc3Q6Z3Vlc3Q='}
+)
+print(result.content)
+```
+
 
 ## Useful links
 - [Scrapingant API doumentation](https://docs.scrapingant.com)

scrapingant_client/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.3.7"
+__version__ = "0.3.8"
 
 from scrapingant_client.client import ScrapingAntClient
 from scrapingant_client.cookie import Cookie

scrapingant_client/client.py

@@ -1,6 +1,6 @@
 import platform
 import sys
-from typing import List, Optional
+from typing import List, Optional, Dict
 
 import requests
 
@@ -14,6 +14,7 @@
     ScrapingantSiteNotReachableException,
     ScrapingantDetectedException,
 )
+from scrapingant_client.headers import convert_headers
 from scrapingant_client.proxy_type import ProxyType
 from scrapingant_client.response import Response
 from scrapingant_client.utils import base64_encode_string
@@ -34,6 +35,7 @@ def general_request(
             self,
             url: str,
             cookies: Optional[List[Cookie]] = None,
+            headers: Optional[Dict[str, str]] = None,
             js_snippet: Optional[str] = None,
             proxy_type: ProxyType = ProxyType.datacenter,
             proxy_country: Optional[str] = None,
@@ -58,6 +60,7 @@ def general_request(
         response = self.requests_session.post(
             SCRAPINGANT_API_BASE_URL + '/general',
             json=request_data,
+            headers=convert_headers(headers),
         )
         if response.status_code == 403:
             raise ScrapingantInvalidTokenException()

scrapingant_client/headers.py

@@ -0,0 +1,10 @@
+from typing import Dict, Optional
+
+
+def convert_headers(headers: Optional[Dict[str, str]]):
+    if headers is None:
+        return None
+    return {
+        f'ant-{header_name}': header_value
+        for header_name, header_value in headers.items()
+    }

tests/test_headers.py

@@ -0,0 +1,16 @@
+from scrapingant_client.headers import convert_headers
+
+
+def test_convert_empty_headers():
+    assert convert_headers(None) is None
+
+
+def test_convert_headers():
+    headers = {
+        'User-Agent': 'test',
+        'Accept-Language': 'en-US'
+    }
+    assert convert_headers(headers) == {
+        'ant-User-Agent': 'test',
+        'ant-Accept-Language': 'en-US'
+    }