🔨 Reformat for mypy, reformat GA yml files (#291)

Author: roniemartinez

.github/workflows/docker-publish.yml

@@ -43,7 +43,6 @@ jobs:
         with:
           cosign-release: 'v1.7.1'
 
-
       # Workaround: https://github.com/docker/build-push-action/issues/461
       - name: Setup Docker buildx
         uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7

.github/workflows/documentation.yml

@@ -18,9 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.11'
+
       - run: |
           pip3 install -U pip setuptools poetry
           poetry config virtualenvs.create false

.github/workflows/pypi.yml

@@ -13,17 +13,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.x'
+          python-version: '3.11'
+
       - name: Install dependencies
         run: |
           pip3 install -U pip setuptools poetry
+
       - name: Build package
         run: |
           poetry config http-basic.pypi ${{ secrets.PYPI_USER }} ${{ secrets.PYPI_PASS }}
           poetry build
+
       - name: Publish package
         run: |
           poetry publish

.github/workflows/python.yml

@@ -35,49 +35,59 @@ jobs:
             poetry-cache: ~\AppData\Local\pypoetry\Cache
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
       - uses: actions/cache@v3
         with:
           path: ${{ matrix.pip-cache }}
           key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python-version }}-pip-
+
       - uses: actions/cache@v3
         with:
           path: ${{ matrix.poetry-cache }}
           key: ${{ runner.os }}-${{ matrix.python-version }}-poetry-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python-version }}-poetry-
+
       - uses: actions/cache@v3
         with:
           path: .mypy_cache
           key: ${{ runner.os }}-${{ matrix.python-version }}-mypy-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python-version }}-mypy-
+
       - uses: actions/cache@v3
         with:
           path: ~/.cache/ms-playwright
           key: ${{ runner.os }}-${{ matrix.python-version }}-ms-playwright-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python-version }}-ms-playwright-
+
       - uses: actions/cache@v3
         with:
           path: ~/.wdm
           key: ${{ runner.os }}-webdriver-manager
+
       - name: Install dependencies
         run: |
           make install-actions
+
       - name: Lint
         run: |
           make lint
+
       - name: Test
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           make test
+
       - name: Code Coverage
         uses: codecov/codecov-action@v3.1.1
         if: ${{ success() }}

.github/workflows/testpypi.yml

@@ -13,18 +13,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.x'
+          python-version: '3.11'
+
       - name: Install dependencies
         run: |
           pip3 install -U pip setuptools poetry
+
       - name: Build package
         run: |
           poetry config repositories.testpypi https://test.pypi.org/legacy/
           poetry config http-basic.testpypi ${{ secrets.TEST_PYPI_USER }} ${{ secrets.TEST_PYPI_PASS }}
           poetry build
+
       - name: Publish package
         run: |
           poetry publish -r testpypi

dude/base.py

@@ -47,9 +47,9 @@ class ScraperBase(ABC):
 
     def __init__(
         self,
-        rules: List[Rule] = None,
-        groups: Dict[Callable, Selector] = None,
-        save_rules: Dict[Tuple[str, bool], Any] = None,
+        rules: Optional[List[Rule]] = None,
+        groups: Optional[Dict[Callable, Selector]] = None,
+        save_rules: Optional[Dict[Tuple[str, bool], Any]] = None,
         events: Optional[DefaultDict] = None,
         has_async: bool = False,
         requests: Optional[Deque] = None,  # only valid for BeautifulSoup4, lxml and Parsel backends
@@ -140,20 +140,20 @@ def run(
 
     def select(
         self,
-        selector: str = None,
-        group: str = None,
+        selector: Optional[str] = None,
+        group: Optional[str] = None,
         setup: bool = False,
         navigate: bool = False,
         url_match: Union[str, Callable] = "*",
         priority: int = 100,
-        css: str = None,
-        xpath: str = None,
-        text: str = None,
-        regex: str = None,
-        group_css: str = None,
-        group_xpath: str = None,
-        group_text: str = None,
-        group_regex: str = None,
+        css: Optional[str] = None,
+        xpath: Optional[str] = None,
+        text: Optional[str] = None,
+        regex: Optional[str] = None,
+        group_css: Optional[str] = None,
+        group_xpath: Optional[str] = None,
+        group_text: Optional[str] = None,
+        group_regex: Optional[str] = None,
     ) -> Callable:
         """
         Decorator to register a handler function to a given selector.
@@ -198,11 +198,11 @@ def wrapper(func: Callable) -> Union[Callable, Coroutine]:
 
     def group(
         self,
-        selector: str = None,
-        css: str = None,
-        xpath: str = None,
-        text: str = None,
-        regex: str = None,
+        selector: Optional[str] = None,
+        css: Optional[str] = None,
+        xpath: Optional[str] = None,
+        text: Optional[str] = None,
+        regex: Optional[str] = None,
     ) -> Callable:
         """
         Decorator to register a handler function to a given group.
@@ -431,9 +431,9 @@ def run_event(self, event_name: str) -> None:
 class ScraperAbstract(ScraperBase):
     def __init__(
         self,
-        rules: List[Rule] = None,
-        groups: Dict[Callable, Selector] = None,
-        save_rules: Dict[Tuple[str, bool], Any] = None,
+        rules: Optional[List[Rule]] = None,
+        groups: Optional[Dict[Callable, Selector]] = None,
+        save_rules: Optional[Dict[Tuple[str, bool], Any]] = None,
         events: Optional[DefaultDict] = None,
         has_async: bool = False,
         requests: Optional[Deque] = None,

dude/optional/beautifulsoup_scraper.py

@@ -23,7 +23,7 @@ def run(
         self,
         urls: Sequence[str],
         pages: int = 1,
-        proxy: ProxiesTypes = None,
+        proxy: Optional[ProxiesTypes] = None,
         output: Optional[str] = None,
         format: str = "json",
         follow_urls: bool = False,
@@ -127,7 +127,7 @@ async def run_async(
                     if i == pages or not await self.navigate_async():
                         break
 
-    def setup(self, soup: BeautifulSoup = None) -> None:
+    def setup(self, soup: Optional[BeautifulSoup] = None) -> None:
         """
         This will only call the pre-setup and post-setup events if extra actions are needed to the soup object.
         :param soup: BeautifulSoup object
@@ -136,7 +136,7 @@ def setup(self, soup: BeautifulSoup = None) -> None:
         self.event_pre_setup(soup)
         self.event_post_setup(soup)
 
-    async def setup_async(self, soup: BeautifulSoup = None) -> None:
+    async def setup_async(self, soup: Optional[BeautifulSoup] = None) -> None:
         """
         This will only call the pre-setup and post-setup events if extra actions are needed to the soup object.
         :param soup: BeautifulSoup object
@@ -152,7 +152,7 @@ async def navigate_async(self) -> bool:
         return False
 
     def collect_elements(
-        self, soup: BeautifulSoup = None, url: str = None
+        self, soup: Optional[BeautifulSoup] = None, url: Optional[str] = None
     ) -> Iterable[Tuple[str, int, int, int, Any, Callable]]:
         assert soup is not None
         assert url is not None

dude/optional/lxml_scraper.py

@@ -24,7 +24,7 @@ def run(
         self,
         urls: Sequence[str],
         pages: int = 1,
-        proxy: ProxiesTypes = None,
+        proxy: Optional[ProxiesTypes] = None,
         output: Optional[str] = None,
         format: str = "json",
         follow_urls: bool = False,
@@ -131,7 +131,7 @@ async def run_async(
                     if i == pages or not await self.navigate_async():
                         break
 
-    def setup(self, tree: _ElementTree = None) -> None:
+    def setup(self, tree: Optional[_ElementTree] = None) -> None:
         """
         This will only call the pre-setup and post-setup events if extra actions are needed to the tree object.
         :param tree: _ElementTree object
@@ -140,7 +140,7 @@ def setup(self, tree: _ElementTree = None) -> None:
         self.event_pre_setup(tree)
         self.event_post_setup(tree)
 
-    async def setup_async(self, tree: _ElementTree = None) -> None:
+    async def setup_async(self, tree: Optional[_ElementTree] = None) -> None:
         """
         This will only call the pre-setup and post-setup events if extra actions are needed to the tree object.
         :param tree: _ElementTree object
@@ -156,7 +156,7 @@ async def navigate_async(self) -> bool:
         return False
 
     def collect_elements(
-        self, tree: _Element = None, url: str = None
+        self, tree: Optional[_Element] = None, url: Optional[str] = None
     ) -> Iterable[Tuple[str, int, int, int, Any, Callable]]:
         assert tree is not None
         assert url is not None

dude/optional/parsel_scraper.py

@@ -23,7 +23,7 @@ def run(
         self,
         urls: Sequence[str],
         pages: int = 1,
-        proxy: ProxiesTypes = None,
+        proxy: Optional[ProxiesTypes] = None,
         output: Optional[str] = None,
         format: str = "json",
         follow_urls: bool = False,
@@ -129,7 +129,7 @@ async def run_async(
                     if i == pages or not await self.navigate_async():
                         break
 
-    def setup(self, selector: ParselSelector = None) -> None:
+    def setup(self, selector: Optional[ParselSelector] = None) -> None:
         """
         This will only call the pre-setup and post-setup events if extra actions are needed to the selector object.
         :param selector: Selector object
@@ -138,7 +138,7 @@ def setup(self, selector: ParselSelector = None) -> None:
         self.event_pre_setup(selector)
         self.event_post_setup(selector)
 
-    async def setup_async(self, selector: ParselSelector = None) -> None:
+    async def setup_async(self, selector: Optional[ParselSelector] = None) -> None:
         """
         This will only call the pre-setup and post-setup events if extra actions are needed to the selector object.
         :param selector: Selector object
@@ -154,7 +154,7 @@ async def navigate_async(self) -> bool:
         return False
 
     def collect_elements(
-        self, selector: ParselSelector = None, url: str = None
+        self, selector: Optional[ParselSelector] = None, url: Optional[str] = None
     ) -> Iterable[Tuple[str, int, int, int, Any, Callable]]:
         assert selector is not None
         assert url is not None

dude/optional/selenium_scraper.py

@@ -73,7 +73,7 @@ def run(
             **kwargs,
         )
 
-    def setup(self, driver: WebDriver = None) -> None:
+    def setup(self, driver: Optional[WebDriver] = None) -> None:
         """
         Executes setup handlers
 
@@ -89,7 +89,7 @@ def setup(self, driver: WebDriver = None) -> None:
 
         self.event_post_setup(driver)
 
-    async def setup_async(self, driver: WebDriver = None) -> None:
+    async def setup_async(self, driver: Optional[WebDriver] = None) -> None:
         """
         Executes setup handlers
 
@@ -108,7 +108,7 @@ async def setup_async(self, driver: WebDriver = None) -> None:
 
         await self.event_post_setup_async(driver)
 
-    def navigate(self, driver: WebDriver = None) -> bool:
+    def navigate(self, driver: Optional[WebDriver] = None) -> bool:
         """
         Executes navigate handlers
 
@@ -122,7 +122,7 @@ def navigate(self, driver: WebDriver = None) -> bool:
                 return True
         return False
 
-    async def navigate_async(self, driver: WebDriver = None) -> bool:
+    async def navigate_async(self, driver: Optional[WebDriver] = None) -> bool:
         """
         Executes navigate handlers
 
@@ -262,7 +262,9 @@ def _get_driver(self, browser_type: str, headless: bool) -> WebDriver:
 
         return driver
 
-    def collect_elements(self, driver: WebDriver = None) -> Iterable[Tuple[str, int, int, int, Any, Callable]]:
+    def collect_elements(
+        self, driver: Optional[WebDriver] = None
+    ) -> Iterable[Tuple[str, int, int, int, Any, Callable]]:
         """
         Collects all the elements and returns a generator of element-handler pair.
         """