feat: rewrite north west leicestershire as pure http scraper

uk_bin_collection/tests/input.json

@@ -1824,13 +1824,13 @@
         "LAD24CD": "E08000022"
     },
     "NorthWestLeicestershire": {
+        "postcode": "DE74 2FZ",
+        "house_number": "1",
         "postcode": "DE74 2FZ",
         "skip_get_url": true,
-        "uprn": "100030572613",
-        "url": "https://www.nwleics.gov.uk/pages/collection_information",
-        "web_driver": "http://selenium:4444",
+        "url": "https://my.nwleics.gov.uk/",
         "wiki_name": "North West Leicestershire",
-        "wiki_note": "Pass the postcode and UPRN. This parser requires a Selenium webdriver.",
+        "wiki_note": "Pass the postcode and house number. No UPRN or Selenium needed.",
         "LAD24CD": "E07000134"
     },
     "NorthYorkshire": {

uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py

@@ -1,17 +1,11 @@
-import re  # Import regular expressions
+import re
 from datetime import datetime, timedelta
 
 from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.support.wait import WebDriverWait
 
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
-# import the wonderful Beautiful Soup and the URL grabber
-
 
 class CouncilClass(AbstractGetBinDataClass):
     """
@@ -20,96 +14,99 @@ class CouncilClass(AbstractGetBinDataClass):
     implementation.
     """
 
+    AUTOCOMPLETE_URL = (
+        "https://my.nwleics.gov.uk/data/ac/addresses.json"
+    )
+    LOCATION_URL = "https://my.nwleics.gov.uk/location"
+    HOME_URL = "https://my.nwleics.gov.uk/"
+
     def parse_data(self, page: str, **kwargs) -> dict:
-        driver = None
-        try:
-            data = {"bins": []}
-
-            user_uprn = kwargs.get("uprn")
-            user_postcode = kwargs.get("postcode")
-            web_driver = kwargs.get("web_driver")
-            headless = kwargs.get("headless")
-            check_uprn(user_uprn)
-            check_postcode(user_postcode)
-            # Create Selenium webdriver
-            page = f"https://my.nwleics.gov.uk/my-property-finder?address={user_postcode}&go=1"
-
-            user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
-            driver = create_webdriver(web_driver, headless, user_agent, __name__)
-            driver.get(page)
-
-            # If you bang in the house number (or property name) and postcode in the box it should find your property
-
-            # iframe_presense = WebDriverWait(driver, 30).until(
-            #    EC.presence_of_element_located((By.ID, "fillform-frame-1"))
-            # )
-
-            # driver.switch_to.frame(iframe_presense)
-            wait = WebDriverWait(driver, 60)
-
-            address_link = wait.until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, f'//a[contains(@href, "{user_uprn}")]')
-                )
+        user_postcode = kwargs.get("postcode")
+        user_paon = kwargs.get("paon")
+        check_postcode(user_postcode)
+
+        nwl_id = self._resolve_address(user_postcode, user_paon)
+
+        session = requests.Session()
+        session.headers.update(
+            {
+                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
+            }
+        )
+
+        session.get(
+            self.LOCATION_URL,
+            params={"put": nwl_id, "rememberme": "0", "redirect": "/"},
+            allow_redirects=True,
+        )
+
+        response = session.get(self.HOME_URL)
+        soup = BeautifulSoup(response.text, features="html.parser")
+
+        refuse_list = soup.find("ul", class_="refuse")
+        if not refuse_list:
+            raise ValueError(
+                "No refuse collection data found for this address"
             )
 
-            address_link.click()
-
-            refuse_element = wait.until(
-                EC.presence_of_element_located(
-                    (By.XPATH, f'//h3[contains(text(), "Refuse Collection Dates")]')
-                )
+        data = {"bins": []}
+        current_year = datetime.now().year
+        current_date = datetime.now().date()
+
+        for li in refuse_list.find_all("li"):
+            date_tag = li.find("strong", class_="date")
+            link_tag = li.find("a")
+            if not date_tag or not link_tag:
+                continue
+
+            date_str = date_tag.text.strip()
+            waste_type = link_tag.text.strip()
+
+            if date_str.lower() == "today":
+                parsed_date = current_date
+            elif date_str.lower() == "tomorrow":
+                parsed_date = current_date + timedelta(days=1)
+            else:
+                date_str = re.sub(r"(st|nd|rd|th)", "", date_str)
+                parsed_date = datetime.strptime(date_str, "%a %d %b").date()
+
+            if parsed_date.year < current_date.year:
+                parsed_date = parsed_date.replace(year=current_year)
+
+            if parsed_date < current_date:
+                parsed_date = parsed_date.replace(year=current_year + 1)
+
+            data["bins"].append(
+                {
+                    "type": waste_type,
+                    "collectionDate": parsed_date.strftime(date_format),
+                }
             )
 
-            soup = BeautifulSoup(driver.page_source, features="html.parser")
-
-            # Find the unordered list containing refuse collection details
-            refuse_list = soup.find("ul", class_="refuse")
-
-            current_year = datetime.now().year
-
-            if refuse_list:
-                # Iterate through list items within the unordered list
-                for li in refuse_list.find_all("li"):
-                    date = li.find(
-                        "strong", class_="date"
-                    ).text.strip()  # Extract the date
-                    waste_type = li.find("a").text.strip()  # Extract the waste type
-
-                    # Parse the date from the string
-                    # check for today and tomorrow
-                    if date.lower() == "today":
-                        parsed_date = datetime.now().date()
-                    elif date.lower() == "tomorrow":
-                        parsed_date = (datetime.now() + timedelta(days=1)).date()
-                    else:
-                        date = re.sub(r"(st|nd|rd|th)", "", date)
-                        parsed_date = datetime.strptime(date, "%a %d %b").date()
-
-                    current_date = datetime.now().date()
-
-                    # double check we've got a year and if not the current one
-                    if parsed_date.year < current_date.year:
-                        parsed_date = parsed_date.replace(year=current_date.year)
-
-                    # check if the date is in the past and if so add a year
-                    if parsed_date < current_date:
-                        parsed_date = parsed_date.replace(year=current_date.year + 1)
-
-                    # Append data to your 'bins' list (this replicates your existing logic)
-                    data["bins"].append(
-                        {
-                            "type": waste_type,
-                            "collectionDate": parsed_date.strftime("%d/%m/%Y"),
-                        }
-                    )
-        except Exception as e:
-            # Here you can log the exception if needed
-            print(f"An error occurred: {e}")
-            # Optionally, re-raise the exception if you want it to propagate
-            raise
-        finally:
-            # This block ensures that the driver is closed regardless of an exception
-            if driver:
-                driver.quit()
         return data
+
+    def _resolve_address(self, postcode: str, house_number: str = None) -> str:
+        response = requests.get(
+            self.AUTOCOMPLETE_URL, params={"term": postcode}
+        )
+        response.raise_for_status()
+        results = response.json()
+
+        if not results:
+            raise ValueError(f"No addresses found for postcode {postcode}")
+
+        if house_number:
+            house_lower = house_number.lower().strip()
+            for entry in results:
+                label = entry.get("label", "").lower()
+                if label.startswith(house_lower + ",") or label.startswith(
+                    house_lower + " "
+                ):
+                    return entry["value"]
+
+        if len(results) == 1:
+            return results[0]["value"]
+
+        raise ValueError(
+            f"Multiple addresses found for {postcode} — provide house_number to disambiguate"
+        )