fix: rewrite fenland scraper to use selenium for cloudflare bypass #2085
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,65 +1,85 @@ | ||
| import json | ||
|
|
||
| from selenium.webdriver.common.by import By | ||
| from selenium.webdriver.support import expected_conditions as EC | ||
| from selenium.webdriver.support.ui import WebDriverWait | ||
|
|
||
| from uk_bin_collection.uk_bin_collection.common import * | ||
| from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass | ||
|
|
||
|
|
||
| class CouncilClass(AbstractGetBinDataClass): | ||
| """ | ||
| Fenland's GIS layer endpoint is behind Cloudflare JS challenge. | ||
| Load the page in Selenium to pass the challenge, then fetch the | ||
| JSON API from within the browser context. | ||
| """ | ||
|
|
||
| def parse_data(self, page: str, **kwargs) -> dict: | ||
| user_uprn = kwargs.get("uprn") | ||
| check_uprn(user_uprn) | ||
|
|
||
| headless = kwargs.get("headless") | ||
| web_driver = kwargs.get("web_driver") | ||
| user_agent = ( | ||
| "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " | ||
| "(KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36" | ||
| ) | ||
| driver = create_webdriver(web_driver, headless, user_agent, __name__) | ||
|
|
||
| try: | ||
| driver.execute_cdp_cmd( | ||
| "Page.addScriptToEvaluateOnNewDocument", | ||
| { | ||
| "source": "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" | ||
| }, | ||
|
Comment on lines
+28
to
+35
Contributor
There was a problem hiding this comment. 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
python - <<'PY'
from pathlib import Path
for path in Path("uk_bin_collection").rglob("common.py"):
text = path.read_text()
marker = "def create_webdriver("
start = text.find(marker)
if start == -1:
continue
rest = text[start:]
next_def = rest.find("\ndef ", 1)
snippet = rest if next_def == -1 else rest[:next_def]
print(f"== {path} ==")
print(snippet)
print()
PY
rg -n -C2 'execute_cdp_cmd|Page.addScriptToEvaluateOnNewDocument|web_driver|Chrom(e|ium)' \
uk_bin_collection/uk_bin_collection/councils/FenlandDistrictCouncil.py \
uk_bin_collection/tests/input.jsonRepository: robbrad/UKBinCollectionData Length of output: 50383 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Show the FenlandDistrictCouncil.py content around the cited areas
python - <<'PY'
import itertools
from pathlib import Path
path = Path("uk_bin_collection/uk_bin_collection/councils/FenlandDistrictCouncil.py")
lines = path.read_text().splitlines()
for start,end in [(1,120),(120,220)]:
print(f"\n===== {path} lines {start}-{end} =====")
for i in range(start-1, min(end, len(lines))):
print(f"{i+1:4d}: {lines[i]}")
PY
# Extract the Fenland entry from tests/input.json
python - <<'PY'
import json
from pathlib import Path
p = Path("uk_bin_collection/tests/input.json")
data = json.loads(p.read_text())
# Find key that matches Fenland (case-sensitive key lookup)
for k in list(data.keys()):
if "Fenland" in k:
print(f"\n=== input.json entry: {k} ===")
print(json.dumps(data[k], indent=2)[:4000])
break
else:
print("No key containing 'Fenland' found in input.json top-level.")
PY
# Search within FenlandDistrictCouncil.py for how JSON is retrieved/parsed
rg -n "execute_async_script|features|upcoming|json|response|status|webdriver|CDP|addScriptToEvaluateOnNewDocument" \
uk_bin_collection/uk_bin_collection/councils/FenlandDistrictCouncil.pyRepository: robbrad/UKBinCollectionData Length of output: 4087 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Find Fenland entry textually (avoid JSON decoding issues)
rg -n -C3 "Fenland" uk_bin_collection/tests/input.json | head -n 200
# Inspect create_webdriver implementation in common.py (to see if any enforcement/guards exist)
python - <<'PY'
from pathlib import Path
p = Path("uk_bin_collection/uk_bin_collection/common.py")
text = p.read_bytes().decode("utf-8", errors="replace").splitlines()
for i,line in enumerate(text, start=1):
if "def create_webdriver" in line:
start=i
break
else:
raise SystemExit("create_webdriver not found")
end = min(len(text), start+120)
for j in range(start, end+1):
print(f"{j:4d}: {text[j-1]}")
PY
# Search for any other guard around execute_cdp_cmd / Chromium in the codebase
rg -n "execute_cdp_cmd\\(|Page\\.addScriptToEvaluateOnNewDocument|CDP|Chrom(e|ium)" uk_bin_collection/uk_bin_collection | head -n 200Repository: robbrad/UKBinCollectionData Length of output: 37389 Make the Chromium/CDP requirement explicit and validate the API response shape/status
🤖 Prompt for AI Agents |
||
| ) | ||
|
|
||
| page_url = "https://www.fenland.gov.uk/article/13114/" | ||
| driver.get(page_url) | ||
|
|
||
| WebDriverWait(driver, 20).until( | ||
| lambda d: d.title != "Just a moment..." | ||
| ) | ||
|
|
||
| api_url = ( | ||
| f"/article/13114/?type=loadlayer&layerId=2" | ||
| f"&uprn={user_uprn}&lat=0.000000000001&lng=0.000000000001" | ||
| ) | ||
|
|
||
| result = driver.execute_async_script( | ||
| """ | ||
| var callback = arguments[arguments.length - 1]; | ||
| fetch(arguments[0], { | ||
| headers: { | ||
| 'X-Requested-With': 'XMLHttpRequest', | ||
| 'Accept': 'application/json' | ||
| } | ||
| }) | ||
| .then(r => r.text()) | ||
| .then(t => callback(t)) | ||
| .catch(e => callback('ERROR: ' + e)); | ||
| """, | ||
| api_url, | ||
| ) | ||
|
|
||
| if result.startswith("ERROR:"): | ||
| raise ValueError(f"API fetch failed: {result}") | ||
|
|
||
| json_data = json.loads(result)["features"][0]["properties"]["upcoming"] | ||
|
coderabbitai[bot] marked this conversation as resolved.
Outdated
|
||
| data = {"bins": []} | ||
|
|
||
| for item in json_data: | ||
| for bin_info in item["collections"]: | ||
| data["bins"].append( | ||
| { | ||
| "type": bin_info["desc"], | ||
| "collectionDate": datetime.strptime( | ||
| bin_info["collectionDate"], "%Y-%m-%dT%H:%M:%SZ" | ||
| ).strftime(date_format), | ||
| } | ||
| ) | ||
|
|
||
| return data | ||
| finally: | ||
| driver.quit() | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.