feature: add new dating category for username modules, add 5 Estonian dating site modules (#621)

* feature: add new dating category for username modules, add 5 Estonian dating site modules

* fix: alternative missing user checks, bot detection error messages

* fix: sanity check Neanky profile username

* fix: sanity check Neanky profile username

Author: kristoisberg

user_scanner/user_scan/adult/iha.py

@@ -0,0 +1,135 @@
+import html
+import re
+from urllib.parse import quote
+
+import httpx
+
+from user_scanner.core.orchestrator import generic_validate
+from user_scanner.core.result import Result
+
+FIELD_NAMES = {
+    "Konto tüüp": "account_type",
+    "Nimi": "name",
+    "Sünniaeg": "birth_date",
+    "Riik": "country",
+    "Elukoht": "location",
+    "Kodulinn": "hometown",
+    "Kasv": "height_cm",
+    "Kaal": "weight_kg",
+    "Kehaehitus": "body_type",
+    "Juuste värv": "hair_color",
+    "Silmade värv": "eye_color",
+    "Orientatsioon": "orientation",
+    "Soovin tutvuda": "looking_for",
+    "Soovitud vanus": "desired_age",
+    "Tutvumise eesmärk": "intent",
+    "Mina (me) oleme": "relationship_status",
+    "Kas sa oled abielus?": "marital_status",
+    "Kas sul on lapsi?": "children",
+    "Keeled": "languages",
+    "Haridus": "education",
+    "Kool/Ülikool kus õppisin": "school",
+    "Hobid": "hobbies",
+    "Avalik e-maili aadress": "public_email",
+    "Kas sa suitsetad?": "smoking",
+    "Suhe alkoholi": "alcohol",
+    "Suhe narkootikumidesse": "drug_use",
+    "Sissetulek": "income",
+    "Minu roll seksis": "sexual_role",
+    "Kui tihti soovin seksida": "sexual_frequency",
+    "Mind erutab": "turn_ons",
+    "Kas sulle meeldib masturbeerida": "masturbation",
+    "BDSM": "bdsm",
+    "Liiklusvahend": "vehicle",
+    "Mind jahutab maha": "turn_offs",
+    "Muusika, mis meeldib": "favorite_music",
+    "Minu magamistoast leiad": "bedroom_items",
+    "Mis muudab minu jaoks seksi kvaliteetseks?": "sexual_preferences",
+    "Lemmikraamat": "favorite_book",
+    "Lemmikfilm": "favorite_movie",
+    "Ametikoht": "occupation",
+    "Minu ettekujutus ideaalsest esimesest kohtingust": "ideal_first_date",
+    "Mida õppisin oma viimasest suhtest": "relationship_lessons",
+}
+
+
+def validate_iha(user: str) -> Result:
+    username = user.strip()
+    url = f"https://www.iha.ee/users/{quote(username, safe='')}"
+
+    def process(response: httpx.Response) -> Result:
+        if response.status_code != 200:
+            return Result.error(f"Unexpected response status: {response.status_code}")
+
+        match = re.search(
+            r'<div class="member_username"[^>]*>\s*([^<]+)', response.text
+        )
+        if match:
+            profile_user = html.unescape(match.group(1)).strip()
+            if profile_user.casefold() != username.casefold():
+                return Result.error(
+                    "Profile response does not match the requested handle"
+                )
+            user_id = re.search(r'<form[^>]+(?:name|id)="_(\d+)"', response.text)
+            extra = _fields(response.text)
+            if user_id:
+                extra["user_id"] = int(user_id.group(1))
+            return Result.taken(
+                extra=extra,
+                media=_media(response.text, user_id.group(1) if user_id else None),
+            )
+
+        if re.search(
+            r"<title>\s*Iha\.ee - Seksikate inimeste kohtumispaik\s*</title>",
+            response.text,
+        ) and (
+            'id="online_list"' in response.text
+            or "Sellist kasutajat ei leitud" in response.text
+        ):
+            return Result.available()
+
+        return Result.error("Could not verify profile page")
+
+    return generic_validate(url, process, show_url=url)
+
+
+def _fields(page: str) -> dict[str, object]:
+    pairs = re.findall(
+        r'<div class="member_form_opt1">\s*<div[^>]*>(.*?)</div>\s*</div>\s*'
+        r'<div class="member_form_opt2">\s*<div[^>]*>(.*?)</div>\s*</div>',
+        page,
+        re.DOTALL,
+    )
+    fields: dict[str, object] = {}
+    for raw_label, raw_value in pairs:
+        label = _text(raw_label)
+        value = _text(raw_value)
+        if label not in FIELD_NAMES or not value:
+            continue
+        field = FIELD_NAMES[label]
+        if field == "birth_date":
+            value = re.sub(r"\s+\(\d+\)$", "", value)
+        fields[field] = value
+    return fields
+
+
+def _media(page: str, user_id: str | None) -> dict[str, str]:
+    if not user_id:
+        return {}
+    photos = list(
+        dict.fromkeys(
+            html.unescape(url)
+            for url in re.findall(r"https://img2\.iha\.ee/[^\"'? )]+", page)
+            if f"/{user_id}e" in url
+        )
+    )
+    if not photos:
+        return {}
+    return {
+        "avatar": photos[0],
+        **{f"photo_{index}": url for index, url in enumerate(photos[1:], 1)},
+    }
+
+
+def _text(value: str) -> str:
+    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", value))).strip()

user_scanner/user_scan/dating/__init__.py

@@ -0,0 +1 @@
+

user_scanner/user_scan/dating/amoremi.py

@@ -0,0 +1,75 @@
+import html
+import re
+from urllib.parse import quote
+
+import httpx
+
+from user_scanner.core.orchestrator import generic_validate
+from user_scanner.core.result import Result
+
+GENDERS = {"mehega": "male", "naisega": "female"}
+
+
+def validate_amoremi(user: str) -> Result:
+    username = user.strip()
+    url = f"https://www.amoremi.ee/user/{quote(username, safe='')}"
+
+    def process(response: httpx.Response) -> Result:
+        if response.status_code == 404 and (
+            "<title>Lehekülge ei leitud</title>" in response.text
+            or f'Kasutajat nimega "{html.escape(username)}" ei eksisteeri.'
+            in response.text
+        ):
+            return Result.available()
+        if response.status_code != 200:
+            return Result.error(f"Unexpected response status: {response.status_code}")
+
+        match = re.search(
+            r'<div class="profile-title text-center">\s*'
+            r'<div class=(?:"page-title"|page-title)>\s*([^<]+)',
+            response.text,
+        )
+        if not match:
+            return Result.error("Could not verify profile page")
+
+        profile_user = html.unescape(match.group(1)).strip()
+        if profile_user.casefold() != username.casefold():
+            return Result.error("Profile response does not match the requested handle")
+
+        extra: dict[str, object] = {}
+        breadcrumb = re.search(
+            r'<li class="breadcrumb-item active">\s*[^<]*?\s+(\d+)\s+a\.',
+            response.text,
+        )
+        if breadcrumb:
+            extra["age"] = int(breadcrumb.group(1))
+        if profile_type := re.search(
+            r'<a href="/tutvus/([^"/]+)">\s*([^<]+)</a>', response.text
+        ):
+            extra["profile_type"] = html.unescape(profile_type.group(2)).strip()
+            if profile_gender := GENDERS.get(profile_type.group(1)):
+                extra["gender"] = profile_gender
+        if description := re.search(
+            r"<div class=profile-description>.*?"
+            r'<div class="short-desc">(.*?)</div>',
+            response.text,
+            re.DOTALL,
+        ):
+            extra["bio"] = _text(description.group(1))
+        if seeking := re.search(
+            r"Tutvun\s+(mehega|naisega)\s+(\d+)-(\d+)\s+a\.", response.text
+        ):
+            extra["looking_for"] = GENDERS[seeking.group(1)]
+            extra["desired_age"] = f"{seeking.group(2)}-{seeking.group(3)}"
+        if title := re.search(r"<title>(.*?)</title>", response.text, re.DOTALL):
+            extra["headline"] = (
+                html.unescape(title.group(1)).strip().removesuffix(f" - {profile_user}")
+            )
+
+        return Result.taken(extra=extra)
+
+    return generic_validate(url, process, show_url=url)
+
+
+def _text(value: str) -> str:
+    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", value))).strip()

user_scanner/user_scan/dating/neanky.py

@@ -0,0 +1,95 @@
+import html
+import re
+from urllib.parse import quote, urljoin
+
+import httpx
+
+from user_scanner.core.orchestrator import generic_validate, make_request
+from user_scanner.core.result import Result
+
+
+def validate_neanky(user: str) -> Result:
+    username = user.strip()
+    url = f"https://neanky.ee/user/{quote(username, safe='')}"
+
+    def process_profile(response: httpx.Response) -> Result:
+        if (
+            response.status_code == 404
+            and "<title>Kasutajat ei leitud | Neanky" in response.text
+        ):
+            return Result.available()
+        if response.status_code != 200:
+            return Result.error(f"Unexpected response status: {response.status_code}")
+
+        if pending := re.search(
+            r"Soovitud kasutaja ei ole hetkel veel aktiveeritud!<br>\s*"
+            r"Konto registreeritud:\s*([^<]+)<br>",
+            response.text,
+        ):
+            return Result.taken(
+                extra={
+                    "account_status": "pending_activation",
+                    "registered_at": pending.group(1).strip(),
+                }
+            )
+
+        match = re.search(
+            r'<h1 class="u-box-name-txt-b text-white">(.*?)</h1>',
+            response.text,
+            re.DOTALL,
+        )
+        if not match:
+            return Result.taken()
+
+        profile_user = _text(match.group(1))
+        if profile_user.casefold() != username.casefold():
+            return Result.error("Profile response does not match the requested handle")
+
+        extra: dict[str, object] = {}
+        if name := re.search(r"Nimi:\s*<strong>(.*?)</strong>", response.text):
+            extra["name"] = _text(name.group(1))
+        if gender := re.search(r"Sugu:\s*<img[^>]*>\s*([^<]+)<br", response.text):
+            extra["gender"] = html.unescape(gender.group(1)).strip()
+        if user_id := re.search(r'\bvar vsID="(\d+)"', response.text):
+            extra["user_id"] = int(user_id.group(1))
+        if online_status := re.search(
+            r'<span[^>]+title="(online|offline)"', match.group(1)
+        ):
+            extra["online_status"] = online_status.group(1)
+        media: dict[str, str] = {}
+        avatar = re.search(
+            r"u-box-avatar-Pic[^>]+background-image:url\([\'\"]?([^\'\")]+)",
+            response.text,
+        )
+        if avatar and not avatar.group(1).endswith("/images/avatar.svg"):
+            media["avatar"] = urljoin("https://neanky.ee", avatar.group(1))
+        return Result.taken(extra=extra, media=media)
+
+    def process_availability(response: httpx.Response) -> Result:
+        if response.status_code != 200:
+            return Result.error(f"Unexpected response status: {response.status_code}")
+        data = response.json()
+        if data.get("user", {}).get("username", "").casefold() != username.casefold():
+            return Result.error(
+                "Availability response does not match the requested handle"
+            )
+        if data.get("state") == 1:
+            return Result.available()
+        if data.get("state") != 0:
+            return Result.error("Could not verify username availability")
+        try:
+            return process_profile(make_request(url))
+        except httpx.HTTPError as exc:
+            return Result.error(exc)
+
+    return generic_validate(
+        "https://neanky.ee/p/register",
+        process_availability,
+        method="POST",
+        data={"username": username, "type": "validateusername"},
+        show_url=url,
+    )
+
+
+def _text(value: str) -> str:
+    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", value))).strip()