get_data.py

import sys
import json
import time
import random
import logging
import requests
from pathlib import Path
from typing import Tuple

from config import load_environment, validate_config, ensure_env_file, ROOT
from discord_data import (
    extract_user_data,
    extract_mutual_friends,
    extract_mutual_guilds,
    create_profile_output,
)

# Configure logging
logger = logging.getLogger(__name__)

# Validate environment
if not ensure_env_file():
    sys.exit(1)

config = load_environment()
if not validate_config(config, ['token', 'my_id']):
    sys.exit(1)

MY_ID = config['my_id'].strip()
TOKEN = config['token'].strip()
json_folder = config['json_folder']
MIN_DELAY = config['min_delay']
MAX_DELAY = config['max_delay']
BATCH_SIZE = config['batch_size']
BATCH_PAUSE = config['batch_pause']
REFRESH_DAYS = config['refresh_days']

json_folder.mkdir(parents=True, exist_ok=True)


def fetch_profile(user_id: str, token: str) -> dict:
    """
    Fetch user profile from Discord API.

    Args:
        user_id: Discord user ID
        token: Discord user token

    Returns:
        JSON response from Discord API

    Raises:
        requests.RequestException: If API request fails
    """
    url = f"https://discord.com/api/v9/users/{user_id}/profile"
    params = {
        'type': 'modal',
        'with_mutual_guilds': 'true',
        'with_mutual_friends': 'true',
        'with_mutual_friends_count': 'true',
    }
    headers = {
        'Authorization': token,
        'User-Agent': 'DiscordRelationshipVisualizer/1.0',
        'Accept': 'application/json',
    }

    logger.debug(f"GET {url}")
    response = requests.get(url, params=params, headers=headers, timeout=30)

    try:
        response.raise_for_status()
    except requests.RequestException as e:
        logger.error(f"API request failed: {e}")
        logger.error(f"Status: {response.status_code}")
        logger.error(f"Response: {response.text[:1000]}")
        raise

    return response.json()


def needs_refresh(path: Path) -> bool:
    """Check if a cached profile needs to be refreshed."""
    if not path.exists():
        return True
    mtime = path.stat().st_mtime
    age_days = (time.time() - mtime) / (60 * 60 * 24)
    return age_days >= REFRESH_DAYS


def main():
    """Fetch profile for the user and all mutual friends."""
    logger.info(f"Fetching profile for user {MY_ID}...")

    try:
        data = fetch_profile(MY_ID, TOKEN)
    except Exception as e:
        logger.error(f"Failed to fetch your profile: {e}")
        sys.exit(1)

    # Extract and save your profile
    user_data = extract_user_data(data)
    mutual_friends = extract_mutual_friends(data)
    mutual_guilds = extract_mutual_guilds(data)
    output = create_profile_output(user_data, mutual_friends, mutual_guilds)

    out_file = json_folder / f"{MY_ID}.json"
    with open(out_file, 'w', encoding='utf-8') as f:
        json.dump(output, f, indent=2, ensure_ascii=False)

    logger.info(f"Saved profile to {out_file}")
    logger.info(f"Mutual friends: {len(mutual_friends)} | Mutual guilds: {len(mutual_guilds)}")

    # Fetch friend profiles
    logger.info(
        f"Fetching {len(mutual_friends)} friend profiles "
        f"with {MIN_DELAY}-{MAX_DELAY}s delays, batch size {BATCH_SIZE}..."
    )

    fetched = 0
    for idx, friend in enumerate(mutual_friends, start=1):
        fid = friend.get('id')
        if not fid:
            continue

        friend_file = json_folder / f"{fid}.json"
        if not needs_refresh(friend_file):
            logger.debug(f"Skipping {fid} (recently fetched)")
            continue

        try:
            logger.info(f"[{idx}/{len(mutual_friends)}] Fetching {fid}...")
            fdata = fetch_profile(fid, TOKEN)

            # Extract and save friend profile
            f_user = extract_user_data(fdata)
            f_friends = extract_mutual_friends(fdata)
            f_guilds = extract_mutual_guilds(fdata)
            f_output = create_profile_output(f_user, f_friends, f_guilds)

            with open(friend_file, 'w', encoding='utf-8') as f:
                json.dump(f_output, f, indent=2, ensure_ascii=False)

            logger.debug(
                f"Wrote {friend_file.name} "
                f"(friends={len(f_friends)}, guilds={len(f_guilds)})"
            )
            fetched += 1

            # Rate limiting
            delay = random.uniform(MIN_DELAY, MAX_DELAY)
            logger.debug(f"Waiting {delay:.2f}s...")
            time.sleep(delay)

            # Batch pause
            if fetched % BATCH_SIZE == 0:
                logger.info(
                    f"Completed {fetched} fetches; "
                    f"pausing {BATCH_PAUSE}s to avoid rate limits..."
                )
                time.sleep(BATCH_PAUSE)

        except requests.RequestException as e:
            logger.error(f"Failed to fetch {fid}: {e}")
            continue
        except Exception as e:
            logger.error(f"Error processing {fid}: {e}")
            continue

    logger.info(f"Successfully fetched {fetched}/{len(mutual_friends)} friend profiles")


if __name__ == '__main__':
    main()