Esport

golgg_fetch_all_game_ids.py

@@ -0,0 +1,45 @@
+from playwright.async_api import async_playwright, Browser, Page, BrowserContext
+from src.utils.scrapers.golgg import GOLGG_TOURNAMENT_API, GOLGG_URL, GolggScraper
+from tqdm.asyncio import tqdm_asyncio
+import json
+
+
+async def main():
+    async with GolggScraper() as scraper:
+        tournaments = await scraper.get_tournaments_in_season(10)
+
+        # Step 1: Extract matches from tournaments
+        all_games = set()
+        print("Extracting matches from tournaments...")
+        match_tasks = [
+            scraper.get_matches_in_tournament(tournament["trname"])
+            for tournament in tournaments
+        ]
+        matches_by_tournament = await tqdm_asyncio.gather(
+            *match_tasks, desc="Tournaments", leave=True
+        )
+
+        # Flatten the matches list
+        all_matches = {match for matches in matches_by_tournament for match in matches}
+
+        # Step 2: Extract games from matches
+        print("Extracting games from matches...")
+        game_tasks = [scraper.get_games_ids_in_match(match) for match in all_matches]
+        games_by_match = await tqdm_asyncio.gather(
+            *game_tasks, desc="Matches", leave=True
+        )
+
+        # Combine all game IDs
+        for games in games_by_match:
+            all_games.update(games)
+
+    games = list(all_games)
+    print(f"Found {len(games)} games.")
+    with open("games.json", "w") as f:
+        json.dump(games, f, indent=4)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

requirements.txt

@@ -1,19 +1,106 @@
-attrs==24.2.0
-cattrs==24.1.2
-certifi==2024.8.30
-charset-normalizer==3.4.0
+absl-py==2.1.0
+annotated-types==0.7.0
+anyio==4.7.0
+asttokens==3.0.0
+beautifulsoup4==4.12.3
+certifi==2024.12.14
+charset-normalizer==3.4.1
+colorama==0.4.6
+comm==0.2.2
+contourpy==1.3.1
 cssselect==1.2.0
+cycler==0.12.1
+dacite==1.8.1
+debugpy==1.8.12
+decorator==5.1.1
+distro==1.9.0
+dnspython==2.7.0
+executing==2.1.0
+filelock==3.17.0
+fonttools==4.55.3
+fsspec==2025.2.0
+greenlet==3.1.1
+grpcio==1.70.0
+h11==0.14.0
+h5py==3.13.0
+httpcore==1.0.7
+httpx==0.28.1
 idna==3.10
+ipykernel==6.29.5
+ipython==8.31.0
+jedi==0.19.2
+Jinja2==3.1.5
+jiter==0.8.2
 jmespath==1.0.1
+joblib==1.4.2
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+kiwisolver==1.4.8
+llvmlite==0.44.0
 lxml==5.3.0
+Markdown==3.7
+markdownify==0.14.1
+MarkupSafe==3.0.2
+matplotlib==3.10.0
+matplotlib-inline==0.1.7
+motor==3.7.0
+mpmath==1.3.0
+nest-asyncio==1.6.0
+networkx==3.4.2
+numba==0.61.0
+numpy==2.1.3
+openai==1.63.0
+openskill==6.0.2
 packaging==24.2
+pandas==2.2.3
 parsel==1.9.1
+parso==0.8.4
+pillow==11.0.0
 platformdirs==4.3.6
-PoroPilot==0.2.10
+playwright==1.51.0
+prompt_toolkit==3.0.48
+protobuf==5.29.3
+psutil==6.1.1
+psycopg2-binary==2.9.10
+pure_eval==0.2.3
+pydantic==2.10.6
+pydantic_core==2.27.2
+pyee==12.1.1
+Pygments==2.19.1
+pymongo==4.11
+pynndescent==0.5.13
+pyparsing==3.2.0
+python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
+pytz==2024.2
+pywin32==308
+pyzmq==26.2.0
 requests==2.32.3
-requests-cache==1.2.1
-six==1.16.0
-url-normalize==1.4.3
-urllib3==2.2.3
+scikit-learn==1.6.1
+scipy==1.15.1
+seaborn==0.13.2
+setuptools==75.8.0
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.6
+SQLAlchemy==2.0.40
+stack-data==0.6.3
+sympy==1.13.1
+tenacity==9.0.0
+tensorboard==2.19.0
+tensorboard-data-server==0.7.2
+threadpoolctl==3.5.0
+torch==2.6.0
+torchaudio==2.6.0
+torchvision==0.21.0
+tornado==6.4.2
+tqdm==4.67.1
+traitlets==5.14.3
+trueskill==0.4.5
+typing_extensions==4.12.2
+tzdata==2024.2
+umap-learn==0.5.7
+urllib3==2.3.0
 w3lib==2.2.1
+wcwidth==0.2.13
+Werkzeug==3.1.3