Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions API.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ This document describes the available endpoints and their usage. All endpoints r
- [POST /api/notifications/test](#post-apinotificationstest)
- [GET /api/yt-dlp/options](#get-apiyt-dlpoptions)
- [GET /api/system/configuration](#get-apisystemconfiguration)
- [GET /api/system/diagnostics](#get-apisystemdiagnostics)
- [GET /api/system/limits](#get-apisystemlimits)
- [POST /api/system/terminal](#post-apisystemterminal)
- [GET /api/system/terminal](#get-apisystemterminal)
Expand Down Expand Up @@ -2632,6 +2633,54 @@ or an error:

---

### GET /api/system/diagnostics
**Purpose**: View system information.

**Response**:
```json
{
"status": "error",
"generated_at": 1713000000,
"summary": {
"total": 10,
"pass": 5,
"fail": 2,
"warn": 1,
"skip": 2,
"required_failed": 2
},
"runtime": {
"app_version": "1.0.0",
"app_branch": "main",
"app_commit_sha": "abcdef12",
"app_build_date": "20260526",
"started": 1712999900,
"uptime_seconds": 100,
"platform": "linux",
"platform_release": "6.8.0",
"platform_machine": "x86_64",
"python_version": "3.13.1",
"python_minimum": "3.13",
"is_native": false,
"console_enabled": false
},
"requirements": {
"python": {
"current": "3.13.1",
"required": "3.13",
"supported": true,
"note": ""
}
},
"checks": []
}
```

**Notes**:
- Unexpected collection errors are returned as an `error`.

---

### GET /api/system/limits
**Purpose**: Get the system limits.

Expand Down
31 changes: 30 additions & 1 deletion app/features/ytdlp/extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

LOG: logging.Logger = logging.getLogger("downloads.extractor")

LIVE_REEXTRACT_STATUSES: set[str] = {"is_live", "post_live"}
REEXTRACT_INFO_KEY = "_ytptube_reextract"


def _ytdlp_logger(target: logging.Logger):
return _YTDLPLogger(target)
Expand Down Expand Up @@ -247,6 +250,23 @@ def _sanitize_config(config: dict[str, Any]) -> dict[str, Any]:
return sanitized


def needs_reextract(info: dict[str, Any]) -> bool:
return bool(info.get("is_live") or info.get("live_status") in LIVE_REEXTRACT_STATUSES)


def _process_safe_info(info: dict[str, Any]) -> dict[str, Any]:
if needs_reextract(info):
info = dict(info)
info[REEXTRACT_INFO_KEY] = True
for key in ("formats", "requested_formats", "requested_downloads", "fragments"):
info.pop(key, None)

if _is_picklable(info):
return info

return YTDLP.sanitize_info(info, remove_private_keys=False)


def extract_info_sync(
config: dict[str, Any],
url: str,
Expand All @@ -255,6 +275,7 @@ def extract_info_sync(
follow_redirect: bool = False,
sanitize_info: bool = False,
capture_logs: int | None = None,
process_safe: bool = False,
**kwargs,
) -> tuple[dict[str, Any] | None, list[str]]:
"""
Expand All @@ -268,6 +289,7 @@ def extract_info_sync(
follow_redirect: Follow URL redirects
sanitize_info: Sanitize the extracted information
capture_logs: If provided (e.g., logging.WARNING), capture logs at this level.
process_safe: Strip non-pickleable data for safe inter-process communication.
**kwargs: Additional arguments

Returns:
Expand Down Expand Up @@ -330,6 +352,7 @@ def extract_info_sync(
follow_redirect=follow_redirect,
sanitize_info=sanitize_info,
capture_logs=capture_logs,
process_safe=process_safe,
captured_logs=captured_logs,
**kwargs,
)
Expand All @@ -348,6 +371,9 @@ def extract_info_sync(

result = YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data

if process_safe and isinstance(result, dict):
result = _process_safe_info(result)

return (result, captured_logs)
finally:
logging.Logger.manager.loggerDict.pop(logger_name, None)
Expand Down Expand Up @@ -404,6 +430,8 @@ async def fetch_info(
loop = asyncio.get_running_loop()

safe_config = _sanitize_config(config)
safe_kwargs = _sanitize_picklable(kwargs)
safe_kwargs.pop("process_safe", None)
timeout = _sleep_timeout(safe_config, extractor_config.timeout, budget_sleep)

try:
Expand All @@ -422,7 +450,8 @@ async def fetch_info(
follow_redirect=follow_redirect,
sanitize_info=sanitize_info,
capture_logs=capture_logs,
**kwargs,
process_safe=True,
**safe_kwargs,
),
),
timeout=timeout,
Expand Down
40 changes: 40 additions & 0 deletions app/features/ytdlp/tests/test_ytdlp_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
from app.features.ytdlp.extractor import (
ExtractorConfig,
ExtractorPool,
REEXTRACT_INFO_KEY,
_LogCapture,
_get_process_pool_kwargs,
_process_safe_info,
_ytdlp_logger,
extract_info_sync,
)
Expand Down Expand Up @@ -158,6 +160,44 @@ def fake_extract_info(url, download=False): # noqa: ARG001
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
assert (logging.WARNING, "[generic_browser] Browser fallback warning") in seen

def test_process_safe_live(self) -> None:
data = {
"id": "live-id",
"is_live": True,
"formats": [{"format_id": "dash", "fragments": ({"url": "https://example.test/sq/1"} for _ in range(1))}],
"requested_formats": [{"format_id": "dash"}],
}

result = _process_safe_info(data)

assert result[REEXTRACT_INFO_KEY] is True
assert "formats" not in result
assert "requested_formats" not in result
pickle.dumps(result)

def test_process_safe_post_live(self) -> None:
data = {
"id": "post-live-id",
"live_status": "post_live",
"formats": [{"format_id": "dash", "fragments": ({"url": "https://example.test/sq/1"} for _ in range(1))}],
}

result = _process_safe_info(data)

assert result[REEXTRACT_INFO_KEY] is True
assert "formats" not in result
pickle.dumps(result)

def test_process_safe_lazy(self) -> None:
from yt_dlp.utils import LazyList

data = {"id": "video-id", "formats": LazyList({"format_id": str(i)} for i in range(2))}

result = _process_safe_info(data)

assert result["formats"] == [{"format_id": "0"}, {"format_id": "1"}]
pickle.dumps(result)


class TestYtdlpLogger:
def test_debug_prefix_uses_debug(self) -> None:
Expand Down
Loading
Loading