Merge pull request #592 from arabcoders/dev

fix: Slowness in deleting big number of items

Author: arabcoders

API.md

@@ -96,6 +96,7 @@ This document describes the available endpoints and their usage. All endpoints r
     - [POST /api/notifications/test](#post-apinotificationstest)
     - [GET /api/yt-dlp/options](#get-apiyt-dlpoptions)
     - [GET /api/system/configuration](#get-apisystemconfiguration)
+    - [GET /api/system/limits](#get-apisystemlimits)
     - [POST /api/system/terminal](#post-apisystemterminal)
     - [GET /api/system/terminal/active](#get-apisystemterminalactive)
     - [GET /api/system/terminal/{session\_id}](#get-apisystemterminalsession_id)
@@ -132,6 +133,7 @@ This document describes the available endpoints and their usage. All endpoints r
         - [`item_updated`](#item_updated)
         - [`item_cancelled`](#item_cancelled)
         - [`item_deleted`](#item_deleted)
+        - [`item_bulk_deleted`](#item_bulk_deleted)
         - [`item_moved`](#item_moved)
         - [`item_status`](#item_status)
         - [`paused`](#paused)
@@ -381,7 +383,7 @@ or an error:
 **Body Parameters**:
 - `type` (string, required): Type of items - `"queue"` or `"done"`
 - `ids` (array, optional): List of specific item IDs to delete. If provided, `status` filter is ignored
-- `status` (string, optional): Filter by status (e.g., `"finished"`, `"!finished"`). Required if `ids` not provided
+- `status` (string, optional): Filter by status (e.g., `"finished"`, `"!finished"`, `"finished,skip"`, `"!finished,!skip"`). Required if `ids` not provided
 - `remove_file` (boolean, optional): Whether to delete files from disk. Default: `true`.
 
 > [!NOTE]
@@ -416,6 +418,15 @@ or an error:
 }
 ```
 
+**Delete all completed and skipped items in one request:**
+```json
+{
+  "type": "done",
+  "status": "finished,skip",
+  "remove_file": false
+}
+```
+
 **Response**:
 ```json
 {
@@ -448,6 +459,7 @@ or an error:
 **Notes**:
 - When using filter mode, all matching items will be deleted.
 - Filter mode with `{ "status": "!finished" }` is useful for cleaning up failed/pending downloads.
+- `status` also accepts comma-separated include filters (`finished,skip`) and comma-separated exclude filters (`!finished,!skip`).
 - Filter mode returns a `deleted` count indicating how many items were removed.
 
 ---
@@ -2479,6 +2491,55 @@ or an error:
 
 ---
 
+### GET /api/system/limits
+**Purpose**: Get the system limits.
+
+**Response**:
+```json
+{
+  "downloads": {
+    "paused": false,
+    "live_bypasses_limits": true,
+    "global": {
+      "limit": 20,
+      "active": 3,
+      "available": 17,
+      "live_active": 1,
+      "queued": 8
+    },
+    "per_extractor": {
+      "default_limit": 2,
+      "items": [
+        {
+          "name": "youtube",
+          "limit": 3,
+          "source": "env_override",
+          "active": 2,
+          "queued": 4,
+          "available": 1
+        }
+      ]
+    }
+  },
+  "extraction": {
+    "concurrency": 4,
+    "timeout_seconds": 70,
+    "info_cache_ttl_seconds": 10800
+  },
+  "live": {
+    "prevent_premiere": true,
+    "premiere_buffer_minutes": 5
+  }
+}
+```
+
+**Notes**:
+- `downloads.global` counts only non-live downloads against the worker limit.
+- `downloads.global.live_active` is reported separately because live downloads bypass the global and per-extractor worker limits.
+- `downloads.per_extractor.items[*].source` is `default` unless an override was provided through `YTP_MAX_WORKERS_FOR_<EXTRACTOR>`.
+
+---
+
 ### POST /api/system/terminal
 **Purpose**: Start a yt-dlp terminal session. Requires `YTP_CONSOLE_ENABLED=true`.
 
@@ -3127,6 +3188,24 @@ Emitted when a download item is deleted from the queue or history.
 
 ---
 
+##### `item_bulk_deleted`
+
+Emitted when multiple items are cleared in bulk operation.
+
+**Event**:
+```json
+{
+  "event": "item_bulk_deleted",
+  "data": {
+    "count": 10001,
+    "status": "finished,skip",
+    "ids": ["id1", "id2", "..."]  // optional, included when clear is driven by explicit ids
+  }
+}
+```
+
+---
+
 ##### `item_moved`
 
 Emitted when a download item is moved between queue and history.

FAQ.md

@@ -30,7 +30,6 @@ or the `environment:` section in `compose.yaml` file.
 | YTP_DEBUG                       | Whether to turn on debug mode                                       | `false`               |
 | YTP_DEBUGPY_PORT                | The port to use for the debugpy debugger                            | `5678`                |
 | YTP_EXTRACT_INFO_TIMEOUT        | The timeout for extracting video information                        | `70`                  |
-| YTP_UI_UPDATE_TITLE             | Whether to update the title of the page with the current stats      | `true`                |
 | YTP_PIP_PACKAGES                | A space separated list of pip packages to install                   | `(not_set)`           |
 | YTP_PIP_IGNORE_UPDATES          | Do not update the custom pip packages                               | `false`               |
 | YTP_PICTURES_BACKENDS           | A comma separated list of pictures urls to use                      | `(not_set)`           |

app/features/notifications/schemas.py

@@ -16,6 +16,7 @@ class NotificationEvents:
     ITEM_COMPLETED: str = Events.ITEM_COMPLETED
     ITEM_CANCELLED: str = Events.ITEM_CANCELLED
     ITEM_DELETED: str = Events.ITEM_DELETED
+    ITEM_BULK_DELETED: str = Events.ITEM_BULK_DELETED
     ITEM_PAUSED: str = Events.ITEM_PAUSED
     ITEM_RESUMED: str = Events.ITEM_RESUMED
     ITEM_MOVED: str = Events.ITEM_MOVED

app/library/DataStore.py

@@ -120,6 +120,48 @@ async def get_by_id(self, id: str) -> Download | None:
 
         return None
 
+    async def get_many_by_ids(self, ids: Iterable[str]) -> list[tuple[str, Download]]:
+        ids_list = list(ids)
+        if not ids_list:
+            return []
+
+        items: list[tuple[str, Download]] = []
+        missing_ids: list[str] = []
+
+        for item_id in ids_list:
+            cached = self._dict.get(item_id)
+            if cached:
+                items.append((item_id, cached))
+                continue
+            missing_ids.append(item_id)
+
+        if StoreType.HISTORY == self._type and missing_ids:
+            loaded = await self._connection.get_many_by_ids(str(self._type), missing_ids)
+            for item_id, item in loaded:
+                self._dict[item_id] = Download(info=item)
+
+        items.extend((item_id, download) for item_id in ids_list if (download := self._dict.get(item_id)))
+
+        seen: set[str] = set()
+        ordered: list[tuple[str, Download]] = []
+        for item_id, download in items:
+            if item_id in seen:
+                continue
+            seen.add(item_id)
+            ordered.append((item_id, download))
+        return ordered
+
+    async def get_many_by_status(self, status_filter: str) -> list[tuple[str, Download]]:
+        if StoreType.HISTORY != self._type:
+            return []
+
+        items = await self._connection.get_many_by_status(str(self._type), status_filter)
+        downloads: list[tuple[str, Download]] = []
+        for item_id, item in items:
+            self._dict[item_id] = Download(info=item)
+            downloads.append((item_id, self._dict[item_id]))
+        return downloads
+
     def items(self):
         return self._dict.items()
 
@@ -175,11 +217,41 @@ async def get_items_paginated(
         return [(item_id, Download(info=item)) for item_id, item in items], total_items, current_page, total_pages
 
     async def bulk_delete(self, ids: Iterable[str]) -> int:
-        deleted = await self._connection.bulk_delete(str(self._type), ids)
-        for _id in ids:
+        ids_list = list(ids)
+        deleted = await self._connection.bulk_delete(str(self._type), ids_list)
+        for _id in ids_list:
             self._dict.pop(_id, None)
         return deleted
 
+    async def bulk_delete_by_status(self, status_filter: str) -> int:
+        deleted = await self._connection.bulk_delete_by_status(str(self._type), status_filter)
+        if deleted > 0:
+            self._drop_cached_by_status(status_filter)
+        return deleted
+
+    def _drop_cached_by_status(self, status_filter: str) -> None:
+        raw_statuses = [entry.strip() for entry in status_filter.split(",") if entry.strip()]
+        if not raw_statuses:
+            return
+
+        if all(entry.startswith("!") for entry in raw_statuses):
+            excluded = {entry[1:].strip() for entry in raw_statuses if entry[1:].strip()}
+            if not excluded:
+                return
+
+            for item_id, download in list(self._dict.items()):
+                if download.info and download.info.status not in excluded:
+                    self._dict.pop(item_id, None)
+            return
+
+        included = {entry for entry in raw_statuses if not entry.startswith("!")}
+        if not included:
+            return
+
+        for item_id, download in list(self._dict.items()):
+            if download.info and download.info.status in included:
+                self._dict.pop(item_id, None)
+
     async def test(self) -> bool:
         await self._connection.count(str(self._type))
         return True

app/library/Events.py

@@ -37,6 +37,7 @@ class Events:
     ITEM_COMPLETED: str = "item_completed"
     ITEM_CANCELLED: str = "item_cancelled"
     ITEM_DELETED: str = "item_deleted"
+    ITEM_BULK_DELETED: str = "item_bulk_deleted"
     ITEM_PAUSED: str = "item_paused"
     ITEM_RESUMED: str = "item_resumed"
     ITEM_MOVED: str = "item_moved"
@@ -87,6 +88,7 @@ def frontend() -> list:
             Events.ITEM_UPDATED,
             Events.ITEM_CANCELLED,
             Events.ITEM_DELETED,
+            Events.ITEM_BULK_DELETED,
             Events.ITEM_MOVED,
             Events.ITEM_STATUS,
             Events.PAUSED,

app/library/config.py

@@ -129,9 +129,6 @@ class Config(metaclass=Singleton):
     apprise_config: str = "{config_path}{os_sep}apprise.yml"
     """The path to the Apprise configuration file."""
 
-    ui_update_title: bool = True
-    """Update the title of the browser tab with the current status."""
-
     pip_packages: str = ""
     """The pip packages to install."""
 
@@ -284,13 +281,12 @@ class Config(metaclass=Singleton):
         "access_log",
         "remove_files",
         "ignore_ui",
-        "ui_update_title",
         "pip_ignore_updates",
         "file_logging",
         "console_enabled",
         "browser_control_enabled",
         "ytdlp_auto_update",
-        "prevent_premiere_live",
+        "prevent_live_premiere",
         "temp_disabled",
         "allow_internal_urls",
         "simple_mode",
@@ -308,7 +304,6 @@ class Config(metaclass=Singleton):
         "output_template",
         "started",
         "remove_files",
-        "ui_update_title",
         "max_workers",
         "max_workers_per_extractor",
         "default_preset",

app/library/downloads/queue_manager.py

@@ -372,6 +372,103 @@ async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, st
 
         return status
 
+    async def clear_bulk(self, ids: list[str], remove_file: bool = False) -> dict[str, int | str]:
+        if not ids:
+            return {"deleted": 0}
+
+        items = await self.done.get_many_by_ids(ids)
+        if not items:
+            return {"deleted": 0}
+
+        if self.config.remove_files is not True:
+            remove_file = False
+
+        removed_files = 0
+        deleted_ids: list[str] = []
+        deleted_titles: list[str] = []
+
+        for item_id, item in items:
+            item_ref: str = f"{item_id=} {item.info.id=} {item.info.title=}"
+            filename: str = ""
+
+            LOG.debug(f"{remove_file=} {item_ref} - Removing local files: {item.info.status=}")
+
+            if remove_file and "finished" == item.info.status and item.info.filename:
+                filename = str(item.info.filename)
+                if item.info.folder:
+                    filename = f"{item.info.folder}/{item.info.filename}"
+
+                try:
+                    rf = Path(
+                        calc_download_path(
+                            base_path=Path(self.config.download_path),
+                            folder=filename,
+                            create_path=False,
+                        )
+                    )
+                    if rf.is_file() and rf.exists():
+                        if rf.stem and rf.suffix:
+                            for file_ref in rf.parent.glob(f"{glob.escape(rf.stem)}.*"):
+                                if file_ref.is_file() and file_ref.exists() and not file_ref.name.startswith("."):
+                                    removed_files += 1
+                                    LOG.debug(f"Removing '{item_ref}' local file '{file_ref.name}'.")
+                                    file_ref.unlink(missing_ok=True)
+                        else:
+                            LOG.debug(f"Removing '{item_ref}' local file '{rf.name}'.")
+                            rf.unlink(missing_ok=True)
+                            removed_files += 1
+                    else:
+                        LOG.warning(f"Failed to remove '{item_ref}' local file '{filename}'. File not found.")
+                except Exception as e:
+                    LOG.error(f"Unable to remove '{item_ref}' local file '{filename}'. {e!s}")
+
+            deleted_ids.append(item_id)
+            deleted_titles.append(item.info.title or item.info.id or item_id)
+
+        deleted_count = await self.done.bulk_delete(deleted_ids)
+        if deleted_count < 1:
+            return {"deleted": 0}
+
+        title = "History Removed" if removed_files > 0 else "History Cleared"
+        message = f"Removed {deleted_count} item{'s' if deleted_count != 1 else ''} from history."
+        if removed_files > 0:
+            message += f" Also removed {removed_files} local file{'s' if removed_files != 1 else ''}."
+
+        self._notify.emit(
+            Events.ITEM_BULK_DELETED,
+            data={"ids": deleted_ids, "count": deleted_count},
+            title=title,
+            message=message,
+        )
+
+        summary = ", ".join(deleted_titles[:5])
+        if deleted_count > 5:
+            summary += ", ..."
+        LOG.info(f"Bulk cleared {deleted_count} history item(s): {summary}")
+
+        return {"deleted": deleted_count}
+
+    async def clear_by_status(self, status_filter: str, remove_file: bool = False) -> dict[str, int | str]:
+        if self.config.remove_files is not True:
+            remove_file = False
+
+        if not remove_file:
+            deleted_count = await self.done.bulk_delete_by_status(status_filter)
+            if deleted_count < 1:
+                return {"deleted": 0}
+
+            self._notify.emit(
+                Events.ITEM_BULK_DELETED,
+                data={"count": deleted_count, "status": status_filter},
+                title="History Cleared",
+                message=f"Cleared {deleted_count} item{'s' if deleted_count != 1 else ''} from history.",
+            )
+            LOG.info(f"Bulk cleared {deleted_count} history item(s) by status '{status_filter}'.")
+            return {"deleted": deleted_count}
+
+        items = await self.done.get_many_by_status(status_filter)
+        return await self.clear_bulk([item_id for item_id, _ in items], remove_file=remove_file)
+
     async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
         """
         Get the download queue and the download history.