Merge pull request #1597 from Capsize-Games/develop

auto detects language and responds in that language (tts and llm)

Author: w4ffl35

setup.py

@@ -46,6 +46,7 @@
         # Summarizations (basic)
         "sumy==0.11.0",
         "sentencepiece==0.2.0",
+        "lingua-language-detector==2.1.0",
     ],
     "agents": [
         "llama-index==0.12.36",
@@ -106,6 +107,8 @@
     ],
     "openvoice_kr": [
         "jamo==0.4.1",
+        "python-mecab-ko==1.3.7",
+        "python-mecab-ko-dic==2.1.1.post2",
     ],
     "openvoice_tw": [
         "g2pkk>=0.1.2",

src/airunner/alembic/versions/82e99b1a4ccc_add_detected_language_columsn_to_.py

@@ -0,0 +1,30 @@
+"""add detected language columsn to application settings'
+
+
+Revision ID: 82e99b1a4ccc
+Revises: 72977a42e2a2
+Create Date: 2025-05-18 16:02:59.334823
+
+"""
+
+from typing import Sequence, Union
+
+from airunner.data.models.application_settings import ApplicationSettings
+from airunner.utils.db import add_column, drop_column
+
+
+# revision identifiers, used by Alembic.
+revision: str = "82e99b1a4ccc"
+down_revision: Union[str, None] = "72977a42e2a2"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    add_column(ApplicationSettings, "detected_language")
+    add_column(ApplicationSettings, "use_detected_language")
+
+
+def downgrade() -> None:
+    drop_column(ApplicationSettings, "detected_language")
+    drop_column(ApplicationSettings, "use_detected_language")

src/airunner/api.py

@@ -862,6 +862,13 @@ def __init__(self, *args, **kwargs):
                     "tohoku-nlp/bert-base-japanese-v3",
                 )
             ),
+            "hfl/chinese-roberta-wwm-ext-large": os.path.expanduser(
+                os.path.join(
+                    self.path_settings.base_path,
+                    "text/models/tts",
+                    "hfl/chinese-roberta-wwm-ext-large",
+                )
+            ),
         }
         self._initialized = True
         self.llm = LLMAPIService(emit_signal=self.emit_signal)

src/airunner/data/bootstrap/openvoice_bootstrap_data.py

@@ -93,4 +93,15 @@
             "vocab.txt",
         ]
     },
+    "hfl/chinese-roberta-wwm-ext-large": {
+        "files": [
+            "added_tokens.json",
+            "config.json",
+            "pytorch_model.bin",
+            "special_tokens_map.json",
+            "tokenizer.json",
+            "tokenizer_config.json",
+            "vocab.txt",
+        ]
+    },
 }

src/airunner/data/models/application_settings.py

@@ -6,6 +6,7 @@
     GeneratorSection,
     CanvasToolName,
     Mode,
+    AvailableLanguage,
 )
 
 
@@ -64,3 +65,5 @@ class ApplicationSettings(BaseModel):
     nodegraph_zoom = Column(Integer, default=0)
     nodegraph_center_x = Column(Integer, default=0)
     nodegraph_center_y = Column(Integer, default=0)
+    detected_language = Column(String, default=AvailableLanguage.EN.value)
+    use_detected_language = Column(Boolean, default=True)

src/airunner/gui/widgets/llm/chat_prompt_widget.py

@@ -5,6 +5,7 @@
 from PySide6.QtCore import Slot, QPropertyAnimation, QTimer, Qt
 from PySide6.QtWidgets import QSpacerItem, QSizePolicy, QApplication
 
+from airunner.data.models.application_settings import ApplicationSettings
 from airunner.enums import (
     SignalCode,
     LLMActionType,
@@ -15,13 +16,13 @@
 from airunner.gui.widgets.llm.templates.chat_prompt_ui import Ui_chat_prompt
 from airunner.gui.widgets.llm.message_widget import MessageWidget
 from airunner.data.models import Conversation
+from airunner.utils.llm.language import detect_language
 from airunner.utils.llm.strip_names_from_message import (
     strip_names_from_message,
 )
 from airunner.utils import create_worker
 from airunner.utils.widgets import load_splitter_settings
 from airunner.handlers.llm.llm_request import LLMRequest
-from airunner.handlers.llm.llm_response import LLMResponse
 from airunner.workers.llm_response_worker import LLMResponseWorker
 from airunner.settings import AIRUNNER_ART_ENABLED
 
@@ -342,6 +343,12 @@ def do_generate(self, prompt_override=None):
             self.logger.warning("Prompt is empty")
             return
 
+        language = detect_language(prompt)
+        ApplicationSettings.objects.update(
+            self.application_settings.id,
+            detected_language=language,
+        )
+
         if self.generating:
             if self.held_message is None:
                 self.held_message = prompt

src/airunner/gui/widgets/stablediffusion/stablediffusion_generator_form.py

@@ -156,7 +156,7 @@ def __init__(self, *args, **kwargs):
             self.generator_settings.quality_effects
         )
         self.ui.infinite_images_button.setChecked(
-            self.generator_settings.generate_infinite_images
+            self.generator_settings.generate_infinite_images if self.generator_settings.generate_infinite_images is not None else False
         )
         self.ui.quality_effects.blockSignals(False)
         self.ui.infinite_images_button.blockSignals(False)

src/airunner/handlers/llm/agent/agents/base.py

@@ -119,6 +119,41 @@ def __init__(
         )
         super().__init__(*args, **kwargs)
 
+    @property
+    def language(self) -> str:
+        # Use a private attribute to avoid recursion
+        if hasattr(self, "_language") and self._language is not None:
+            lang = self._language
+        elif hasattr(self, "application_settings") and getattr(
+            self.application_settings, "use_detected_language", False
+        ):
+            lang = getattr(
+                self.application_settings, "detected_language", "EN"
+            )
+        else:
+            lang = "EN"
+        # Map language codes to full names if needed
+        if lang == "FR":
+            return "French"
+        elif lang == "DE":
+            return "German"
+        elif lang == "ES":
+            return "Spanish"
+        elif lang == "KO":
+            return "Korean"
+        elif lang == "RU":
+            return "Russian"
+        elif lang == "ZH":
+            return "Chinese"
+        elif lang == "JA":
+            return "Japanese"
+        else:
+            return "English"
+
+    @language.setter
+    def language(self, value: str):
+        self._language = value
+
     @property
     def use_memory(self) -> bool:
         use_memory = self._use_memory
@@ -1027,6 +1062,7 @@ def system_prompt(self) -> str:
             )
         else:
             conversation_timestamp_prompt = ""
+
         prompt = (
             f"Your name is {self.botname}.\n"
             f"- The user ({self.username}) is having a conversation with the assistant ({self.botname}).\n"
@@ -1066,7 +1102,12 @@ def system_prompt(self) -> str:
             f"The conversation is between user ({self.username}) and assistant ({self.botname}).\n"
             f"{conversation_timestamp_prompt}"
             f"{section_prompt}"
+            f"------\n"
         )
+
+        if self.language:
+            prompt += "Response in " + self.language + "\n"
+
         prompt = prompt.replace("{{ username }}", self.username)
         prompt = prompt.replace("{{ botname }}", self.botname)
         prompt = prompt.replace("{{ speaker_name }}", self.username)

src/airunner/handlers/tts/openvoice_model_manager.py

@@ -3,6 +3,7 @@
 import os
 import torch
 from airunner.settings import AIRUNNER_BASE_PATH
+from airunner.utils.llm.language import detect_language
 
 torch.hub.set_dir(
     os.environ.get("TORCH_HOME", os.path.join(AIRUNNER_BASE_PATH, "torch/hub"))
@@ -127,10 +128,28 @@ def __init__(self, *args, **kwargs):
         self.model: Optional[TTS] = None
         self.src_path: str = f"{self._output_dir}/tmp.wav"
         self._speed: float = 1.0
-        self._language: AvailableLanguage = AvailableLanguage(
-            self.openvoice_settings.language
-        )
         self._reference_speaker = speaker_recording_path
+        self._language: AvailableLanguage = (
+            AvailableLanguage.EN_NEWEST
+        )  # Use a private attribute
+
+    @property
+    def language(self) -> AvailableLanguage:
+        """
+        Get the language setting for TTS.
+        """
+        if hasattr(self, "application_settings") and getattr(
+            self.application_settings, "use_detected_language", False
+        ):
+            language = self.application_settings.detected_language
+            lang = AvailableLanguage[language]
+        else:
+            lang = self._language
+        return lang
+
+    @language.setter
+    def language(self, value: AvailableLanguage):
+        self._language = value
 
     @property
     def device(self):
@@ -156,42 +175,77 @@ def tone_color_converter(self) -> StreamingToneColorConverter:
             )
         return self._tone_color_converter
 
-    def generate(self, tts_request: Type[TTSRequest]):
+    _source_se: Optional[torch.Tensor] = None
+
+    @property
+    def speaker_key(self) -> str:
         """
-        Generate speech using OpenVoice and apply tone color conversion.
+        Get the speaker key for the TTS model.
         """
-        message = tts_request.message
-        speaker_ids = self.model.hps.data.spk2id
-        for speaker_key in speaker_ids.keys():
-            speaker_id = speaker_ids[speaker_key]
-            speaker_key = speaker_key.lower().replace("_", "-")
+        if self.language is AvailableLanguage.EN:
+            return "en-newest"
+        return self.language.value.lower()
+
+    @property
+    def speaker_id(self) -> str:
+        # ['EN-US', 'EN-BR', 'EN_INDIA', 'EN-AU', 'EN-Default']
+        if self.language is AvailableLanguage.EN:
+            return "EN-Default"
+        return self.language.value
 
-            source_se = torch.load(
+    @property
+    def source_se(self) -> torch.Tensor:
+        if self._source_se is None:
+            self._source_se = torch.load(
                 os.path.join(
                     self.path_settings.tts_model_path,
-                    f"openvoice/checkpoints_v2/base_speakers/ses/{speaker_key}.pth",
+                    f"openvoice/checkpoints_v2/base_speakers/ses/{self.speaker_key}.pth",
                 ),
                 map_location=self.device,
             )
+        return self._source_se
 
-            self.model.tts_to_file(
-                message, speaker_id, self.src_path, speed=self._speed
-            )
+    def generate(self, tts_request: Type[TTSRequest]):
+        """
+        Generate speech using OpenVoice and apply tone color conversion.
+        """
+        message = tts_request.message
+        language = AvailableLanguage(detect_language(tts_request.message))
+        if self.language != language:
+            self._source_se = None
+            self.language = language
+            self.model.language = self.language
+        speaker_ids = self.model.hps.data.spk2id
+        print(speaker_ids.keys())
+        # print("SPEAKER KEY", speaker_key)
+        # key = speaker_key.replace("-", "_").split("_")[0].upper()
+        # if key == "En-Default":
+        #     key = "EN_NEWEST"
+        # speaker_key = speaker_key.lower().replace("_", "-")
 
-            output_path = os.path.join(
-                self.path_settings.tts_model_path,
-                f"openvoice/{self._output_dir}/output_v2_{speaker_key}.wav",
-            )
+        print(self.model.language)
 
-            response = self.tone_color_converter.convert(
-                audio_src_path=self.src_path,
-                src_se=source_se,
-                tgt_se=self._target_se,
-                output_path=output_path,
-            )
+        self.model.tts_to_file(
+            message,
+            speaker_ids[self.speaker_id],
+            self.src_path,
+            speed=self._speed,
+        )
+
+        output_path = os.path.join(
+            self.path_settings.tts_model_path,
+            f"openvoice/{self._output_dir}/output_v2_{self.speaker_key}.wav",
+        )
+
+        response = self.tone_color_converter.convert(
+            audio_src_path=self.src_path,
+            src_se=self.source_se,
+            tgt_se=self._target_se,
+            output_path=output_path,
+        )
 
-            if response is not None:
-                self.api.tts.add_to_stream(response)
+        if response is not None:
+            self.api.tts.add_to_stream(response)
 
     def load(self, _target_model=None):
         """
@@ -201,7 +255,7 @@ def load(self, _target_model=None):
         self.unload()
         self.change_model_status(ModelType.TTS, ModelStatus.LOADING)
         self._initialize()
-        self.model = TTS(language=self._language)
+        self.model = TTS(language=self.language)
         self.change_model_status(ModelType.TTS, ModelStatus.LOADED)
 
     def unload(self):

src/airunner/utils/llm/language.py

@@ -0,0 +1,28 @@
+from lingua import Language, LanguageDetectorBuilder
+from airunner.enums import AvailableLanguage
+
+
+def detect_language(txt: str) -> str:
+    languages = [
+        Language.ENGLISH,
+        Language.FRENCH,
+        Language.GERMAN,
+        Language.SPANISH,
+        Language.KOREAN,
+        Language.RUSSIAN,
+        Language.SPANISH,
+        Language.CHINESE,
+        Language.JAPANESE,
+    ]
+    detector = LanguageDetectorBuilder.from_languages(*languages).build()
+    language = detector.detect_language_of(txt)
+    name = language.iso_code_639_1.name
+    if name == "JA":
+        return AvailableLanguage.JP.value
+    if name == "KO":
+        return AvailableLanguage.KR.value
+    try:
+        return AvailableLanguage(name).value
+    except KeyError:
+        print(f"Language {name} not found in AvailableLanguage enum.")
+        return AvailableLanguage.EN.value