Merge pull request #17 from Abandon-ht/dev

Author: Forairaaaaa

docs/cn.md

@@ -0,0 +1,41 @@
+# LLM Module Arduino Quick Start
+
+## Overview
+
+The `LLM Module` can be used with various M5 controllers. This tutorial demonstrates how to control the LLM Module using the `M5Core` series in the `Arduino IDE` with the LLM Module driver library.
+
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_module_device_01.jpg" width="70%" />
+
+## Environment Setup
+
+- 1.Arduino IDE Installation: Refer to the [Arduino IDE Installation Guide](/en/arduino/arduino_ide) to complete the IDE installation.
+
+- 2.Board Manager Installation: Refer to the [Basic Environment Setup Guide](/en/arduino/arduino_board) to complete the M5Stack board manager installation and select the `M5Core` development board.
+
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/arduino/m5core/quickstart_arduino_core_selectboard.png" width="70%" />
+
+- 3.Library Installation: Refer to the [Library Management Guide](/en/arduino/arduino_library) to install the `LLM Module` driver library. (Follow prompts to install the dependency library `M5Unified`)
+
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_lib_01.jpg" width="70%" />
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_lib_02.jpg" width="70%" />
+
+## Program Compilation & Upload
+
+Open the example program "kws_asr" in the driver library, click the upload button, and the program will automatically compile and upload.The wake-up word used in the example program is "HELLO". After waiting for the device to be initialized, it will be woken up using the keyword.
+
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_example_01.jpg" width="70%" />
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_example_02.jpg" width="70%" />
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_example_03.jpg" width="70%" />
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_example_04.jpg" width="70%" />
+
+- Examples:
+  - `kws_asr`: Uses KWS to wake up and triggers ASR for speech-to-text conversion. (KWS+ASR)
+  - `text_assistant`: Inputs text into the LLM model, performs inference, and outputs the result in text form. (LLM)
+  - `tts`: Uses the TTS unit to convert text to speech for playback. (TTS)
+  - `voice_assistant`: Uses KWS to wake up, triggers ASR for speech-to-text conversion, inputs the converted text into the LLM for inference, and outputs the inference result through TTS as speech. (KWS+ASR+LLM+TTS)
+
+## Related Links
+
+- [LLM Module Arduino Lib](https://github.com/m5stack/M5Module-LLM)
+- [LLM Module Arduino Lib API](/en/guide/llm/llm/arduino_api)
+

docs/en.md

@@ -12,13 +12,15 @@ bool M5ModuleLLM::begin(Stream* serialPort)
     msg.init(&comm);
     sys.init(&msg);
     llm.init(&msg);
+    vlm.init(&msg);
     audio.init(&msg);
     tts.init(&msg);
     melotts.init(&msg);
     kws.init(&msg);
     asr.init(&msg);
     yolo.init(&msg);
     camera.init(&msg);
+    depthanything.init(&msg);
     return true;
 }
 

src/M5ModuleLLM.cpp

@@ -9,12 +9,14 @@
 #include "utils/msg.h"
 #include "api/api_sys.h"
 #include "api/api_llm.h"
+#include "api/api_vlm.h"
 #include "api/api_audio.h"
 #include "api/api_tts.h"
 #include "api/api_melotts.h"
 #include "api/api_kws.h"
 #include "api/api_asr.h"
 #include "api/api_yolo.h"
+#include "api/api_depth_anything.h"
 #include "api/api_camera.h"
 #include "api/api_version.h"
 
@@ -55,6 +57,12 @@ class M5ModuleLLM {
      */
     m5_module_llm::ApiLlm llm;
 
+    /**
+     * @brief VLM module api set
+     *
+     */
+    m5_module_llm::ApiVlm vlm;
+
     /**
      * @brief Audio module api set
      *
@@ -97,6 +105,12 @@ class M5ModuleLLM {
      */
     m5_module_llm::ApiYolo yolo;
 
+    /**
+     * @brief DepthAnything module api set
+     *
+     */
+    m5_module_llm::ApiDepthAnything depthanything;
+
     /**
      * @brief MSG module to handle module response message
      *

src/M5ModuleLLM.h

@@ -51,3 +51,24 @@ String ApiAsr::setup(ApiAsrSetupConfig_t config, String request_id, String langu
         10000);
     return work_id;
 }
+
+String ApiAsr::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}

src/api/api_asr.cpp

@@ -34,6 +34,15 @@ class ApiAsr {
     String setup(ApiAsrSetupConfig_t config = ApiAsrSetupConfig_t(), String request_id = "asr_setup",
                  String language = "en_US");
 
+    /**
+     * @brief Exit module ASR, return ASR work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "asr_exit");
+
 private:
     ModuleMsg* _module_msg = nullptr;
 };

src/api/api_asr.h

@@ -40,3 +40,24 @@ String ApiAudio::setup(ApiAudioSetupConfig_t config, String request_id)
         5000);
     return work_id;
 }
+
+String ApiAudio::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}

src/api/api_audio.cpp

@@ -23,14 +23,23 @@ class ApiAudio {
     void init(ModuleMsg* moduleMsg);
 
     /**
-     * @brief Setup module audio, return work_id
+     * @brief Setup module audio, return audio work_id
      *
      * @param config
      * @param request_id
      * @return String
      */
     String setup(ApiAudioSetupConfig_t config = ApiAudioSetupConfig_t(), String request_id = "audio_setup");
 
+    /**
+     * @brief Exit module audio, return audio work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "audio_exit");
+
 private:
     ModuleMsg* _module_msg = nullptr;
 };

src/api/api_audio.h

@@ -39,3 +39,24 @@ String ApiCamera::setup(ApiCameraSetupConfig_t config, String request_id)
         5000);
     return work_id;
 }
+
+String ApiCamera::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}

src/api/api_camera.cpp

@@ -22,14 +22,23 @@ class ApiCamera {
     void init(ModuleMsg* moduleMsg);
 
     /**
-     * @brief Setup module camera, return work_id
+     * @brief Setup module camera, return camera work_id
      *
      * @param config
      * @param request_id
      * @return String
      */
     String setup(ApiCameraSetupConfig_t config = ApiCameraSetupConfig_t(), String request_id = "camera_setup");
 
+    /**
+     * @brief Exit module camera, return camera work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "camera_exit");
+
 private:
     ModuleMsg* _module_msg = nullptr;
 };