Skip to content

Commit 400b2a9

Browse files
Abandon-htAbandon-ht
authored
Merge pull request #26 from m5stack/dev
Dev
2 parents 5025891 + 8cf241d commit 400b2a9

File tree

23 files changed

+691
-79
lines changed

23 files changed

+691
-79
lines changed

‎examples/KWS_VAD_Whisper_LLM/KWS_VAD_Whisper_LLM.ino‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
3+
*
4+
* SPDX-License-Identifier: MIT
5+
*/
6+
#include <Arduino.h>
7+
#include <M5Unified.h>
8+
#include <M5ModuleLLM.h>
9+
10+
M5ModuleLLM module_llm;
11+
12+
/* Must be capitalized */
13+
String wake_up_keyword = "HELLO";
14+
// String wake_up_keyword = "你好你好";
15+
String kws_work_id;
16+
String vad_work_id;
17+
String whisper_work_id;
18+
String llm_work_id;
19+
String language;
20+
21+
void setup()
22+
{
23+
M5.begin();
24+
M5.Display.setTextSize(2);
25+
M5.Display.setTextScroll(true);
26+
M5.Display.setFont(&fonts::efontCN_12); // Support Chinese display
27+
// M5.Display.setFont(&fonts::efontJA_12); // Support Japanese display
28+
29+
language = "en_US";
30+
// language = "zh_CN";
31+
32+
/* Init module serial port */
33+
// int rxd = 16, txd = 17; // Basic
34+
// int rxd = 13, txd = 14; // Core2
35+
// int rxd = 18, txd = 17; // CoreS3
36+
int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
37+
int txd = M5.getPin(m5::pin_name_t::port_c_txd);
38+
Serial2.begin(115200, SERIAL_8N1, rxd, txd);
39+
40+
/* Init module */
41+
module_llm.begin(&Serial2);
42+
43+
/* Make sure module is connected */
44+
M5.Display.printf(">> Check ModuleLLM connection..\n");
45+
while (1) {
46+
if (module_llm.checkConnection()) {
47+
break;
48+
}
49+
}
50+
51+
/* Reset ModuleLLM */
52+
M5.Display.printf(">> Reset ModuleLLM..\n");
53+
module_llm.sys.reset();
54+
55+
/* Setup Audio module */
56+
M5.Display.printf(">> Setup audio..\n");
57+
module_llm.audio.setup();
58+
59+
/* Setup KWS module and save returned work id */
60+
M5.Display.printf(">> Setup kws..\n");
61+
m5_module_llm::ApiKwsSetupConfig_t kws_config;
62+
kws_config.kws = wake_up_keyword;
63+
kws_work_id = module_llm.kws.setup(kws_config, "kws_setup", language);
64+
65+
/* Setup VAD module and save returned work id */
66+
M5.Display.printf(">> Setup vad..\n");
67+
m5_module_llm::ApiVadSetupConfig_t vad_config;
68+
vad_config.input = {"sys.pcm", kws_work_id};
69+
vad_work_id = module_llm.vad.setup(vad_config, "vad_setup");
70+
71+
/* Setup Whisper module and save returned work id */
72+
M5.Display.printf(">> Setup whisper..\n");
73+
m5_module_llm::ApiWhisperSetupConfig_t whisper_config;
74+
whisper_config.input = {"sys.pcm", kws_work_id, vad_work_id};
75+
whisper_config.language = "en";
76+
// whisper_config.language = "zh";
77+
// whisper_config.language = "ja";
78+
whisper_work_id = module_llm.whisper.setup(whisper_config, "whisper_setup");
79+
80+
M5.Display.printf(">> Setup llm..\n");
81+
llm_work_id = module_llm.llm.setup();
82+
83+
M5.Display.printf(">> Setup ok\n>> Say \"%s\" to wakeup\n", wake_up_keyword.c_str());
84+
}
85+
86+
void loop()
87+
{
88+
/* Update ModuleLLM */
89+
module_llm.update();
90+
91+
/* Handle module response messages */
92+
for (auto& msg : module_llm.msg.responseMsgList) {
93+
/* If KWS module message */
94+
if (msg.work_id == kws_work_id) {
95+
M5.Display.setTextColor(TFT_GREENYELLOW);
96+
M5.Display.printf(">> Keyword detected\n");
97+
}
98+
99+
/* If ASR module message */
100+
if (msg.work_id == whisper_work_id) {
101+
/* Check message object type */
102+
if (msg.object == "asr.utf-8") {
103+
/* Parse message json and get ASR result */
104+
JsonDocument doc;
105+
deserializeJson(doc, msg.raw_msg);
106+
String asr_result = doc["data"].as<String>();
107+
108+
M5.Display.setTextColor(TFT_YELLOW);
109+
M5.Display.printf(">> %s\n", asr_result.c_str());
110+
module_llm.llm.inferenceAndWaitResult(llm_work_id, asr_result.c_str(), [](String& result) {
111+
/* Show result on screen */
112+
M5.Display.printf("%s", result.c_str());
113+
});
114+
}
115+
}
116+
}
117+
118+
/* Clear handled messages */
119+
module_llm.msg.responseMsgList.clear();
120+
}

‎examples/KWS_VAD_Whisper_LLM_TTS/KWS_VAD_Whisper_LLM_TTS.ino‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*
2+
* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
3+
*
4+
* SPDX-License-Identifier: MIT
5+
*/
6+
#include <Arduino.h>
7+
#include <M5Unified.h>
8+
#include <M5ModuleLLM.h>
9+
10+
M5ModuleLLM module_llm;
11+
12+
/* Must be capitalized */
13+
String wake_up_keyword = "HELLO";
14+
// String wake_up_keyword = "你好你好";
15+
String kws_work_id;
16+
String vad_work_id;
17+
String whisper_work_id;
18+
String llm_work_id;
19+
String melotts_work_id;
20+
String language;
21+
22+
void setup()
23+
{
24+
M5.begin();
25+
M5.Display.setTextSize(2);
26+
M5.Display.setTextScroll(true);
27+
// M5.Display.setFont(&fonts::efontCN_12); // Support Chinese display
28+
// M5.Display.setFont(&fonts::efontJA_12); // Support Japanese display
29+
30+
language = "en_US";
31+
// language = "zh_CN";
32+
33+
/* Init module serial port */
34+
// int rxd = 16, txd = 17; // Basic
35+
// int rxd = 13, txd = 14; // Core2
36+
// int rxd = 18, txd = 17; // CoreS3
37+
int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
38+
int txd = M5.getPin(m5::pin_name_t::port_c_txd);
39+
Serial2.begin(115200, SERIAL_8N1, rxd, txd);
40+
41+
/* Init module */
42+
module_llm.begin(&Serial2);
43+
44+
/* Make sure module is connected */
45+
M5.Display.printf(">> Check ModuleLLM connection..\n");
46+
while (1) {
47+
if (module_llm.checkConnection()) {
48+
break;
49+
}
50+
}
51+
52+
/* Reset ModuleLLM */
53+
M5.Display.printf(">> Reset ModuleLLM..\n");
54+
module_llm.sys.reset();
55+
56+
/* Setup Audio module */
57+
M5.Display.printf(">> Setup audio..\n");
58+
module_llm.audio.setup();
59+
60+
/* Setup KWS module and save returned work id */
61+
M5.Display.printf(">> Setup kws..\n");
62+
m5_module_llm::ApiKwsSetupConfig_t kws_config;
63+
kws_config.kws = wake_up_keyword;
64+
kws_work_id = module_llm.kws.setup(kws_config, "kws_setup", language);
65+
66+
/* Setup VAD module and save returned work id */
67+
M5.Display.printf(">> Setup vad..\n");
68+
m5_module_llm::ApiVadSetupConfig_t vad_config;
69+
vad_config.input = {"sys.pcm", kws_work_id};
70+
vad_work_id = module_llm.vad.setup(vad_config, "vad_setup");
71+
72+
/* Setup Whisper module and save returned work id */
73+
M5.Display.printf(">> Setup whisper..\n");
74+
m5_module_llm::ApiWhisperSetupConfig_t whisper_config;
75+
whisper_config.input = {"sys.pcm", kws_work_id, vad_work_id};
76+
whisper_config.language = "en";
77+
// whisper_config.language = "zh";
78+
// whisper_config.language = "ja";
79+
whisper_work_id = module_llm.whisper.setup(whisper_config, "whisper_setup");
80+
81+
M5.Display.printf(">> Setup llm..\n");
82+
llm_work_id = module_llm.llm.setup();
83+
84+
M5.Display.printf(">> Setup melotts..\n\n");
85+
m5_module_llm::ApiMelottsSetupConfig_t melotts_config;
86+
melotts_config.input = {"tts.utf-8.stream", llm_work_id};
87+
melotts_work_id = module_llm.melotts.setup(melotts_config, "melotts_setup", language);
88+
89+
M5.Display.printf(">> Setup ok\n>> Say \"%s\" to wakeup\n", wake_up_keyword.c_str());
90+
}
91+
92+
void loop()
93+
{
94+
/* Update ModuleLLM */
95+
module_llm.update();
96+
97+
/* Handle module response messages */
98+
for (auto& msg : module_llm.msg.responseMsgList) {
99+
/* If KWS module message */
100+
if (msg.work_id == kws_work_id) {
101+
M5.Display.setTextColor(TFT_GREENYELLOW);
102+
M5.Display.printf(">> Keyword detected\n");
103+
}
104+
105+
if (msg.work_id == vad_work_id) {
106+
M5.Display.setTextColor(TFT_GREENYELLOW);
107+
M5.Display.printf(">> vad detected\n");
108+
}
109+
/* If ASR module message */
110+
if (msg.work_id == whisper_work_id) {
111+
/* Check message object type */
112+
if (msg.object == "asr.utf-8") {
113+
/* Parse message json and get ASR result */
114+
JsonDocument doc;
115+
deserializeJson(doc, msg.raw_msg);
116+
String asr_result = doc["data"].as<String>();
117+
118+
M5.Display.setTextColor(TFT_YELLOW);
119+
M5.Display.printf(">> %s\n", asr_result.c_str());
120+
121+
module_llm.llm.inferenceAndWaitResult(llm_work_id, asr_result.c_str(), [](String& result) {
122+
/* Show result on screen */
123+
handleLLMResult(result);
124+
});
125+
}
126+
}
127+
}
128+
129+
/* Clear handled messages */
130+
module_llm.msg.responseMsgList.clear();
131+
}
132+
133+
void handleLLMResult(String& result)
134+
{
135+
M5.Display.printf("%s", result.c_str());
136+
}

‎examples/MeloTTS/MeloTTS.ino‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
3+
*
4+
* SPDX-License-Identifier: MIT
5+
*/
6+
#include <Arduino.h>
7+
#include <M5Unified.h>
8+
#include <M5ModuleLLM.h>
9+
10+
M5ModuleLLM module_llm;
11+
String melotts_work_id;
12+
String language;
13+
14+
void setup()
15+
{
16+
M5.begin();
17+
M5.Display.setTextSize(2);
18+
M5.Display.setTextScroll(true);
19+
// M5.Display.setFont(&fonts::efontCN_12); // Support Chinese display
20+
// M5.Display.setFont(&fonts::efontJA_12); // Support Japanese display
21+
22+
language = "en_US";
23+
// language = "zh_CN";
24+
// language = "ja_JP";
25+
26+
/* Init module serial port */
27+
// int rxd = 16, txd = 17; // Basic
28+
// int rxd = 13, txd = 14; // Core2
29+
// int rxd = 18, txd = 17; // CoreS3
30+
int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
31+
int txd = M5.getPin(m5::pin_name_t::port_c_txd);
32+
Serial2.begin(115200, SERIAL_8N1, rxd, txd);
33+
34+
/* Init module */
35+
module_llm.begin(&Serial2);
36+
37+
/* Make sure module is connected */
38+
M5.Display.printf(">> Check ModuleLLM connection..\n");
39+
while (1) {
40+
if (module_llm.checkConnection()) {
41+
break;
42+
}
43+
}
44+
45+
/* Reset ModuleLLM */
46+
M5.Display.printf(">> Reset ModuleLLM..\n");
47+
module_llm.sys.reset();
48+
49+
/* Setup Audio module */
50+
M5.Display.printf(">> Setup audio..\n");
51+
module_llm.audio.setup();
52+
53+
/* Setup MeloTTS module and save returned work id */
54+
M5.Display.printf(">> Setup melotts..\n\n");
55+
m5_module_llm::ApiMelottsSetupConfig_t melotts_config;
56+
melotts_work_id = module_llm.melotts.setup(melotts_config, "melotts_setup", language);
57+
}
58+
59+
void loop()
60+
{
61+
/* Make a text for speech: {i} plus {i} equals to {i + i} */
62+
static int i = 0;
63+
i++;
64+
std::string text = std::to_string(i) + " plus " + std::to_string(i) + " equals " + std::to_string(i + i) + ".";
65+
// std::string text = std::to_string(i) + " 加 " + std::to_string(i) + " 等于 " + std::to_string(i + i) + ".";
66+
67+
M5.Display.setTextColor(TFT_GREEN);
68+
M5.Display.printf("<< %s\n\n", text.c_str());
69+
70+
/* Push text to TTS module and wait inference result */
71+
module_llm.tts.inference(melotts_work_id, text.c_str(), 10000);
72+
73+
delay(500);
74+
}

0 commit comments

Comments
 (0)