1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #include "wakeup_engine_impl.h"
16
17 #include "audio_asr.h"
18 #include "audio_system_manager.h"
19 #include "ipc_skeleton.h"
20 #include "v1_2/intell_voice_engine_types.h"
21 #include "adapter_callback_service.h"
22 #include "intell_voice_info.h"
23 #include "intell_voice_log.h"
24 #include "history_info_mgr.h"
25 #include "intell_voice_util.h"
26 #include "string_util.h"
27 #include "intell_voice_engine_manager.h"
28 #include "engine_callback_message.h"
29 #include "engine_host_manager.h"
30 #include "swing_service_wrapper.h"
31
32 #define LOG_TAG "WakeupEngineImpl"
33
34 using namespace OHOS::IntellVoice;
35 using namespace OHOS::AudioStandard;
36 using namespace OHOS::HDI::IntelligentVoice::Engine::V1_0;
37 using namespace OHOS::IntellVoiceUtils;
38
39 namespace OHOS {
40 namespace IntellVoiceEngine {
41 static constexpr uint32_t MIN_BUFFER_SIZE = 640;
42 static constexpr uint32_t INTERVAL = 100;
43 static constexpr int32_t CHANNEL_CNT = 1;
44 static constexpr int32_t BITS_PER_SAMPLE = 16;
45 static constexpr int32_t SAMPLE_RATE = 16000;
46 static const std::string WAKEUP_SOURCE_CHANNEL = "wakeup_source_channel";
47 static const std::string IS_CALLBACK_EXIST = "is_callback_exist";
48 static constexpr std::string_view DEFAULT_WAKEUP_PHRASE = "\xE5\xB0\x8F\xE8\x89\xBA\xE5\xB0\x8F\xE8\x89\xBA";
49 static constexpr int64_t RECOGNIZING_TIMEOUT_US = 10 * 1000 * 1000; //10s
50 static constexpr int64_t RECOGNIZE_COMPLETE_TIMEOUT_US = 2 * 1000 * 1000; //2s
51 static constexpr int64_t READ_CAPTURER_TIMEOUT_US = 10 * 1000 * 1000; //10s
52 static constexpr uint32_t MAX_WAKEUP_TASK_NUM = 200;
53 static const std::string WAKEUP_THREAD_NAME = "WakeupEngThread";
54 static const std::string WAKEUP_CONFORMER_EVENT_TYPE = "AUDIO_WAKEUP_1_5";
55 static const std::string WAKEUP_CONFORMER_DOMAIN_ID = "7";
56
WakeupEngineImpl()57 WakeupEngineImpl::WakeupEngineImpl() : ModuleStates(State(IDLE), "WakeupEngineImpl", "WakeupThread"),
58 TaskExecutor(WAKEUP_THREAD_NAME, MAX_WAKEUP_TASK_NUM)
59 {
60 InitStates();
61 capturerOptions_.streamInfo.samplingRate = AudioSamplingRate::SAMPLE_RATE_16000;
62 capturerOptions_.streamInfo.encoding = AudioEncodingType::ENCODING_PCM;
63 capturerOptions_.streamInfo.format = AudioSampleFormat::SAMPLE_S16LE;
64 capturerOptions_.streamInfo.channels = AudioChannel::MONO;
65 capturerOptions_.capturerInfo.sourceType = SourceType::SOURCE_TYPE_WAKEUP;
66 capturerOptions_.capturerInfo.capturerFlags = 0;
67 adapterListener_ = std::make_shared<WakeupAdapterListener>(
68 std::bind(&WakeupEngineImpl::OnWakeupEvent, this, std::placeholders::_1));
69 if (adapterListener_ == nullptr) {
70 INTELL_VOICE_LOG_ERROR("adapterListener_ is nullptr");
71 }
72 TaskExecutor::StartThread();
73 }
74
~WakeupEngineImpl()75 WakeupEngineImpl::~WakeupEngineImpl()
76 {
77 }
78
InitStates()79 bool WakeupEngineImpl::InitStates()
80 {
81 for (int i = IDLE; i <= READ_CAPTURER; i++) {
82 ForState(State(i))
83 .ACT(SET_LISTENER, HandleSetListener)
84 .ACT(SET_PARAM, HandleSetParam)
85 .ACT(GET_PARAM, HandleGetParam);
86 }
87
88 ForState(IDLE)
89 .ACT(INIT, HandleInit)
90 .ACT(RESET_ADAPTER, HandleResetAdapter);
91
92 ForState(INITIALIZING)
93 .ACT(INIT_DONE, HandleInitDone);
94
95 ForState(INITIALIZED)
96 .ACT(START_RECOGNIZE, HandleStart);
97
98 ForState(RECOGNIZING)
99 .WaitUntil(RECOGNIZING_TIMEOUT,
100 std::bind(&WakeupEngineImpl::HandleRecognizingTimeout, this, std::placeholders::_1, std::placeholders::_2),
101 RECOGNIZING_TIMEOUT_US)
102 .ACT(STOP_RECOGNIZE, HandleStop)
103 .ACT(RECOGNIZE_COMPLETE, HandleRecognizeComplete);
104
105 ForState(RECOGNIZED)
106 .WaitUntil(RECOGNIZE_COMPLETE_TIMEOUT,
107 std::bind(&WakeupEngineImpl::HandleStopCapturer, this, std::placeholders::_1, std::placeholders::_2),
108 RECOGNIZE_COMPLETE_TIMEOUT_US)
109 .ACT(GET_WAKEUP_PCM, HandleGetWakeupPcm)
110 .ACT(START_CAPTURER, HandleStartCapturer);
111
112 ForState(READ_CAPTURER)
113 .WaitUntil(READ_CAPTURER_TIMEOUT,
114 std::bind(&WakeupEngineImpl::HandleStopCapturer, this, std::placeholders::_1, std::placeholders::_2),
115 READ_CAPTURER_TIMEOUT_US)
116 .ACT(READ, HandleRead)
117 .ACT(STOP_CAPTURER, HandleStopCapturer);
118
119 FromState(INITIALIZING, READ_CAPTURER)
120 .ACT(RELEASE_ADAPTER, HandleRelease)
121 .ACT(RELEASE, HandleRelease);
122
123 return IsStatesInitSucc();
124 }
125
Handle(const StateMsg & msg)126 int32_t WakeupEngineImpl::Handle(const StateMsg &msg)
127 {
128 if (!IsStatesInitSucc()) {
129 INTELL_VOICE_LOG_ERROR("failed to init state");
130 return -1;
131 }
132
133 return ModuleStates::HandleMsg(msg);
134 }
135
GetWakeupSourceChannel()136 OHOS::AudioStandard::AudioChannel WakeupEngineImpl::GetWakeupSourceChannel()
137 {
138 auto ret = EngineCallbackMessage::CallFunc(TRIGGERMGR_GET_PARAMETER, WAKEUP_SOURCE_CHANNEL);
139 std::string channel = "";
140 if (ret.has_value()) {
141 try {
142 channel = std::any_cast<std::string>(*ret);
143 } catch (const std::bad_any_cast&) {
144 INTELL_VOICE_LOG_ERROR("msg bus bad any cast");
145 return AudioChannel::MONO;
146 }
147 } else {
148 INTELL_VOICE_LOG_ERROR("msg bus return no value");
149 return AudioChannel::MONO;
150 }
151
152 if (channel == "") {
153 INTELL_VOICE_LOG_INFO("no channle info");
154 return AudioChannel::MONO;
155 }
156
157 int32_t channelCnt = 0;
158 if (!StringUtil::StringToInt(channel, channelCnt)) {
159 INTELL_VOICE_LOG_ERROR("failed to get channel cnt");
160 return AudioChannel::MONO;
161 }
162
163 if ((channelCnt < AudioChannel::MONO) || (channelCnt > AudioChannel::CHANNEL_4)) {
164 INTELL_VOICE_LOG_INFO("invalid channel cnt:%{public}d", channelCnt);
165 return AudioChannel::MONO;
166 }
167
168 INTELL_VOICE_LOG_INFO("channel cnt:%{public}d", static_cast<int32_t>(channelCnt));
169 return static_cast<OHOS::AudioStandard::AudioChannel>(channelCnt);
170 }
171
SetCallbackInner()172 bool WakeupEngineImpl::SetCallbackInner()
173 {
174 INTELL_VOICE_LOG_INFO("enter");
175 if (adapter_ == nullptr) {
176 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
177 return false;
178 }
179
180 if (adapterListener_ == nullptr) {
181 INTELL_VOICE_LOG_ERROR("adapter listener is nullptr");
182 return false;
183 }
184
185 callback_ = sptr<IIntellVoiceEngineCallback>(new (std::nothrow) AdapterCallbackService(adapterListener_));
186 if (callback_ == nullptr) {
187 INTELL_VOICE_LOG_ERROR("callback_ is nullptr");
188 return false;
189 }
190
191 adapter_->SetCallback(callback_);
192 return true;
193 }
194
AttachInner(const IntellVoiceEngineInfo & info)195 int32_t WakeupEngineImpl::AttachInner(const IntellVoiceEngineInfo &info)
196 {
197 INTELL_VOICE_LOG_INFO("enter");
198 if (adapter_ == nullptr) {
199 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
200 return -1;
201 }
202
203 isPcmFromExternal_ = info.isPcmFromExternal;
204
205 IntellVoiceEngineAdapterInfo adapterInfo = {
206 .wakeupPhrase = info.wakeupPhrase,
207 .minBufSize = info.minBufSize,
208 .sampleChannels = info.sampleChannels,
209 .bitsPerSample = info.bitsPerSample,
210 .sampleRate = info.sampleRate,
211 };
212 return adapter_->Attach(adapterInfo);
213 }
214
SetParamOnAudioStart(int32_t uuid)215 void WakeupEngineImpl::SetParamOnAudioStart(int32_t uuid)
216 {
217 INTELL_VOICE_LOG_INFO("enter");
218 auto audioSystemManager = AudioSystemManager::GetInstance();
219 if (audioSystemManager == nullptr) {
220 INTELL_VOICE_LOG_ERROR("audioSystemManager is nullptr");
221 return;
222 }
223
224 audioSystemManager->SetAsrAecMode(AsrAecMode::STANDARD);
225 if (uuid == PROXIMAL_WAKEUP_MODEL_UUID) {
226 audioSystemManager->SetAsrNoiseSuppressionMode(AsrNoiseSuppressionMode::NEAR_FIELD);
227 } else {
228 audioSystemManager->SetAsrNoiseSuppressionMode(AsrNoiseSuppressionMode::STANDARD);
229 }
230 }
231
SetParamOnAudioStop()232 void WakeupEngineImpl::SetParamOnAudioStop()
233 {
234 INTELL_VOICE_LOG_INFO("enter");
235 auto audioSystemManager = AudioSystemManager::GetInstance();
236 if (audioSystemManager == nullptr) {
237 INTELL_VOICE_LOG_ERROR("audioSystemManager is nullptr");
238 return;
239 }
240 audioSystemManager->SetAsrAecMode(AsrAecMode::BYPASS);
241 audioSystemManager->SetAsrNoiseSuppressionMode(AsrNoiseSuppressionMode::BYPASS);
242 }
243
CreateWakeupSourceStopCallback()244 bool WakeupEngineImpl::CreateWakeupSourceStopCallback()
245 {
246 if (wakeupSourceStopCallback_ != nullptr) {
247 INTELL_VOICE_LOG_INFO("wakeup close cb is already created");
248 return true;
249 }
250
251 auto audioSystemManager = AudioSystemManager::GetInstance();
252 if (audioSystemManager == nullptr) {
253 INTELL_VOICE_LOG_ERROR("audioSystemManager is nullptr");
254 return false;
255 }
256
257 wakeupSourceStopCallback_ = std::make_shared<WakeupSourceStopCallback>();
258 if (wakeupSourceStopCallback_ == nullptr) {
259 INTELL_VOICE_LOG_ERROR("wakeup source stop callback is nullptr");
260 return false;
261 }
262
263 audioSystemManager->SetWakeUpSourceCloseCallback(wakeupSourceStopCallback_);
264 return true;
265 }
266
DestroyWakeupSourceStopCallback()267 void WakeupEngineImpl::DestroyWakeupSourceStopCallback()
268 {
269 if (wakeupSourceStopCallback_ == nullptr) {
270 INTELL_VOICE_LOG_INFO("wakeup close cb is already destroyed");
271 return;
272 }
273
274 auto audioSystemManager = AudioSystemManager::GetInstance();
275 if (audioSystemManager == nullptr) {
276 INTELL_VOICE_LOG_ERROR("audioSystemManager is nullptr");
277 return;
278 }
279
280 audioSystemManager->SetWakeUpSourceCloseCallback(nullptr);
281 wakeupSourceStopCallback_ = nullptr;
282 }
283
StartAudioSource()284 bool WakeupEngineImpl::StartAudioSource()
285 {
286 capturerOptions_.streamInfo.channels = GetWakeupSourceChannel();
287 auto listener = std::make_unique<AudioSourceListener>(
288 [&](uint8_t *buffer, uint32_t size, bool isEnd) {
289 ReadBufferCallback(buffer, size, isEnd);
290 },
291 [&]() {
292 INTELL_VOICE_LOG_INFO("end of pcm");
293 if (adapter_ != nullptr) {
294 adapter_->SetParameter("end_of_pcm=true");
295 }
296 });
297 if (listener == nullptr) {
298 INTELL_VOICE_LOG_ERROR("create listener failed");
299 return false;
300 }
301
302 WakeupSourceProcess::Init(capturerOptions_.streamInfo.channels);
303
304 audioSource_ = std::make_unique<AudioSource>(MIN_BUFFER_SIZE * static_cast<uint32_t>(
305 capturerOptions_.streamInfo.channels), INTERVAL, std::move(listener), capturerOptions_);
306 if (audioSource_ == nullptr) {
307 INTELL_VOICE_LOG_ERROR("create audio source failed");
308 WakeupSourceProcess::Release();
309 return false;
310 }
311
312 if (!audioSource_->Start()) {
313 INTELL_VOICE_LOG_ERROR("start capturer failed");
314 audioSource_ = nullptr;
315 WakeupSourceProcess::Release();
316 return false;
317 }
318
319 return true;
320 }
321
StopAudioSource()322 void WakeupEngineImpl::StopAudioSource()
323 {
324 INTELL_VOICE_LOG_INFO("enter");
325 if (audioSource_ == nullptr) {
326 INTELL_VOICE_LOG_INFO("audio source is nullptr, no need to stop");
327 return;
328 }
329 SetParamOnAudioStop();
330 audioSource_->Stop();
331 audioSource_ = nullptr;
332 WakeupSourceProcess::Release();
333 }
334
OnWakeupEvent(const OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent & event)335 void WakeupEngineImpl::OnWakeupEvent(
336 const OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent &event)
337 {
338 if (event.msgId == INTELL_VOICE_ENGINE_MSG_INIT_DONE) {
339 TaskExecutor::AddAsyncTask([this, result = event.result]() { OnInitDone(result); },
340 "WakeupEngineImpl::InitDone", false);
341 } else if (event.msgId == INTELL_VOICE_ENGINE_MSG_RECOGNIZE_COMPLETE) {
342 TaskExecutor::AddAsyncTask([this, result = event.result, info = event.info]() {
343 OnWakeupRecognition(result, info);
344 }, "WakeupEngineImpl::RecognizeComplete", false);
345 } else {
346 INTELL_VOICE_LOG_WARN("invalid msg id:%{public}d", event.msgId);
347 }
348 }
349
OnInitDone(int32_t result)350 void WakeupEngineImpl::OnInitDone(int32_t result)
351 {
352 INTELL_VOICE_LOG_INFO("on Init Done, result:%{public}d", result);
353 StateMsg msg(INIT_DONE, &result, sizeof(int32_t));
354 Handle(msg);
355 }
356
OnWakeupRecognition(int32_t result,const std::string & info)357 void WakeupEngineImpl::OnWakeupRecognition(int32_t result, const std::string &info)
358 {
359 INTELL_VOICE_LOG_INFO("on wakeup recognition, result:%{public}d", result);
360 OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent event;
361 event.msgId = INTELL_VOICE_ENGINE_MSG_RECOGNIZE_COMPLETE;
362 event.result = static_cast<OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineErrors>(result);
363 event.info = info;
364 StateMsg msg(RECOGNIZE_COMPLETE, reinterpret_cast<void *>(&event),
365 sizeof(OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent));
366 Handle(msg);
367 }
368
HandleSetParam(const StateMsg & msg,State &)369 int32_t WakeupEngineImpl::HandleSetParam(const StateMsg &msg, State & /* nextState */)
370 {
371 StringParam *param = reinterpret_cast<StringParam *>(msg.inMsg);
372 if (param == nullptr) {
373 INTELL_VOICE_LOG_INFO("param is nullptr");
374 return -1;
375 }
376
377 return EngineUtil::SetParameter(param->strParam);
378 }
379
HandleGetParam(const StateMsg & msg,State &)380 int32_t WakeupEngineImpl::HandleGetParam(const StateMsg &msg, State & /* nextState */)
381 {
382 StringParam *key = reinterpret_cast<StringParam *>(msg.inMsg);
383 StringParam *value = reinterpret_cast<StringParam *>(msg.outMsg);
384 if ((key == nullptr) || (value == nullptr)) {
385 INTELL_VOICE_LOG_INFO("key or value is nullptr");
386 return -1;
387 }
388
389 if (key->strParam == WAKEUP_SOURCE_CHANNEL) {
390 value->strParam = std::to_string(static_cast<uint32_t>(capturerOptions_.streamInfo.channels));
391 } else if (key->strParam == IS_CALLBACK_EXIST) {
392 if (adapterListener_ == nullptr) {
393 INTELL_VOICE_LOG_ERROR("adapterListener_ is nullptr");
394 value->strParam = "false";
395 } else {
396 value->strParam = adapterListener_->GetCallbackStatus();
397 }
398 } else {
399 value->strParam = EngineUtil::GetParameter(key->strParam);
400 }
401 INTELL_VOICE_LOG_INFO("key:%{public}s, value:%{public}s", key->strParam.c_str(), value->strParam.c_str());
402 return 0;
403 }
404
HandleInit(const StateMsg &,State & nextState)405 int32_t WakeupEngineImpl::HandleInit(const StateMsg & /* msg */, State &nextState)
406 {
407 INTELL_VOICE_LOG_INFO("enter");
408 if (!EngineUtil::CreateAdapterInner(EngineHostManager::GetInstance(), WAKEUP_ADAPTER_TYPE)) {
409 INTELL_VOICE_LOG_ERROR("failed to create adapter");
410 return -1;
411 }
412
413 if (!SetCallbackInner()) {
414 INTELL_VOICE_LOG_ERROR("failed to set callback");
415 return -1;
416 }
417 SetWakeupModel();
418 EngineUtil::SetLanguage();
419 EngineUtil::SetArea();
420 EngineUtil::SetSensibility();
421 SetDspFeatures();
422
423 IntellVoiceEngineInfo info = {
424 .wakeupPhrase = GetWakeupPhrase(),
425 .isPcmFromExternal = false,
426 .minBufSize = MIN_BUFFER_SIZE,
427 .sampleChannels = CHANNEL_CNT,
428 .bitsPerSample = BITS_PER_SAMPLE,
429 .sampleRate = SAMPLE_RATE,
430 };
431 if (AttachInner(info) != 0) {
432 INTELL_VOICE_LOG_ERROR("failed to attach");
433 return -1;
434 }
435
436 UpdateDspModel();
437
438 nextState = State(INITIALIZING);
439 return 0;
440 }
441
HandleInitDone(const StateMsg & msg,State & nextState)442 int32_t WakeupEngineImpl::HandleInitDone(const StateMsg &msg, State &nextState)
443 {
444 INTELL_VOICE_LOG_INFO("enter");
445 int32_t *result = reinterpret_cast<int32_t *>(msg.inMsg);
446 if ((result == nullptr) || (*result != 0)) {
447 INTELL_VOICE_LOG_ERROR("init done failed");
448 return -1;
449 }
450
451 nextState = State(INITIALIZED);
452 return 0;
453 }
454
HandleSetListener(const StateMsg & msg,State &)455 int32_t WakeupEngineImpl::HandleSetListener(const StateMsg &msg, State & /* nextState */)
456 {
457 SetListenerMsg *listenerMsg = reinterpret_cast<SetListenerMsg *>(msg.inMsg);
458 if (listenerMsg == nullptr || adapterListener_ == nullptr) {
459 INTELL_VOICE_LOG_ERROR("listenerMsg or adapter listener is nullptr");
460 return -1;
461 }
462
463 adapterListener_->SetCallback(listenerMsg->callback);
464 return 0;
465 }
466
HandleStart(const StateMsg & msg,State & nextState)467 int32_t WakeupEngineImpl::HandleStart(const StateMsg &msg, State &nextState)
468 {
469 int32_t *msgBody = reinterpret_cast<int32_t *>(msg.inMsg);
470 if (msgBody == nullptr) {
471 INTELL_VOICE_LOG_ERROR("msgBody is nullptr");
472 return -1;
473 }
474
475 if (*msgBody == PROXIMAL_WAKEUP_MODEL_UUID) {
476 channelId_ = CHANNEL_ID_1;
477 EngineUtil::SetParameter("WakeupType=3");
478 } else {
479 channelId_ = CHANNEL_ID_0;
480 EngineUtil::SetParameter("WakeupType=0");
481 }
482 INTELL_VOICE_LOG_INFO("enter, channel id is %{public}d", channelId_);
483
484 EngineUtil::SetParameter("VprTrdType=0;WakeupScene=0");
485 EngineUtil::SetScreenStatus();
486
487 if (adapter_ == nullptr) {
488 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
489 return -1;
490 }
491
492 StartInfo info = {
493 .isLast = true,
494 };
495 if (adapter_->Start(info)) {
496 INTELL_VOICE_LOG_ERROR("start adapter failed");
497 return -1;
498 }
499
500 if (!CreateWakeupSourceStopCallback()) {
501 INTELL_VOICE_LOG_ERROR("create wakeup source stop callback failed");
502 adapter_->Stop();
503 return -1;
504 }
505
506 SetParamOnAudioStart(*msgBody);
507
508 if (!StartAudioSource()) {
509 INTELL_VOICE_LOG_ERROR("start audio source failed");
510 SetParamOnAudioStop();
511 adapter_->Stop();
512 return -1;
513 }
514
515 INTELL_VOICE_LOG_INFO("exit");
516 nextState = State(RECOGNIZING);
517 return 0;
518 }
519
HandleStop(const StateMsg &,State & nextState)520 int32_t WakeupEngineImpl::HandleStop(const StateMsg & /* msg */, State &nextState)
521 {
522 StopAudioSource();
523 EngineUtil::Stop();
524 nextState = State(INITIALIZED);
525 return 0;
526 }
527
HandleRecognizeComplete(const StateMsg & msg,State & nextState)528 int32_t WakeupEngineImpl::HandleRecognizeComplete(const StateMsg &msg, State &nextState)
529 {
530 OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent *event =
531 reinterpret_cast<OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent *>(msg.inMsg);
532 if (event == nullptr) {
533 INTELL_VOICE_LOG_ERROR("result is nullptr");
534 return -1;
535 }
536
537 if (adapterListener_ != nullptr) {
538 adapterListener_->Notify(*event);
539 }
540
541 if (event->result != 0) {
542 INTELL_VOICE_LOG_ERROR("wakeup failed");
543 StopAudioSource();
544 nextState = State(INITIALIZED);
545 } else {
546 nextState = State(RECOGNIZED);
547 }
548 EngineUtil::Stop();
549 return 0;
550 }
551
HandleStartCapturer(const StateMsg & msg,State & nextState)552 int32_t WakeupEngineImpl::HandleStartCapturer(const StateMsg &msg, State &nextState)
553 {
554 INTELL_VOICE_LOG_INFO("enter");
555 auto ret = IntellVoiceUtil::VerifySystemPermission(OHOS_MICROPHONE_PERMISSION);
556 if (ret != INTELLIGENT_VOICE_SUCCESS) {
557 return ret;
558 }
559
560 int32_t *msgBody = reinterpret_cast<int32_t *>(msg.inMsg);
561 if (msgBody == nullptr) {
562 INTELL_VOICE_LOG_ERROR("msgBody is nullptr");
563 return INTELLIGENT_VOICE_START_CAPTURER_FAILED;
564 }
565 channels_ = *msgBody;
566 callerTokenId_ = IPCSkeleton::GetCallingTokenID();
567 IntellVoiceUtil::RecordPermissionPrivacy(OHOS_MICROPHONE_PERMISSION, callerTokenId_,
568 INTELL_VOICE_PERMISSION_START);
569 nextState = State(READ_CAPTURER);
570 return 0;
571 }
572
HandleRead(const StateMsg & msg,State &)573 int32_t WakeupEngineImpl::HandleRead(const StateMsg &msg, State & /* nextState */)
574 {
575 CapturerData *capturerData = reinterpret_cast<CapturerData *>(msg.outMsg);
576 if (capturerData == nullptr) {
577 INTELL_VOICE_LOG_ERROR("capturer data is nullptr");
578 return -1;
579 }
580
581 auto ret = WakeupSourceProcess::Read(capturerData->data, channels_);
582 if (ret != 0) {
583 INTELL_VOICE_LOG_ERROR("read capturer data failed");
584 return ret;
585 }
586
587 ResetTimerDelay();
588 return 0;
589 }
590
HandleStopCapturer(const StateMsg &,State & nextState)591 int32_t WakeupEngineImpl::HandleStopCapturer(const StateMsg & /* msg */, State &nextState)
592 {
593 INTELL_VOICE_LOG_INFO("enter");
594 StopAudioSource();
595 EngineUtil::Stop();
596 if (CurrState() == State(READ_CAPTURER)) {
597 IntellVoiceUtil::RecordPermissionPrivacy(OHOS_MICROPHONE_PERMISSION, callerTokenId_,
598 INTELL_VOICE_PERMISSION_STOP);
599 }
600 nextState = State(INITIALIZED);
601 return 0;
602 }
603
HandleRecognizingTimeout(const StateMsg &,State & nextState)604 int32_t WakeupEngineImpl::HandleRecognizingTimeout(const StateMsg & /* msg */, State &nextState)
605 {
606 INTELL_VOICE_LOG_INFO("enter");
607 StopAudioSource();
608 EngineUtil::Stop();
609 nextState = State(INITIALIZED);
610 return 0;
611 }
612
HandleGetWakeupPcm(const StateMsg & msg,State &)613 int32_t WakeupEngineImpl::HandleGetWakeupPcm(const StateMsg &msg, State & /* nextState */)
614 {
615 CapturerData *capturerData = reinterpret_cast<CapturerData *>(msg.outMsg);
616 if (capturerData == nullptr) {
617 INTELL_VOICE_LOG_ERROR("capturer data is nullptr");
618 return -1;
619 }
620 auto ret = EngineUtil::GetWakeupPcm(capturerData->data);
621 if (ret != 0) {
622 INTELL_VOICE_LOG_ERROR("get wakeup pcm failed");
623 return ret;
624 }
625 return 0;
626 }
627
HandleResetAdapter(const StateMsg &,State & nextState)628 int32_t WakeupEngineImpl::HandleResetAdapter(const StateMsg & /* msg */, State &nextState)
629 {
630 INTELL_VOICE_LOG_INFO("enter");
631 if (!EngineUtil::CreateAdapterInner(EngineHostManager::GetInstance(), WAKEUP_ADAPTER_TYPE)) {
632 INTELL_VOICE_LOG_ERROR("failed to create adapter");
633 return -1;
634 }
635 SetWakeupModel();
636 adapter_->SetCallback(callback_);
637 EngineUtil::SetLanguage();
638 EngineUtil::SetArea();
639 SetDspFeatures();
640
641 IntellVoiceEngineAdapterInfo adapterInfo = {
642 .wakeupPhrase = GetWakeupPhrase(),
643 .minBufSize = MIN_BUFFER_SIZE,
644 .sampleChannels = CHANNEL_CNT,
645 .bitsPerSample = BITS_PER_SAMPLE,
646 .sampleRate = SAMPLE_RATE,
647 };
648 if (adapter_->Attach(adapterInfo) != 0) {
649 INTELL_VOICE_LOG_ERROR("failed to attach");
650 EngineUtil::ReleaseAdapterInner(EngineHostManager::GetInstance());
651 return -1;
652 }
653
654 UpdateDspModel();
655
656 nextState = State(INITIALIZING);
657 return 0;
658 }
659
HandleRelease(const StateMsg &,State & nextState)660 int32_t WakeupEngineImpl::HandleRelease(const StateMsg & /* msg */, State &nextState)
661 {
662 DestroyWakeupSourceStopCallback();
663 StopAudioSource();
664 if (CurrState() == State(READ_CAPTURER)) {
665 IntellVoiceUtil::RecordPermissionPrivacy(OHOS_MICROPHONE_PERMISSION, callerTokenId_,
666 INTELL_VOICE_PERMISSION_STOP);
667 }
668 if (adapter_ != nullptr) {
669 adapter_->Detach();
670 ReleaseAdapterInner(EngineHostManager::GetInstance());
671 }
672 nextState = State(IDLE);
673 return 0;
674 }
675
SetWakeupModel()676 void WakeupEngineImpl::SetWakeupModel()
677 {
678 auto ret = EngineCallbackMessage::CallFunc(QUERY_SWITCH_STATUS, SHORTWORD_KEY);
679 bool switchstatus = false;
680 if (ret.has_value()) {
681 try {
682 switchstatus = std::any_cast<bool>(*ret);
683 } catch (const std::bad_any_cast&) {
684 INTELL_VOICE_LOG_ERROR("msg bus bad any cast");
685 return;
686 }
687 } else {
688 INTELL_VOICE_LOG_ERROR("msg bus return no value");
689 return;
690 }
691 if (switchstatus) {
692 adapter_->SetParameter("WakeupMode=1");
693 } else {
694 adapter_->SetParameter("WakeupMode=0");
695 }
696 }
697
UpdateDspModel()698 void WakeupEngineImpl::UpdateDspModel()
699 {
700 auto ret = EngineCallbackMessage::CallFunc(QUERY_SWITCH_STATUS, SHORTWORD_KEY);
701 bool switchstatus = false;
702 if (ret.has_value()) {
703 try {
704 switchstatus = std::any_cast<bool>(*ret);
705 } catch (const std::bad_any_cast&) {
706 INTELL_VOICE_LOG_ERROR("msg bus bad any cast");
707 return;
708 }
709 } else {
710 INTELL_VOICE_LOG_ERROR("msg bus return no value");
711 return;
712 }
713
714 if (switchstatus) {
715 SubscribeSwingEvent();
716 ProcDspModel(ReadDspModel(static_cast<OHOS::HDI::IntelligentVoice::Engine::V1_0::ContentType>(
717 OHOS::HDI::IntelligentVoice::Engine::V1_2::SHORT_WORD_DSP_MODEL)));
718 } else {
719 UnSubscribeSwingEvent();
720 ProcDspModel(ReadDspModel(OHOS::HDI::IntelligentVoice::Engine::V1_0::DSP_MODLE));
721 }
722 }
723
SubscribeSwingEvent()724 void WakeupEngineImpl::SubscribeSwingEvent()
725 {
726 INTELL_VOICE_LOG_INFO("enter");
727 isSubscribeSwingEvent_.store(true);
728 int32_t ret = SwingServiceWrapper::GetInstance().SubscribeSwingEvent(WAKEUP_CONFORMER_EVENT_TYPE,
729 WAKEUP_CONFORMER_DOMAIN_ID);
730 if (ret != 0) {
731 INTELL_VOICE_LOG_ERROR("load tiny model failed: %{public}d", ret);
732 return;
733 }
734 INTELL_VOICE_LOG_INFO("load tiny model");
735 }
736
UnSubscribeSwingEvent()737 void WakeupEngineImpl::UnSubscribeSwingEvent()
738 {
739 INTELL_VOICE_LOG_INFO("enter");
740 if (!isSubscribeSwingEvent_.load()) {
741 INTELL_VOICE_LOG_INFO("no need to UnSubscribeSwingEvent.");
742 return;
743 }
744 isSubscribeSwingEvent_.store(false);
745 int32_t ret = SwingServiceWrapper::GetInstance().UnSubscribeSwingEvent(WAKEUP_CONFORMER_EVENT_TYPE,
746 WAKEUP_CONFORMER_DOMAIN_ID);
747 if (ret != 0) {
748 INTELL_VOICE_LOG_ERROR("unload tiny model failed: %{public}d", ret);
749 return;
750 }
751 INTELL_VOICE_LOG_INFO("unload tiny model");
752 }
753
GetWakeupPhrase()754 std::string WakeupEngineImpl::GetWakeupPhrase()
755 {
756 std::string wakeupPhrase = HistoryInfoMgr::GetInstance().GetStringKVPair(KEY_WAKEUP_PHRASE);
757 if (wakeupPhrase.empty()) {
758 INTELL_VOICE_LOG_WARN("no phrase, use default phrase");
759 return std::string(DEFAULT_WAKEUP_PHRASE);
760 }
761
762 return wakeupPhrase;
763 }
764
ReadBufferCallback(uint8_t * buffer,uint32_t size,bool isEnd)765 void WakeupEngineImpl::ReadBufferCallback(uint8_t *buffer, uint32_t size, bool isEnd)
766 {
767 std::vector<std::vector<uint8_t>> audioData;
768 auto ret = IntellVoiceUtil::DeinterleaveAudioData(reinterpret_cast<int16_t *>(buffer),
769 size / sizeof(int16_t), static_cast<int32_t>(capturerOptions_.streamInfo.channels), audioData);
770 if ((!ret) || ((audioData.size() != static_cast<uint32_t>(capturerOptions_.streamInfo.channels))) ||
771 (channelId_ >= audioData.size())) {
772 INTELL_VOICE_LOG_ERROR("failed to deinterleave, ret:%{public}d, id:%{public}d", ret, channelId_);
773 return;
774 }
775 if ((adapter_ != nullptr) && !isEnd) {
776 if (channelId_ == CHANNEL_ID_1) { // whisper wakeup, need to write channel0 and channel1 data
777 std::vector<uint8_t> data(audioData[CHANNEL_ID_0].begin(), audioData[CHANNEL_ID_0].end());
778 data.insert(data.end(), audioData[CHANNEL_ID_1].begin(), audioData[CHANNEL_ID_1].end());
779 adapter_->WriteAudio(data);
780 } else {
781 adapter_->WriteAudio(audioData[channelId_]);
782 }
783 }
784 WakeupSourceProcess::Write(audioData);
785 }
786 }
787 }
788