1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #include "enroll_engine.h"
16 #include <fstream>
17 #include "ipc_skeleton.h"
18 #include "securec.h"
19 #include "intell_voice_info.h"
20 #include "intell_voice_log.h"
21 #include "intell_voice_util.h"
22 #include "enroll_adapter_listener.h"
23 #include "time_util.h"
24 #include "scope_guard.h"
25 #include "adapter_callback_service.h"
26 #include "intell_voice_engine_manager.h"
27 #include "update_engine_utils.h"
28 #include "engine_host_manager.h"
29 #include "history_info_mgr.h"
30
31 #define LOG_TAG "EnrollEngine"
32
33 using namespace OHOS::IntellVoice;
34 using namespace OHOS::HDI::IntelligentVoice::Engine::V1_0;
35 using namespace OHOS::IntellVoiceUtils;
36 using namespace OHOS::AudioStandard;
37
38 namespace OHOS {
39 namespace IntellVoiceEngine {
40 static constexpr uint32_t MIN_BUFFER_SIZE = 1280; // 16 * 2 * 40ms
41 static constexpr uint32_t INTERVAL = 125; // 125 * 40ms = 5s
42 static constexpr uint32_t MAX_ENROLL_TASK_NUM = 100;
43 static const std::string ENROLL_THREAD_NAME = "EnrollEngThread";
44
EnrollEngine()45 EnrollEngine::EnrollEngine() : TaskExecutor(ENROLL_THREAD_NAME, MAX_ENROLL_TASK_NUM)
46 {
47 INTELL_VOICE_LOG_INFO("enter");
48
49 capturerOptions_.streamInfo.samplingRate = AudioSamplingRate::SAMPLE_RATE_16000;
50 capturerOptions_.streamInfo.encoding = AudioEncodingType::ENCODING_PCM;
51 capturerOptions_.streamInfo.format = AudioSampleFormat::SAMPLE_S16LE;
52 capturerOptions_.streamInfo.channels = AudioChannel::MONO;
53 capturerOptions_.capturerInfo.sourceType = SourceType::SOURCE_TYPE_VOICE_RECOGNITION;
54 capturerOptions_.capturerInfo.capturerFlags = 0;
55 TaskExecutor::StartThread();
56 }
57
~EnrollEngine()58 EnrollEngine::~EnrollEngine()
59 {
60 INTELL_VOICE_LOG_INFO("enter");
61 callback_ = nullptr;
62 }
63
OnEnrollEvent(const OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent & event)64 void EnrollEngine::OnEnrollEvent(
65 const OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent &event)
66 {
67 if (event.msgId == INTELL_VOICE_ENGINE_MSG_ENROLL_COMPLETE) {
68 TaskExecutor::AddAsyncTask([this, result = event.result, info = event.info]() {
69 OnEnrollComplete(result, info);
70 }, "EnrollEngine::OnEnrollEvent", false);
71 } else if (event.msgId == INTELL_VOICE_ENGINE_MSG_COMMIT_ENROLL_COMPLETE) {
72 enrollResult_.store(static_cast<int32_t>(event.result));
73 IntellVoiceEngineManager::SetEnrollResult(INTELL_VOICE_ENROLL,
74 (static_cast<int32_t>(event.result) == 0 ? true : false));
75 }
76 }
77
OnEnrollComplete(int32_t result,const std::string & info)78 void EnrollEngine::OnEnrollComplete(int32_t result, const std::string &info)
79 {
80 std::lock_guard<std::mutex> lock(mutex_);
81 OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent event;
82 event.msgId = INTELL_VOICE_ENGINE_MSG_ENROLL_COMPLETE;
83 event.result = static_cast<OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineErrors>(result);
84 event.info = info;
85 if (adapterListener_ != nullptr) {
86 adapterListener_->Notify(event);
87 }
88 StopAudioSource();
89 }
90
Init(const std::string & param,bool reEnroll)91 bool EnrollEngine::Init(const std::string ¶m, bool reEnroll)
92 {
93 if (!EngineUtil::CreateAdapterInner(EngineHostManager::GetInstance(), ENROLL_ADAPTER_TYPE)) {
94 INTELL_VOICE_LOG_ERROR("failed to create adapter");
95 return false;
96 }
97 return true;
98 }
99
SetCallback(sptr<IRemoteObject> object)100 void EnrollEngine::SetCallback(sptr<IRemoteObject> object)
101 {
102 std::lock_guard<std::mutex> lock(mutex_);
103 INTELL_VOICE_LOG_INFO("enter");
104 if (adapter_ == nullptr) {
105 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
106 return;
107 }
108
109 sptr<IIntelligentVoiceEngineCallback> callback = iface_cast<IIntelligentVoiceEngineCallback>(object);
110 if (callback == nullptr) {
111 INTELL_VOICE_LOG_ERROR("callback is nullptr");
112 return;
113 }
114
115 adapterListener_ = std::make_shared<EnrollAdapterListener>(callback,
116 std::bind(&EnrollEngine::OnEnrollEvent, this, std::placeholders::_1));
117 if (adapterListener_ == nullptr) {
118 INTELL_VOICE_LOG_ERROR("adapter listener is nullptr");
119 return;
120 }
121
122 callback_ = sptr<IIntellVoiceEngineCallback>(new (std::nothrow) AdapterCallbackService(adapterListener_));
123 if (callback_ == nullptr) {
124 INTELL_VOICE_LOG_ERROR("callback_ is nullptr");
125 return;
126 }
127
128 adapter_->SetCallback(callback_);
129 }
130
Attach(const IntellVoiceEngineInfo & info)131 int32_t EnrollEngine::Attach(const IntellVoiceEngineInfo &info)
132 {
133 std::lock_guard<std::mutex> lock(mutex_);
134 INTELL_VOICE_LOG_INFO("attach");
135 if (adapter_ == nullptr) {
136 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
137 return -1;
138 }
139
140 SetDspFeatures();
141 EngineUtil::SetImproveParam();
142
143 isPcmFromExternal_ = info.isPcmFromExternal;
144 wakeupPhrase_ = info.wakeupPhrase;
145
146 IntellVoiceEngineAdapterInfo adapterInfo = {
147 .wakeupPhrase = info.wakeupPhrase,
148 .minBufSize = info.minBufSize,
149 .sampleChannels = info.sampleChannels,
150 .bitsPerSample = info.bitsPerSample,
151 .sampleRate = info.sampleRate,
152 };
153 return adapter_->Attach(adapterInfo);
154 }
155
Detach(void)156 int32_t EnrollEngine::Detach(void)
157 {
158 INTELL_VOICE_LOG_INFO("enter");
159 std::lock_guard<std::mutex> lock(mutex_);
160 StopAudioSource();
161 if (adapter_ == nullptr) {
162 INTELL_VOICE_LOG_WARN("already detach");
163 return 0;
164 }
165
166 if (enrollResult_.load() == 0) {
167 EngineUtil::ProcDspModel(ReadDspModel(OHOS::HDI::IntelligentVoice::Engine::V1_0::DSP_MODLE));
168 HistoryInfoMgr::GetInstance().SetStringKVPair(KEY_WAKEUP_PHRASE, wakeupPhrase_);
169 /* save new version number */
170 UpdateEngineUtils::SaveWakeupVesion();
171 INTELL_VOICE_LOG_INFO("enroll save version");
172 }
173
174 int32_t ret = adapter_->Detach();
175 ReleaseAdapterInner(EngineHostManager::GetInstance());
176 return ret;
177 }
178
Start(bool isLast)179 int32_t EnrollEngine::Start(bool isLast)
180 {
181 std::lock_guard<std::mutex> lock(mutex_);
182 INTELL_VOICE_LOG_INFO("enter");
183 auto ret = IntellVoiceUtil::VerifySystemPermission(OHOS_MICROPHONE_PERMISSION);
184 if (ret != INTELLIGENT_VOICE_SUCCESS) {
185 return ret;
186 }
187
188 if (audioSource_ != nullptr) {
189 INTELL_VOICE_LOG_ERROR("audioSource_ existed, wait for last start to finish");
190 return -1;
191 }
192
193 if (adapter_ == nullptr) {
194 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
195 return -1;
196 }
197
198 StartInfo info = {
199 .isLast = isLast,
200 };
201 if (adapter_->Start(info)) {
202 INTELL_VOICE_LOG_ERROR("start adapter failed");
203 return -1;
204 }
205
206 if (isPcmFromExternal_) {
207 INTELL_VOICE_LOG_INFO("pcm is from external");
208 return 0;
209 }
210
211 SelectInputDevice(DeviceType::DEVICE_TYPE_MIC);
212
213 if (!StartAudioSource()) {
214 INTELL_VOICE_LOG_ERROR("start audio source failed");
215 adapter_->Stop();
216 return -1;
217 }
218
219 INTELL_VOICE_LOG_INFO("exit");
220 return 0;
221 }
222
Stop()223 int32_t EnrollEngine::Stop()
224 {
225 std::lock_guard<std::mutex> lock(mutex_);
226 StopAudioSource();
227
228 return EngineUtil::Stop();
229 }
230
SetParameter(const std::string & keyValueList)231 int32_t EnrollEngine::SetParameter(const std::string &keyValueList)
232 {
233 std::lock_guard<std::mutex> lock(mutex_);
234 if (SetParameterInner(keyValueList)) {
235 INTELL_VOICE_LOG_INFO("inner parameter");
236 return 0;
237 }
238
239 INTELL_VOICE_LOG_INFO("EnrollEngine SetParameter:%{public}s", keyValueList.c_str());
240
241 return EngineUtil::SetParameter(keyValueList);
242 }
243
SetParameterInner(const std::string & keyValueList)244 bool EnrollEngine::SetParameterInner(const std::string &keyValueList)
245 {
246 HistoryInfoMgr &historyInfoMgr = HistoryInfoMgr::GetInstance();
247
248 std::map<std::string, std::string> kvpairs;
249 IntellVoiceUtil::SplitStringToKVPair(keyValueList, kvpairs);
250 for (auto it : kvpairs) {
251 if (it.first == std::string("wakeup_bundle_name")) {
252 INTELL_VOICE_LOG_INFO("set wakeup bundle name:%{public}s", it.second.c_str());
253 historyInfoMgr.SetStringKVPair(KEY_WAKEUP_ENGINE_BUNDLE_NAME, it.second);
254 return true;
255 }
256 if (it.first == std::string("wakeup_ability_name")) {
257 INTELL_VOICE_LOG_INFO("set wakeup ability name:%{public}s", it.second.c_str());
258 historyInfoMgr.SetStringKVPair(KEY_WAKEUP_ENGINE_ABILITY_NAME, it.second);
259 return true;
260 }
261 if (it.first == std::string("language")) {
262 INTELL_VOICE_LOG_DEBUG("set language:%{public}s", it.second.c_str());
263 historyInfoMgr.SetStringKVPair(KEY_LANGUAGE, it.second);
264 continue;
265 }
266 if (it.first == std::string("area")) {
267 INTELL_VOICE_LOG_DEBUG("set area:%{public}s", it.second.c_str());
268 historyInfoMgr.SetStringKVPair(KEY_AREA, it.second);
269 continue;
270 }
271 if (it.first == std::string("Sensibility")) {
272 INTELL_VOICE_LOG_INFO("set Sensibility:%{public}s", it.second.c_str());
273 historyInfoMgr.SetStringKVPair(KEY_SENSIBILITY, it.second);
274 continue;
275 }
276 }
277
278 return false;
279 }
280
GetParameter(const std::string & key)281 std::string EnrollEngine::GetParameter(const std::string &key)
282 {
283 std::lock_guard<std::mutex> lock(mutex_);
284 return EngineUtil::GetParameter(key);
285 }
286
WriteAudio(const uint8_t * buffer,uint32_t size)287 int32_t EnrollEngine::WriteAudio(const uint8_t *buffer, uint32_t size)
288 {
289 std::lock_guard<std::mutex> lock(mutex_);
290 return EngineUtil::WriteAudio(buffer, size);
291 }
292
Evaluate(const std::string & word,EvaluationResult & result)293 int32_t EnrollEngine::Evaluate(const std::string &word, EvaluationResult &result)
294 {
295 std::lock_guard<std::mutex> lock(mutex_);
296 OHOS::HDI::IntelligentVoice::Engine::V1_2::EvaluationResultInfo info;
297 int32_t ret = EngineUtil::Evaluate(word, info);
298 if (ret == 0) {
299 result.score = info.score;
300 result.resultCode = info.resultCode;
301 }
302
303 return ret;
304 }
305
StartAudioSource()306 bool EnrollEngine::StartAudioSource()
307 {
308 callerTokenId_ = IPCSkeleton::GetCallingTokenID();
309 if (!IntellVoiceUtil::RecordPermissionPrivacy(OHOS_MICROPHONE_PERMISSION, callerTokenId_,
310 INTELL_VOICE_PERMISSION_START)) {
311 INTELL_VOICE_LOG_ERROR("record permissionPrivacy failed");
312 return false;
313 }
314
315 auto listener = std::make_unique<AudioSourceListener>([&] (uint8_t *buffer, uint32_t size, bool isEnd) {
316 if ((adapter_ != nullptr) && (!isEnd)) {
317 std::vector<uint8_t> audioBuff(&buffer[0], &buffer[size]);
318 adapter_->WriteAudio(audioBuff);
319 }}, [&] () {
320 INTELL_VOICE_LOG_INFO("end of pcm");
321 if (adapter_ != nullptr) {
322 adapter_->SetParameter("end_of_pcm=true");
323 }
324 });
325 if (listener == nullptr) {
326 INTELL_VOICE_LOG_ERROR("create listener failed");
327 return false;
328 }
329
330 audioSource_ = std::make_unique<AudioSource>(MIN_BUFFER_SIZE, INTERVAL, std::move(listener),
331 capturerOptions_);
332 if (audioSource_ == nullptr) {
333 INTELL_VOICE_LOG_ERROR("create audio source failed");
334 return false;
335 }
336
337 if (!audioSource_->Start()) {
338 INTELL_VOICE_LOG_ERROR("start capturer failed");
339 audioSource_ = nullptr;
340 IntellVoiceUtil::RecordPermissionPrivacy(OHOS_MICROPHONE_PERMISSION, callerTokenId_,
341 INTELL_VOICE_PERMISSION_STOP);
342 return false;
343 }
344 return true;
345 }
346
StopAudioSource()347 void EnrollEngine::StopAudioSource()
348 {
349 INTELL_VOICE_LOG_INFO("enter");
350 if (audioSource_ != nullptr) {
351 INTELL_VOICE_LOG_INFO("stop audio sopurce");
352 audioSource_->Stop();
353 audioSource_ = nullptr;
354 IntellVoiceUtil::RecordPermissionPrivacy(OHOS_MICROPHONE_PERMISSION, callerTokenId_,
355 INTELL_VOICE_PERMISSION_STOP);
356 }
357 }
358 }
359 }
360