1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #include "enroll_engine.h"
16 #include <fstream>
17 #include "securec.h"
18 #include "intell_voice_log.h"
19
20 #include "v1_0/iintell_voice_engine_manager.h"
21 #include "v1_0/iintell_voice_engine_callback.h"
22
23 #include "enroll_adapter_listener.h"
24 #include "time_util.h"
25 #include "scope_guard.h"
26 #include "trigger_manager.h"
27 #include "adapter_callback_service.h"
28 #include "intell_voice_service_manager.h"
29
30 #define LOG_TAG "EnrollEngine"
31
32 using namespace OHOS::IntellVoiceTrigger;
33 using namespace OHOS::HDI::IntelligentVoice::Engine::V1_0;
34 using namespace OHOS::IntellVoiceUtils;
35 using namespace OHOS::AudioStandard;
36
37 namespace OHOS {
38 namespace IntellVoiceEngine {
39 static constexpr uint32_t MIN_BUFFER_SIZE = 1280; // 16 * 2 * 40ms
40 static constexpr uint32_t INTERVAL = 125; // 125 * 40ms = 5s
EnrollEngine()41 EnrollEngine::EnrollEngine()
42 {
43 INTELL_VOICE_LOG_INFO("enter");
44
45 capturerOptions_.streamInfo.samplingRate = AudioSamplingRate::SAMPLE_RATE_16000;
46 capturerOptions_.streamInfo.encoding = AudioEncodingType::ENCODING_PCM;
47 capturerOptions_.streamInfo.format = AudioSampleFormat::SAMPLE_S16LE;
48 capturerOptions_.streamInfo.channels = AudioChannel::MONO;
49 capturerOptions_.capturerInfo.sourceType = SourceType::SOURCE_TYPE_MIC;
50 capturerOptions_.capturerInfo.capturerFlags = 0;
51 }
52
~EnrollEngine()53 EnrollEngine::~EnrollEngine()
54 {
55 auto mgr = IIntellVoiceEngineManager::Get();
56 if (mgr != nullptr) {
57 mgr->ReleaseAdapter(desc_);
58 }
59 adapter_ = nullptr;
60 callback_ = nullptr;
61 if (audioSource_ != nullptr) {
62 audioSource_->Stop();
63 audioSource_ = nullptr;
64 }
65 }
66
OnEnrollEvent(int32_t msgId,int32_t result)67 void EnrollEngine::OnEnrollEvent(int32_t msgId, int32_t result)
68 {
69 if (msgId == INTELL_VOICE_ENGINE_MSG_ENROLL_COMPLETE) {
70 std::thread(&EnrollEngine::OnEnrollComplete, this).detach();
71 } else if (msgId == INTELL_VOICE_ENGINE_MSG_COMMIT_ENROLL_COMPLETE) {
72 std::lock_guard<std::mutex> lock(mutex_);
73 enrollResult_ = result;
74 IntellVoiceServiceManager::SetEnrollResult(result == 0 ? true : false);
75 } else {
76 }
77 }
78
OnEnrollComplete()79 void EnrollEngine::OnEnrollComplete()
80 {
81 StopAudioSource();
82 }
83
Init()84 bool EnrollEngine::Init()
85 {
86 desc_.adapterType = ENROLL_ADAPTER_TYPE;
87 auto mgr = IIntellVoiceEngineManager::Get();
88 if (mgr == nullptr) {
89 INTELL_VOICE_LOG_ERROR("failed to get engine manager");
90 return false;
91 }
92
93 mgr->CreateAdapter(desc_, adapter_);
94 if (adapter_ == nullptr) {
95 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
96 return false;
97 }
98
99 return true;
100 }
101
SetCallback(sptr<IRemoteObject> object)102 void EnrollEngine::SetCallback(sptr<IRemoteObject> object)
103 {
104 std::lock_guard<std::mutex> lock(mutex_);
105 if (adapter_ == nullptr) {
106 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
107 return;
108 }
109
110 sptr<IIntelligentVoiceEngineCallback> callback = iface_cast<IIntelligentVoiceEngineCallback>(object);
111 if (callback == nullptr) {
112 INTELL_VOICE_LOG_ERROR("callback is nullptr");
113 return;
114 }
115
116 std::shared_ptr<IntellVoiceAdapterListener> listener = std::make_shared<EnrollAdapterListener>(callback,
117 std::bind(&EnrollEngine::OnEnrollEvent, this, std::placeholders::_1, std::placeholders::_2));
118 if (listener == nullptr) {
119 INTELL_VOICE_LOG_ERROR("listener is nullptr");
120 return;
121 }
122
123 callback_ = sptr<IIntellVoiceEngineCallback>(new (std::nothrow) AdapterCallbackService(listener));
124 if (callback_ == nullptr) {
125 INTELL_VOICE_LOG_ERROR("callback_ is nullptr");
126 return;
127 }
128
129 adapter_->SetCallback(callback_);
130 }
131
Attach(const IntellVoiceEngineInfo & info)132 int32_t EnrollEngine::Attach(const IntellVoiceEngineInfo &info)
133 {
134 std::lock_guard<std::mutex> lock(mutex_);
135 INTELL_VOICE_LOG_INFO("attach");
136 if (adapter_ == nullptr) {
137 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
138 return -1;
139 }
140
141 isPcmFromExternal_ = info.isPcmFromExternal;
142
143 IntellVoiceEngineAdapterInfo adapterInfo = {
144 .wakeupPhrase = info.wakeupPhrase,
145 .minBufSize = info.minBufSize,
146 .sampleChannels = info.sampleChannels,
147 .bitsPerSample = info.bitsPerSample,
148 .sampleRate = info.sampleRate,
149 };
150 return adapter_->Attach(adapterInfo);
151 }
152
Detach(void)153 int32_t EnrollEngine::Detach(void)
154 {
155 INTELL_VOICE_LOG_ERROR("EnrollEngine::Detach");
156 std::lock_guard<std::mutex> lock(mutex_);
157 if (adapter_ == nullptr) {
158 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
159 return -1;
160 }
161
162 if (enrollResult_ == 0) {
163 ProcDspModel();
164 }
165
166 return adapter_->Detach();
167 }
168
Start(bool isLast)169 int32_t EnrollEngine::Start(bool isLast)
170 {
171 std::lock_guard<std::mutex> lock(mutex_);
172 INTELL_VOICE_LOG_INFO("enter");
173 if (audioSource_ != nullptr) {
174 INTELL_VOICE_LOG_ERROR("audioSource_ existed, wait for last start to finish");
175 return -1;
176 }
177
178 if (adapter_ == nullptr) {
179 INTELL_VOICE_LOG_ERROR("adapter is nullptr");
180 return -1;
181 }
182
183 if (IntellVoiceServiceManager::GetInstance()->ApplyArbitration(INTELL_VOICE_ENROLL, ENGINE_EVENT_START) !=
184 ARBITRATION_OK) {
185 INTELL_VOICE_LOG_ERROR("policy manager reject to start engine");
186 return 0;
187 }
188
189 StartInfo info = {
190 .isLast = isLast,
191 };
192 if (adapter_->Start(info)) {
193 INTELL_VOICE_LOG_ERROR("start adapter failed");
194 return -1;
195 }
196
197 if (isPcmFromExternal_) {
198 INTELL_VOICE_LOG_INFO("pcm is from external");
199 return 0;
200 }
201
202 if (!StartAudioSource()) {
203 INTELL_VOICE_LOG_ERROR("start audio source failed");
204 adapter_->Stop();
205 return -1;
206 }
207
208 INTELL_VOICE_LOG_INFO("exit");
209 return 0;
210 }
211
Stop()212 int32_t EnrollEngine::Stop()
213 {
214 StopAudioSource();
215
216 return EngineBase::Stop();
217 }
218
SetParameter(const std::string & keyValueList)219 int32_t EnrollEngine::SetParameter(const std::string &keyValueList)
220 {
221 if (SetParameterInner(keyValueList)) {
222 INTELL_VOICE_LOG_INFO("inner parameter");
223 return 0;
224 }
225
226 return EngineBase::SetParameter(keyValueList);
227 }
228
SetParameterInner(const std::string & keyValueList)229 bool EnrollEngine::SetParameterInner(const std::string &keyValueList)
230 {
231 std::lock_guard<std::mutex> lock(mutex_);
232
233 const std::unique_ptr<HistoryInfoMgr> &historyInfoMgr =
234 IntellVoiceServiceManager::GetInstance()->GetHistoryInfoMgr();
235 if (historyInfoMgr == nullptr) {
236 INTELL_VOICE_LOG_ERROR("historyInfoMgr is nullptr");
237 return false;
238 }
239
240 std::map<std::string, std::string> kvpairs;
241 SplitStringToKVPair(keyValueList, kvpairs);
242 for (auto it : kvpairs) {
243 if (it.first == std::string("wakeup_bundle_name")) {
244 INTELL_VOICE_LOG_INFO("set wakeup bundle name:%{public}s", it.second.c_str());
245 historyInfoMgr->SetWakeupEngineBundleName(it.second);
246 return true;
247 }
248 if (it.first == std::string("wakeup_ability_name")) {
249 INTELL_VOICE_LOG_INFO("set wakeup ability name:%{public}s", it.second.c_str());
250 historyInfoMgr->SetWakeupEngineAbilityName(it.second);
251 return true;
252 }
253 }
254
255 return false;
256 }
257
WriteBufferFromAshmem(uint8_t * & buffer,uint32_t size,sptr<OHOS::Ashmem> ashmem)258 void EnrollEngine::WriteBufferFromAshmem(uint8_t *&buffer, uint32_t size, sptr<OHOS::Ashmem> ashmem)
259 {
260 if (!ashmem->MapReadOnlyAshmem()) {
261 INTELL_VOICE_LOG_ERROR("map ashmem failed");
262 return;
263 }
264
265 const uint8_t *tmpBuffer = static_cast<const uint8_t *>(ashmem->ReadFromAshmem(size, 0));
266 if (tmpBuffer == nullptr) {
267 INTELL_VOICE_LOG_ERROR("read from ashmem failed");
268 return;
269 }
270
271 buffer = new (std::nothrow) uint8_t[size];
272 if (buffer == nullptr) {
273 INTELL_VOICE_LOG_ERROR("allocate buffer failed");
274 return;
275 }
276
277 if (memcpy_s(buffer, size, tmpBuffer, size) != 0) {
278 INTELL_VOICE_LOG_ERROR("memcpy_s failed");
279 return;
280 }
281 }
282
ProcDspModel()283 void EnrollEngine::ProcDspModel()
284 {
285 INTELL_VOICE_LOG_ERROR("enter");
286 uint8_t *buffer = nullptr;
287 uint32_t size = 0;
288 sptr<Ashmem> ashmem;
289 adapter_->Read(DSP_MODLE, ashmem);
290 if (ashmem == nullptr) {
291 INTELL_VOICE_LOG_ERROR("ashmem is nullptr");
292 return;
293 }
294
295 ON_SCOPE_EXIT_WITH_NAME(ashmemExit)
296 {
297 INTELL_VOICE_LOG_DEBUG("close ashmem");
298 ashmem->UnmapAshmem();
299 ashmem->CloseAshmem();
300 };
301
302 size = static_cast<uint32_t>(ashmem->GetAshmemSize());
303 if (size == 0) {
304 INTELL_VOICE_LOG_ERROR("size is zero");
305 return;
306 }
307
308 WriteBufferFromAshmem(buffer, size, ashmem);
309 if (buffer == nullptr) {
310 INTELL_VOICE_LOG_ERROR("buffer is nullptr");
311 return;
312 }
313
314 ON_SCOPE_EXIT_WITH_NAME(bufferExit)
315 {
316 INTELL_VOICE_LOG_DEBUG("now delete buffer");
317 delete[] buffer;
318 buffer = nullptr;
319 };
320
321 std::shared_ptr<GenericTriggerModel> model = std::make_shared<GenericTriggerModel>(
322 (IntellVoiceServiceManager::GetEnrollModelUuid()), 1);
323 if (model == nullptr) {
324 INTELL_VOICE_LOG_ERROR("model is null");
325 return;
326 }
327
328 model->SetData(buffer, size);
329 auto triggerMgr = TriggerManager::GetInstance();
330 if (triggerMgr == nullptr) {
331 INTELL_VOICE_LOG_ERROR("trigger manager is nullptr");
332 return;
333 }
334 triggerMgr->UpdateModel(model);
335 }
336
StartAudioSource()337 bool EnrollEngine::StartAudioSource()
338 {
339 auto listener = std::make_unique<AudioSourceListener>([&] (uint8_t *buffer, uint32_t size) {
340 if (adapter_ != nullptr) {
341 std::vector<uint8_t> audioBuff(&buffer[0], &buffer[size]);
342 adapter_->WriteAudio(audioBuff);
343 }}, [&] (bool isError) {
344 INTELL_VOICE_LOG_INFO("end of pcm, isError:%{public}d", isError);
345 if ((adapter_ != nullptr) && (!isError)) {
346 adapter_->SetParameter("end_of_pcm=true");
347 }
348 });
349 if (listener == nullptr) {
350 INTELL_VOICE_LOG_ERROR("create listener failed");
351 return false;
352 }
353
354 audioSource_ = std::make_unique<AudioSource>(MIN_BUFFER_SIZE, INTERVAL, std::move(listener),
355 capturerOptions_);
356 if (audioSource_ == nullptr) {
357 INTELL_VOICE_LOG_ERROR("create audio source failed");
358 return false;
359 }
360
361 if (!audioSource_->Start()) {
362 INTELL_VOICE_LOG_ERROR("start capturer failed");
363 audioSource_ = nullptr;
364 return false;
365 }
366
367 return true;
368 }
369
StopAudioSource()370 void EnrollEngine::StopAudioSource()
371 {
372 std::lock_guard<std::mutex> lock(mutex_);
373 INTELL_VOICE_LOG_INFO("enter");
374 if (audioSource_ != nullptr) {
375 audioSource_->Stop();
376 audioSource_ = nullptr;
377 }
378 }
379 }
380 }