1 /* 2 * Copyright (c) 2023 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 #ifndef INTELL_VOICE_TRIGGER_HELPER_H 16 #define INTELL_VOICE_TRIGGER_HELPER_H 17 18 #include <cstdint> 19 #include <map> 20 #include "msg_handle_thread.h" 21 #include "trigger_base_type.h" 22 #include "i_intell_voice_trigger_recognition_callback.h" 23 #include "i_intell_voice_trigger_connector_module.h" 24 #include "i_intell_voice_trigger_connector_callback.h" 25 #include "trigger_connector_common_type.h" 26 #ifdef SUPPORT_TELEPHONY_SERVICE 27 #include "telephony_observer.h" 28 #endif 29 #include "audio_system_manager.h" 30 #include "audio_stream_manager.h" 31 #include "audio_info.h" 32 #ifdef POWER_MANAGER_ENABLE 33 #include "sync_hibernate_callback_stub.h" 34 #include "sync_sleep_callback_stub.h" 35 #include "power_mgr_client.h" 36 #endif 37 #ifdef SUPPORT_WINDOW_MANAGER 38 #include "display_manager_lite.h" 39 #endif 40 41 namespace OHOS { 42 namespace IntellVoiceTrigger { 43 using OHOS::AudioStandard::AudioCapturerSourceCallback; 44 using OHOS::AudioStandard::AudioRendererStateChangeCallback; 45 using OHOS::AudioStandard::AudioRendererChangeInfo; 46 using OHOS::AudioStandard::AudioManagerAudioSceneChangedCallback; 47 using OHOS::AudioStandard::AudioScene; 48 #ifdef SUPPORT_WINDOW_MANAGER 49 using FoldStatus = OHOS::Rosen::FoldStatus; 50 #endif 51 52 enum ModelState { MODEL_NOTLOADED, MODEL_LOADED, MODEL_STARTED, MODEL_STATE_BUT }; 53 54 class TriggerModelData { 55 public: 56 explicit TriggerModelData(int32_t uuid); 57 ~TriggerModelData(); 58 void SetCallback(std::shared_ptr<IIntellVoiceTriggerRecognitionCallback> callback); 59 std::shared_ptr<IIntellVoiceTriggerRecognitionCallback> GetCallback(); 60 void SetModel(std::shared_ptr<GenericTriggerModel> model); 61 std::shared_ptr<GenericTriggerModel> GetModel(); 62 void SetState(ModelState state); 63 ModelState GetState() const; 64 void SetModelHandle(int32_t handle); 65 int32_t GetModelHandle() const; 66 void SetRequested(bool requested); 67 bool GetRequested() const; 68 69 bool SameModel(std::shared_ptr<GenericTriggerModel> model); 70 void Clear(); 71 void ClearCallback(); 72 73 public: 74 int32_t uuid_ = -1; 75 76 private: 77 ModelState state_ = MODEL_NOTLOADED; 78 std::shared_ptr<GenericTriggerModel> model_ = nullptr; 79 std::shared_ptr<IIntellVoiceTriggerRecognitionCallback> callback_ = nullptr; 80 int32_t modelHandle_ = 0; 81 bool requested_ = false; 82 }; 83 84 class TriggerHelper : public IIntellVoiceTriggerConnectorCallback, public std::enable_shared_from_this<TriggerHelper> { 85 public: 86 ~TriggerHelper(); 87 88 static std::shared_ptr<TriggerHelper> Create(); 89 90 int32_t StartGenericRecognition(int32_t uuid, std::shared_ptr<GenericTriggerModel> model, 91 std::shared_ptr<IIntellVoiceTriggerRecognitionCallback> callback); 92 int32_t StopGenericRecognition(int32_t uuid, std::shared_ptr<IIntellVoiceTriggerRecognitionCallback> callback); 93 void UnloadGenericTriggerModel(int32_t uuid); 94 int32_t SetParameter(const std::string &key, const std::string &value); 95 std::string GetParameter(const std::string &key); 96 void AttachAudioCaptureListener(); 97 void DetachAudioCaptureListener(); 98 void AttachAudioRendererEventListener(); 99 void DetachAudioRendererEventListener(); 100 void AttachAudioSceneEventListener(); 101 void DetachAudioSceneEventListener(); 102 #ifdef SUPPORT_TELEPHONY_SERVICE 103 void AttachTelephonyObserver(); 104 void DetachTelephonyObserver(); 105 #endif 106 #ifdef POWER_MANAGER_ENABLE 107 void AttachHibernateObserver(); 108 void DetachHibernateObserver(); 109 #endif 110 111 private: 112 TriggerHelper(); 113 bool GetModule(); 114 std::shared_ptr<TriggerModelData> GetTriggerModelData(int32_t uuid); 115 std::shared_ptr<TriggerModelData> CreateTriggerModelData(int32_t uuid); 116 int32_t InitRecognition(std::shared_ptr<TriggerModelData> modelData, bool unload); 117 int32_t PrepareForRecognition(std::shared_ptr<TriggerModelData> modelData); 118 int32_t StartRecognition(std::shared_ptr<TriggerModelData> modelData); 119 int32_t StopRecognition(std::shared_ptr<TriggerModelData> modelData); 120 int32_t LoadModel(std::shared_ptr<TriggerModelData> modelData); 121 int32_t UnloadModel(std::shared_ptr<TriggerModelData> modelData); 122 void OnUpdateAllRecognitionState(); 123 bool IsConflictSceneActive(); 124 125 void OnRecognition(int32_t modelHandle, const IntellVoiceRecognitionEvent &event) override; 126 void OnCapturerStateChange(bool isActive); 127 void OnUpdateRendererState(int32_t streamUsage, bool isPlaying); 128 #ifdef POWER_MANAGER_ENABLE 129 void OnHibernateStateUpdated(bool isHibernate); 130 #endif 131 void OnAudioSceneChange(const AudioScene audioScene); 132 #ifdef SUPPORT_TELEPHONY_SERVICE 133 void OnCallStateUpdated(int32_t callState); 134 class TelephonyStateObserver : public Telephony::TelephonyObserver { 135 public: TelephonyStateObserver(const std::shared_ptr<TriggerHelper> helper)136 explicit TelephonyStateObserver(const std::shared_ptr<TriggerHelper> helper) : helper_(helper) 137 {} ~TelephonyStateObserver()138 ~TelephonyStateObserver() 139 { 140 helper_ = nullptr; 141 } 142 void OnCallStateUpdated(int32_t slotId, int32_t callState, const std::u16string &phoneNumber) override; 143 144 public: 145 std::shared_ptr<TriggerHelper> helper_ = nullptr; 146 }; 147 148 private: 149 sptr<TelephonyStateObserver> telephonyObserver0_ = nullptr; 150 #endif 151 152 #ifdef SUPPORT_WINDOW_MANAGER 153 public: 154 void AttachFoldStatusListener(); 155 void DetachFoldStatusListener(); 156 157 private: 158 void UpdateGenericTriggerModel(std::shared_ptr<GenericTriggerModel> model); 159 std::shared_ptr<GenericTriggerModel> ReadWhisperModel(); 160 void ReLoadWhisperModel(std::shared_ptr<TriggerModelData> modelData); 161 void FoldStatusOperation(std::shared_ptr<TriggerModelData> modelData); 162 void SetFoldStatus(); 163 void RegisterFoldStatusListener(); 164 void StartAllRecognition(); 165 void StopAllRecognition(); 166 void RestartAllRecognition(); 167 void OnFoldStatusChanged(FoldStatus foldStatus); 168 std::string GetFoldStatusInfo(); 169 bool GetParameterInner(const std::string &key, std::string &value); 170 class FoldStatusListener : public OHOS::Rosen::DisplayManagerLite::IFoldStatusListener { 171 public: FoldStatusListener(const std::shared_ptr<TriggerHelper> helper)172 explicit FoldStatusListener(const std::shared_ptr<TriggerHelper> helper) : helper_(helper) 173 {} 174 ~FoldStatusListener()175 ~FoldStatusListener() 176 { 177 helper_ = nullptr; 178 } 179 void OnFoldStatusChanged(FoldStatus foldStatus) override; 180 181 public: 182 std::shared_ptr<TriggerHelper> helper_ = nullptr; 183 }; 184 private: 185 sptr<OHOS::Rosen::DisplayManagerLite::IFoldStatusListener> foldStatusListener_; 186 FoldStatus curFoldStatus_ = FoldStatus::UNKNOWN; 187 bool isFoldStatusDetached_ = false; 188 bool isFoldable_ = false; 189 std::mutex foldStatusMutex_; 190 #endif 191 192 class AudioSceneChangeCallback : public AudioManagerAudioSceneChangedCallback { 193 public: AudioSceneChangeCallback(const std::shared_ptr<TriggerHelper> helper)194 explicit AudioSceneChangeCallback(const std::shared_ptr<TriggerHelper> helper) : helper_(helper) 195 {} ~AudioSceneChangeCallback()196 ~AudioSceneChangeCallback() 197 { 198 helper_ = nullptr; 199 } 200 void OnAudioSceneChange(const AudioScene audioScene) override; 201 202 public: 203 std::shared_ptr<TriggerHelper> helper_ = nullptr; 204 }; 205 206 class AudioCapturerSourceChangeCallback : public AudioCapturerSourceCallback { 207 public: AudioCapturerSourceChangeCallback(const std::shared_ptr<TriggerHelper> helper)208 explicit AudioCapturerSourceChangeCallback(const std::shared_ptr<TriggerHelper> helper) : helper_(helper) 209 {} ~AudioCapturerSourceChangeCallback()210 ~AudioCapturerSourceChangeCallback() 211 { 212 helper_ = nullptr; 213 } 214 void OnCapturerState(bool isActive) override; 215 216 public: 217 std::shared_ptr<TriggerHelper> helper_ = nullptr; 218 }; 219 220 class AudioRendererStateChangeCallbackImpl : public AudioRendererStateChangeCallback { 221 public: AudioRendererStateChangeCallbackImpl(const std::shared_ptr<TriggerHelper> helper)222 explicit AudioRendererStateChangeCallbackImpl(const std::shared_ptr<TriggerHelper> helper) 223 : helper_(helper) 224 {} ~AudioRendererStateChangeCallbackImpl()225 ~AudioRendererStateChangeCallbackImpl() 226 { 227 helper_ = nullptr; 228 rendererStateMap_.clear(); 229 } 230 231 void OnRendererStateChange( 232 const std::vector<std::shared_ptr<AudioRendererChangeInfo>> &audioRendererChangeInfos) override; 233 private: 234 std::mutex mutex_; 235 std::shared_ptr<TriggerHelper> helper_ = nullptr; 236 std::map<int32_t, bool> rendererStateMap_; 237 }; 238 239 #ifdef POWER_MANAGER_ENABLE 240 class HibernateCallback : public PowerMgr::SyncHibernateCallbackStub { 241 public: HibernateCallback(const std::shared_ptr<TriggerHelper> helper)242 explicit HibernateCallback(const std::shared_ptr<TriggerHelper> helper) 243 : helper_(helper) 244 {} ~HibernateCallback()245 ~HibernateCallback() 246 { 247 helper_ = nullptr; 248 } 249 250 void OnSyncHibernate() override; 251 void OnSyncWakeup(bool hibernateResult = false) override; 252 253 private: 254 std::shared_ptr<TriggerHelper> helper_ = nullptr; 255 }; 256 257 class SleepCallback : public PowerMgr::SyncSleepCallbackStub { 258 public: SleepCallback(const std::shared_ptr<TriggerHelper> helper)259 explicit SleepCallback(const std::shared_ptr<TriggerHelper> helper) 260 : helper_(helper) 261 {} ~SleepCallback()262 ~SleepCallback() 263 { 264 helper_ = nullptr; 265 } 266 267 void OnSyncSleep(bool onForceSleep); 268 void OnSyncWakeup(bool onForceSleep); 269 270 private: 271 std::shared_ptr<TriggerHelper> helper_ = nullptr; 272 }; 273 #endif 274 275 private: 276 std::mutex mutex_; 277 std::mutex telephonyMutex_; 278 std::mutex rendererMutex_; 279 #ifdef POWER_MANAGER_ENABLE 280 std::mutex hiberateMutex_; 281 #endif 282 std::mutex sceneMutex_; 283 284 std::map<int32_t, std::shared_ptr<TriggerModelData>> modelDataMap_; 285 std::shared_ptr<IIntellVoiceTriggerConnectorModule> module_ = nullptr; 286 std::vector<TriggerConnectorModuleDesc> moduleDesc_; 287 std::shared_ptr<AudioCapturerSourceChangeCallback> audioCapturerSourceChangeCallback_ = nullptr; 288 std::shared_ptr<AudioRendererStateChangeCallbackImpl> audioRendererStateChangeCallback_ = nullptr; 289 std::shared_ptr<AudioSceneChangeCallback> audioSceneChangeCallback_ = nullptr; 290 bool callActive_ = false; 291 bool systemHibernate_ = false; 292 bool audioCaptureActive_ = false; 293 AudioScene audioScene_ = AudioScene::AUDIO_SCENE_DEFAULT; 294 #ifdef SUPPORT_TELEPHONY_SERVICE 295 bool isTelephonyDetached_ = false; 296 #endif 297 bool isRendererDetached_ = false; 298 #ifdef POWER_MANAGER_ENABLE 299 sptr<HibernateCallback> hibernateCallback_ = nullptr; 300 sptr<SleepCallback> sleepCallback_ = nullptr; 301 bool isHibernateDetached_ = false; 302 #endif 303 bool isSceneDetached_ = false; 304 }; 305 } // namespace IntellVoiceTrigger 306 } // namespace OHOS 307 #endif