1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_MEDIA_SPATIALIZER_H 18 #define ANDROID_MEDIA_SPATIALIZER_H 19 20 #include <android-base/stringprintf.h> 21 #include <android/media/BnEffect.h> 22 #include <android/media/BnSpatializer.h> 23 #include <android/media/SpatializationLevel.h> 24 #include <android/media/SpatializationMode.h> 25 #include <android/media/SpatializerHeadTrackingMode.h> 26 #include <android/media/audio/common/AudioLatencyMode.h> 27 #include <audio_utils/SimpleLog.h> 28 #include <math.h> 29 #include <media/AudioEffect.h> 30 #include <media/audiohal/EffectsFactoryHalInterface.h> 31 #include <media/VectorRecorder.h> 32 #include <media/audiohal/EffectHalInterface.h> 33 #include <media/stagefright/foundation/ALooper.h> 34 #include <system/audio_effects/effect_spatializer.h> 35 #include <string> 36 37 #include "SpatializerPoseController.h" 38 39 namespace android { 40 41 42 // ---------------------------------------------------------------------------- 43 44 /** 45 * A callback interface from the Spatializer object or its parent AudioPolicyService. 46 * This is implemented by the audio policy service hosting the Spatializer to perform 47 * actions needed when a state change inside the Spatializer requires some audio system 48 * changes that cannot be performed by the Spatializer. For instance opening or closing a 49 * spatializer output stream when the spatializer is enabled or disabled 50 */ 51 class SpatializerPolicyCallback { 52 public: 53 /** Called when a stage change occurs that requires the parent audio policy service to take 54 * some action. 55 */ 56 virtual void onCheckSpatializer() = 0; 57 58 virtual ~SpatializerPolicyCallback() = default; 59 }; 60 /** 61 * The Spatializer class implements all functional controlling the multichannel spatializer 62 * with head tracking implementation in the native audio service: audio policy and audio flinger. 63 * It presents an AIDL interface available to the java audio service to discover the availability 64 * of the feature and options, control its state and register an active head tracking sensor. 65 * It maintains the current state of the platform spatializer and applies the stored parameters 66 * when the spatializer engine is created and enabled. 67 * Based on the requested spatializer level, it will request the creation of a specialized output 68 * mixer to the audio policy service which will in turn notify the Spatializer of the output 69 * stream on which a spatializer engine should be created, configured and enabled. 70 * The spatializer also hosts the head tracking management logic. This logic receives the 71 * desired head tracking mode and selected head tracking sensor, registers a sensor event listener 72 * and derives the compounded head pose information to the spatializer engine. 73 * 74 * Workflow: 75 * - Initialization: when the audio policy service starts, it checks if a spatializer effect 76 * engine exists and if the audio policy manager reports a dedicated spatializer output profile. 77 * If both conditions are met, a Spatializer object is created 78 * - Capabilities discovery: AudioService will call AudioSystem::canBeSpatialized() and if true, 79 * acquire an ISpatializer interface with AudioSystem::getSpatializer(). This interface 80 * will be used to query the implementation capabilities and configure the spatializer. 81 * - Enabling: when ISpatializer::setLevel() sets a level different from NONE the spatializer 82 * is considered enabled. The audio policy callback onCheckSpatializer() is called. This 83 * triggers a request to audio policy manager to open a spatialization output stream and a 84 * spatializer mixer is created in audio flinger. When an output is returned by audio policy 85 * manager, Spatializer::attachOutput() is called which creates and enables the spatializer 86 * stage engine on the specified output. 87 * - Disabling: when the spatialization level is set to NONE, the spatializer is considered 88 * disabled. The audio policy callback onCheckSpatializer() is called. This triggers a call 89 * to Spatializer::detachOutput() and the spatializer engine is released. Then a request is 90 * made to audio policy manager to release and close the spatializer output stream and the 91 * spatializer mixer thread is destroyed. 92 */ 93 class Spatializer : public media::BnSpatializer, 94 public AudioEffect::IAudioEffectCallback, 95 public IBinder::DeathRecipient, 96 private SpatializerPoseController::Listener, 97 public virtual AudioSystem::SupportedLatencyModesCallback { 98 public: 99 static sp<Spatializer> create(SpatializerPolicyCallback* callback, 100 const sp<EffectsFactoryHalInterface>& effectsFactoryHal); 101 102 ~Spatializer() override; 103 104 /** RefBase */ 105 void onFirstRef(); 106 107 /** ISpatializer, see ISpatializer.aidl */ 108 binder::Status release() override; 109 binder::Status getSupportedLevels(std::vector<media::SpatializationLevel>* levels) override; 110 binder::Status setLevel(media::SpatializationLevel level) override; 111 binder::Status getLevel(media::SpatializationLevel *level) override; 112 binder::Status isHeadTrackingSupported(bool *supports); 113 binder::Status getSupportedHeadTrackingModes( 114 std::vector<media::SpatializerHeadTrackingMode>* modes) override; 115 binder::Status setDesiredHeadTrackingMode( 116 media::SpatializerHeadTrackingMode mode) override; 117 binder::Status getActualHeadTrackingMode( 118 media::SpatializerHeadTrackingMode* mode) override; 119 binder::Status recenterHeadTracker() override; 120 binder::Status setGlobalTransform(const std::vector<float>& screenToStage) override; 121 binder::Status setHeadSensor(int sensorHandle) override; 122 binder::Status setScreenSensor(int sensorHandle) override; 123 binder::Status setDisplayOrientation(float physicalToLogicalAngle) override; 124 binder::Status setHingeAngle(float hingeAngle) override; 125 binder::Status setFoldState(bool folded) override; 126 binder::Status getSupportedModes(std::vector<media::SpatializationMode>* modes) override; 127 binder::Status registerHeadTrackingCallback( 128 const sp<media::ISpatializerHeadTrackingCallback>& callback) override; 129 binder::Status setParameter(int key, const std::vector<unsigned char>& value) override; 130 binder::Status getParameter(int key, std::vector<unsigned char> *value) override; 131 binder::Status getOutput(int *output); 132 133 /** IBinder::DeathRecipient. Listen to the death of the INativeSpatializerCallback. */ 134 virtual void binderDied(const wp<IBinder>& who); 135 136 /** SupportedLatencyModesCallback */ 137 void onSupportedLatencyModesChanged( 138 audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) override; 139 140 /** Registers a INativeSpatializerCallback when a client is attached to this Spatializer 141 * by audio policy service. 142 */ 143 status_t registerCallback(const sp<media::INativeSpatializerCallback>& callback); 144 145 status_t loadEngineConfiguration(sp<EffectHalInterface> effect); 146 147 /** Level getter for use by local classes. */ getLevel()148 media::SpatializationLevel getLevel() const { std::lock_guard lock(mLock); return mLevel; } 149 150 /** Called by audio policy service when the special output mixer dedicated to spatialization 151 * is opened and the spatializer engine must be created. 152 */ 153 status_t attachOutput(audio_io_handle_t output, size_t numActiveTracks); 154 /** Called by audio policy service when the special output mixer dedicated to spatialization 155 * is closed and the spatializer engine must be release. 156 */ 157 audio_io_handle_t detachOutput(); 158 /** Returns the output stream the spatializer is attached to. */ getOutput()159 audio_io_handle_t getOutput() const { std::lock_guard lock(mLock); return mOutput; } 160 161 void updateActiveTracks(size_t numActiveTracks); 162 163 /** Gets the channel mask, sampling rate and format set for the spatializer input. */ 164 audio_config_base_t getAudioInConfig() const; 165 166 void calculateHeadPose(); 167 168 /** Convert fields in Spatializer and sub-modules to a string. Disable thread-safety-analysis 169 * here because we want to dump mutex guarded members even try_lock failed to provide as much 170 * information as possible for debugging purpose. */ 171 std::string toString(unsigned level) const NO_THREAD_SAFETY_ANALYSIS; 172 toString(audio_latency_mode_t mode)173 static std::string toString(audio_latency_mode_t mode) { 174 // We convert to the AIDL type to print (eventually the legacy type will be removed). 175 const auto result = legacy2aidl_audio_latency_mode_t_AudioLatencyMode(mode); 176 return result.has_value() ? 177 media::audio::common::toString(*result) : "unknown_latency_mode"; 178 } 179 180 // If the Spatializer is not created, we send the status for metrics purposes. 181 // OK: Spatializer not expected to be created. 182 // NO_INIT: Spatializer creation failed. 183 static void sendEmptyCreateSpatializerMetricWithStatus(status_t status); 184 185 private: 186 Spatializer(effect_descriptor_t engineDescriptor, 187 SpatializerPolicyCallback *callback); 188 189 static void engineCallback(int32_t event, void* user, void *info); 190 191 // From VirtualizerStageController::Listener 192 void onHeadToStagePose(const media::Pose3f& headToStage) override; 193 void onActualModeChange(media::HeadTrackingMode mode) override; 194 195 void onHeadToStagePoseMsg(const std::vector<float>& headToStage); 196 void onActualModeChangeMsg(media::HeadTrackingMode mode); 197 void onSupportedLatencyModesChangedMsg( 198 audio_io_handle_t output, std::vector<audio_latency_mode_t>&& modes); 199 200 static constexpr int kMaxEffectParamValues = 10; 201 /** 202 * Get a parameter from spatializer engine by calling the effect HAL command method directly. 203 * To be used when the engine instance mEngine is not yet created in the effect framework. 204 * When MULTI_VALUES is false, the expected reply is only one value of type T. 205 * When MULTI_VALUES is true, the expected reply is made of a number (of type T) indicating 206 * how many values are returned, followed by this number for values of type T. 207 */ 208 template<bool MULTI_VALUES, typename T> getHalParameter(sp<EffectHalInterface> effect,uint32_t type,std::vector<T> * values)209 status_t getHalParameter(sp<EffectHalInterface> effect, uint32_t type, 210 std::vector<T> *values) { 211 static_assert(sizeof(T) <= sizeof(uint32_t), "The size of T must less than 32 bits"); 212 213 uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 1]; 214 uint32_t reply[sizeof(effect_param_t) / sizeof(uint32_t) + 2 + kMaxEffectParamValues]; 215 216 effect_param_t *p = (effect_param_t *)cmd; 217 p->psize = sizeof(uint32_t); 218 if (MULTI_VALUES) { 219 p->vsize = (kMaxEffectParamValues + 1) * sizeof(T); 220 } else { 221 p->vsize = sizeof(T); 222 } 223 *(uint32_t *)p->data = type; 224 uint32_t replySize = sizeof(effect_param_t) + p->psize + p->vsize; 225 226 status_t status = effect->command(EFFECT_CMD_GET_PARAM, 227 sizeof(effect_param_t) + sizeof(uint32_t), cmd, 228 &replySize, reply); 229 if (status != NO_ERROR) { 230 return status; 231 } 232 if (p->status != NO_ERROR) { 233 return p->status; 234 } 235 if (replySize < 236 sizeof(effect_param_t) + sizeof(uint32_t) + (MULTI_VALUES ? 2 : 1) * sizeof(T)) { 237 return BAD_VALUE; 238 } 239 240 T *params = (T *)((uint8_t *)reply + sizeof(effect_param_t) + sizeof(uint32_t)); 241 int numParams = 1; 242 if (MULTI_VALUES) { 243 numParams = (int)*params++; 244 } 245 if (numParams > kMaxEffectParamValues) { 246 return BAD_VALUE; 247 } 248 (*values).clear(); 249 std::copy(¶ms[0], ¶ms[numParams], back_inserter(*values)); 250 return NO_ERROR; 251 } 252 253 /** 254 * Set a parameter to spatializer engine by calling setParameter on mEngine AudioEffect object. 255 * It is possible to pass more than one value of type T according to the parameter type 256 * according to values vector size. 257 */ 258 template<typename T> setEffectParameter_l(uint32_t type,const std::vector<T> & values)259 status_t setEffectParameter_l(uint32_t type, const std::vector<T>& values) REQUIRES(mLock) { 260 static_assert(sizeof(T) <= sizeof(uint32_t), "The size of T must less than 32 bits"); 261 262 uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 1 + values.size()]; 263 effect_param_t *p = (effect_param_t *)cmd; 264 p->psize = sizeof(uint32_t); 265 p->vsize = sizeof(T) * values.size(); 266 *(uint32_t *)p->data = type; 267 memcpy((uint32_t *)p->data + 1, values.data(), sizeof(T) * values.size()); 268 269 status_t status = mEngine->setParameter(p); 270 if (status != NO_ERROR) { 271 return status; 272 } 273 if (p->status != NO_ERROR) { 274 return p->status; 275 } 276 return NO_ERROR; 277 } 278 279 /** 280 * Get a parameter from spatializer engine by calling getParameter on AudioEffect object. 281 * It is possible to read more than one value of type T according to the parameter type 282 * by specifying values vector size. 283 */ 284 template<typename T> getEffectParameter_l(uint32_t type,std::vector<T> * values)285 status_t getEffectParameter_l(uint32_t type, std::vector<T> *values) REQUIRES(mLock) { 286 static_assert(sizeof(T) <= sizeof(uint32_t), "The size of T must less than 32 bits"); 287 288 uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 1 + values->size()]; 289 effect_param_t *p = (effect_param_t *)cmd; 290 p->psize = sizeof(uint32_t); 291 p->vsize = sizeof(T) * values->size(); 292 *(uint32_t *)p->data = type; 293 294 status_t status = mEngine->getParameter(p); 295 296 if (status != NO_ERROR) { 297 return status; 298 } 299 if (p->status != NO_ERROR) { 300 return p->status; 301 } 302 303 int numValues = std::min(p->vsize / sizeof(T), values->size()); 304 (*values).clear(); 305 T *retValues = (T *)((uint8_t *)p->data + sizeof(uint32_t)); 306 std::copy(&retValues[0], &retValues[numValues], back_inserter(*values)); 307 308 return NO_ERROR; 309 } 310 311 virtual void onFramesProcessed(int32_t framesProcessed) override; 312 313 /** 314 * Checks if head and screen sensors must be actively monitored based on 315 * spatializer state and playback activity and configures the pose controller 316 * accordingly. 317 */ 318 void checkSensorsState_l() REQUIRES(mLock); 319 320 /** 321 * Checks if the head pose controller should be created or destroyed according 322 * to desired head tracking mode. 323 */ 324 void checkPoseController_l() REQUIRES(mLock); 325 326 /** 327 * Checks if the spatializer effect should be enabled based on 328 * playback activity and requested level. 329 */ 330 void checkEngineState_l() REQUIRES(mLock); 331 332 /** 333 * Reset head tracking mode and recenter pose in engine: Called when the head tracking 334 * is disabled. 335 */ 336 void resetEngineHeadPose_l() REQUIRES(mLock); 337 338 /** Effect engine descriptor */ 339 const effect_descriptor_t mEngineDescriptor; 340 /** Callback interface to parent audio policy service */ 341 SpatializerPolicyCallback* const mPolicyCallback; 342 343 /** Currently there is only one version of the spatializer running */ 344 static constexpr const char* kDefaultMetricsId = 345 AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER "0"; 346 const std::string mMetricsId = kDefaultMetricsId; 347 348 /** Mutex protecting internal state */ 349 mutable std::mutex mLock; 350 351 /** Client AudioEffect for the engine */ 352 sp<AudioEffect> mEngine GUARDED_BY(mLock); 353 /** Output stream the spatializer mixer thread is attached to */ 354 audio_io_handle_t mOutput GUARDED_BY(mLock) = AUDIO_IO_HANDLE_NONE; 355 356 /** Callback interface to the client (AudioService) controlling this`Spatializer */ 357 sp<media::INativeSpatializerCallback> mSpatializerCallback GUARDED_BY(mLock); 358 359 /** Callback interface for head tracking */ 360 sp<media::ISpatializerHeadTrackingCallback> mHeadTrackingCallback GUARDED_BY(mLock); 361 362 /** Requested spatialization level */ 363 media::SpatializationLevel mLevel GUARDED_BY(mLock) = media::SpatializationLevel::NONE; 364 365 /** Control logic for head-tracking, etc. */ 366 std::shared_ptr<SpatializerPoseController> mPoseController GUARDED_BY(mLock); 367 368 /** Last requested head tracking mode */ 369 media::HeadTrackingMode mDesiredHeadTrackingMode GUARDED_BY(mLock) 370 = media::HeadTrackingMode::STATIC; 371 372 /** Last-reported actual head-tracking mode. */ 373 media::SpatializerHeadTrackingMode mActualHeadTrackingMode GUARDED_BY(mLock) 374 = media::SpatializerHeadTrackingMode::DISABLED; 375 376 /** Selected Head pose sensor */ 377 int32_t mHeadSensor GUARDED_BY(mLock) = SpatializerPoseController::INVALID_SENSOR; 378 379 /** Selected Screen pose sensor */ 380 int32_t mScreenSensor GUARDED_BY(mLock) = SpatializerPoseController::INVALID_SENSOR; 381 382 /** Last display orientation received */ 383 float mDisplayOrientation GUARDED_BY(mLock) = 0.f; // aligned to natural up orientation. 384 385 /** Last folded state */ 386 bool mFoldedState GUARDED_BY(mLock) = false; // foldable: true means folded. 387 388 /** Last hinge angle */ 389 float mHingeAngle GUARDED_BY(mLock) = 0.f; // foldable: 0.f is closed, M_PI flat open. 390 391 std::vector<media::SpatializationLevel> mLevels; 392 std::vector<media::SpatializerHeadTrackingMode> mHeadTrackingModes; 393 std::vector<media::SpatializationMode> mSpatializationModes; 394 std::vector<audio_channel_mask_t> mChannelMasks; 395 bool mSupportsHeadTracking; 396 397 // Looper thread for mEngine callbacks 398 class EngineCallbackHandler; 399 400 sp<ALooper> mLooper; 401 sp<EngineCallbackHandler> mHandler; 402 403 size_t mNumActiveTracks GUARDED_BY(mLock) = 0; 404 std::vector<audio_latency_mode_t> mSupportedLatencyModes GUARDED_BY(mLock); 405 406 static const std::vector<const char*> sHeadPoseKeys; 407 408 // Local log for command messages. 409 static constexpr int mMaxLocalLogLine = 10; 410 SimpleLog mLocalLog{mMaxLocalLogLine}; 411 412 /** 413 * @brief Calculate and record sensor data. 414 * Dump to local log with max/average pose angle every mPoseRecordThreshold. 415 */ 416 // Record one log line per second (up to mMaxLocalLogLine) to capture most recent sensor data. GUARDED_BY(mLock)417 media::VectorRecorder mPoseRecorder GUARDED_BY(mLock) { 418 6 /* vectorSize */, std::chrono::seconds(1), mMaxLocalLogLine, { 3 } /* delimiterIdx */}; 419 // Record one log line per minute (up to mMaxLocalLogLine) to capture durable sensor data. GUARDED_BY(mLock)420 media::VectorRecorder mPoseDurableRecorder GUARDED_BY(mLock) { 421 6 /* vectorSize */, std::chrono::minutes(1), mMaxLocalLogLine, { 3 } /* delimiterIdx */}; 422 }; // Spatializer 423 424 }; // namespace android 425 426 #endif // ANDROID_MEDIA_SPATIALIZER_H 427