/* * Copyright (C) 2022 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include // #define LOG_NDEBUG 0 #define LOG_TAG "AudioEffectAnalyser" #include #include #include #include #include #include #include #include "audio_test_utils.h" #include "pffft.hpp" #include "test_execution_tracer.h" #define CHECK_OK(expr, msg) \ mStatus = (expr); \ if (OK != mStatus) { \ mMsg = (msg); \ return; \ } using namespace android; constexpr float kDefAmplitude = 0.60f; constexpr float kPlayBackDurationSec = 1.5; constexpr float kCaptureDurationSec = 1.0; constexpr float kPrimeDurationInSec = 0.5; // chosen to safely sample largest center freq of eq bands constexpr uint32_t kSamplingFrequency = 48000; // allows no fmt conversion before fft constexpr audio_format_t kFormat = AUDIO_FORMAT_PCM_FLOAT; // playback and capture are done with channel mask configured to mono. // effect analysis should not depend on mask, mono makes it easier. constexpr int kNPointFFT = 16384; constexpr float kBinWidth = (float)kSamplingFrequency / kNPointFFT; // frequency used to generate testing tone constexpr uint32_t kTestFrequency = 1400; // Tolerance of audio gain difference in dB, which is 10^(0.1/20) (around 1.0116%) difference in // amplitude constexpr float kAudioGainDiffTolerancedB = .1f; const std::string kDataTempPath = "/data/local/tmp"; const char* gPackageName = "AudioEffectAnalyser"; static_assert(kPrimeDurationInSec + 2 * kNPointFFT / kSamplingFrequency < kCaptureDurationSec, "capture at least, prime, pad, nPointFft size of samples"); static_assert(kPrimeDurationInSec + 2 * kNPointFFT / kSamplingFrequency < kPlayBackDurationSec, "playback needs to be active during capture"); struct CaptureEnv { // input args uint32_t mSampleRate{kSamplingFrequency}; audio_format_t mFormat{kFormat}; audio_channel_mask_t mChannelMask{AUDIO_CHANNEL_IN_MONO}; float mCaptureDuration{kCaptureDurationSec}; // output val status_t mStatus{OK}; std::string mMsg; std::string mDumpFileName; ~CaptureEnv(); void capture(); }; CaptureEnv::~CaptureEnv() { if (!mDumpFileName.empty()) { std::ifstream f(mDumpFileName); if (f.good()) { f.close(); remove(mDumpFileName.c_str()); } } } void CaptureEnv::capture() { audio_port_v7 port; CHECK_OK(getPortByAttributes(AUDIO_PORT_ROLE_SOURCE, AUDIO_PORT_TYPE_DEVICE, AUDIO_DEVICE_IN_REMOTE_SUBMIX, "0", port), "Could not find port") const auto capture = sp::make(AUDIO_SOURCE_REMOTE_SUBMIX, mSampleRate, mFormat, mChannelMask); CHECK_OK(capture->create(), "record creation failed") CHECK_OK(capture->setRecordDuration(mCaptureDuration), "set record duration failed") CHECK_OK(capture->enableRecordDump(), "enable record dump failed") auto cbCapture = sp::make(); CHECK_OK(capture->getAudioRecordHandle()->addAudioDeviceCallback(cbCapture), "addAudioDeviceCallback failed") CHECK_OK(capture->start(), "start recording failed") CHECK_OK(capture->audioProcess(), "recording process failed") CHECK_OK(cbCapture->waitForAudioDeviceCb(), "audio device callback notification timed out"); DeviceIdVector routedDeviceIds = capture->getAudioRecordHandle()->getRoutedDeviceIds(); if (port.id != routedDeviceIds[0]) { CHECK_OK(BAD_VALUE, "Capture NOT routed on expected port") } CHECK_OK(getPortByAttributes(AUDIO_PORT_ROLE_SINK, AUDIO_PORT_TYPE_DEVICE, AUDIO_DEVICE_OUT_REMOTE_SUBMIX, "0", port), "Could not find port") CHECK_OK(capture->stop(), "record stop failed") mDumpFileName = capture->getRecordDumpFileName(); } struct PlaybackEnv { // input args uint32_t mSampleRate{kSamplingFrequency}; audio_format_t mFormat{kFormat}; audio_channel_mask_t mChannelMask{AUDIO_CHANNEL_OUT_MONO}; audio_session_t mSessionId{AUDIO_SESSION_NONE}; std::string mRes; // output val status_t mStatus{OK}; std::string mMsg; void play(); }; void PlaybackEnv::play() { const auto ap = sp::make(mSampleRate, mFormat, mChannelMask, AUDIO_OUTPUT_FLAG_NONE, mSessionId, AudioTrack::TRANSFER_OBTAIN); CHECK_OK(ap->loadResource(mRes.c_str()), "Unable to open Resource") const auto cbPlayback = sp::make(); CHECK_OK(ap->create(), "track creation failed") ap->getAudioTrackHandle()->setVolume(1.0f); CHECK_OK(ap->getAudioTrackHandle()->addAudioDeviceCallback(cbPlayback), "addAudioDeviceCallback failed") CHECK_OK(ap->start(), "audio track start failed") CHECK_OK(cbPlayback->waitForAudioDeviceCb(), "audio device callback notification timed out") CHECK_OK(ap->onProcess(), "playback process failed") ap->stop(); } void generateMultiTone(const std::vector& toneFrequencies, float samplingFrequency, float duration, float amplitude, float* buffer, int numSamples) { int totalFrameCount = (samplingFrequency * duration); int limit = std::min(totalFrameCount, numSamples); for (auto i = 0; i < limit; i++) { buffer[i] = 0; for (auto j = 0; j < toneFrequencies.size(); j++) { buffer[i] += sin(2 * M_PI * toneFrequencies[j] * i / samplingFrequency); } buffer[i] *= (amplitude / toneFrequencies.size()); } } sp createEffect(const effect_uuid_t* type, audio_session_t sessionId = AUDIO_SESSION_OUTPUT_MIX) { std::string packageName{gPackageName}; AttributionSourceState attributionSource; attributionSource.packageName = packageName; attributionSource.uid = VALUE_OR_FATAL(legacy2aidl_uid_t_int32_t(getuid())); attributionSource.pid = VALUE_OR_FATAL(legacy2aidl_pid_t_int32_t(getpid())); attributionSource.token = sp::make(); sp effect = sp::make(attributionSource); effect->set(type, nullptr, 0, nullptr, sessionId, AUDIO_IO_HANDLE_NONE, {}, false, false); return effect; } void computeFilterGainsAtTones(float captureDuration, int nPointFft, std::vector binOffsets, float* inputMag, float* gaindB, const std::string res, audio_session_t sessionId, const std::string res2 = "", audio_session_t sessionId2 = AUDIO_SESSION_NONE) { int totalFrameCount = captureDuration * kSamplingFrequency; auto output = pffft::AlignedVector(totalFrameCount); auto fftOutput = pffft::AlignedVector(nPointFft); PlaybackEnv argsP, argsP2; argsP.mRes = res; argsP.mSessionId = sessionId; CaptureEnv argsR; argsR.mCaptureDuration = captureDuration; std::thread playbackThread(&PlaybackEnv::play, &argsP); std::optional playbackThread2; if (res2 != "") { argsP2 = {.mSessionId = sessionId2, .mRes = res2}; playbackThread2 = std::thread(&PlaybackEnv::play, &argsP2); } std::thread captureThread(&CaptureEnv::capture, &argsR); captureThread.join(); playbackThread.join(); if (playbackThread2 != std::nullopt) { playbackThread2->join(); } ASSERT_EQ(OK, argsR.mStatus) << argsR.mMsg; ASSERT_EQ(OK, argsP.mStatus) << argsP.mMsg; ASSERT_FALSE(argsR.mDumpFileName.empty()) << "recorded not written to file"; std::ifstream fin(argsR.mDumpFileName, std::ios::in | std::ios::binary); fin.read((char*)output.data(), totalFrameCount * sizeof(output[0])); fin.close(); pffft::detail::PFFFT_Setup* handle = pffft_new_setup(nPointFft, pffft::detail::PFFFT_REAL); // ignore first few samples. This is to not analyse until audio track is re-routed to remote // submix source, also for the effect filter response to reach steady-state (priming / pruning // samples). int rerouteOffset = kPrimeDurationInSec * kSamplingFrequency; pffft_transform_ordered(handle, output.data() + rerouteOffset, fftOutput.data(), nullptr, pffft::detail::PFFFT_FORWARD); pffft_destroy_setup(handle); for (auto i = 0; i < binOffsets.size(); i++) { auto k = binOffsets[i]; auto outputMag = sqrt((fftOutput[k * 2] * fftOutput[k * 2]) + (fftOutput[k * 2 + 1] * fftOutput[k * 2 + 1])); if (inputMag == nullptr) { gaindB[i] = 20 * log10(outputMag); } else { gaindB[i] = 20 * log10(outputMag / inputMag[i]); } } } std::tuple roundToFreqCenteredToFftBin(float binWidth, float freq) { int bin_index = std::round(freq / binWidth); int cfreq = std::round(bin_index * binWidth); return std::make_tuple(bin_index, cfreq); } TEST(AudioEffectTest, CheckEqualizerEffect) { audio_session_t sessionId = (audio_session_t)AudioSystem::newAudioUniqueId(AUDIO_UNIQUE_ID_USE_SESSION); sp equalizer = createEffect(SL_IID_EQUALIZER, sessionId); ASSERT_EQ(OK, equalizer->initCheck()); ASSERT_EQ(NO_ERROR, equalizer->setEnabled(true)); if ((equalizer->descriptor().flags & EFFECT_FLAG_HW_ACC_MASK) != 0) { GTEST_SKIP() << "effect processed output inaccessible, skipping test"; } #define MAX_PARAMS 64 uint32_t buf32[sizeof(effect_param_t) / sizeof(uint32_t) + MAX_PARAMS]; effect_param_t* eqParam = (effect_param_t*)(&buf32); // get num of presets eqParam->psize = sizeof(uint32_t); eqParam->vsize = sizeof(uint16_t); *(int32_t*)eqParam->data = EQ_PARAM_GET_NUM_OF_PRESETS; EXPECT_EQ(0, equalizer->getParameter(eqParam)); EXPECT_EQ(0, eqParam->status); int numPresets = *((uint16_t*)((int32_t*)eqParam->data + 1)); // get num of bands eqParam->psize = sizeof(uint32_t); eqParam->vsize = sizeof(uint16_t); *(int32_t*)eqParam->data = EQ_PARAM_NUM_BANDS; EXPECT_EQ(0, equalizer->getParameter(eqParam)); EXPECT_EQ(0, eqParam->status); int numBands = *((uint16_t*)((int32_t*)eqParam->data + 1)); const int totalFrameCount = kSamplingFrequency * kPlayBackDurationSec; // get band center frequencies std::vector centerFrequencies; std::vector binOffsets; for (auto i = 0; i < numBands; i++) { eqParam->psize = sizeof(uint32_t) * 2; eqParam->vsize = sizeof(uint32_t); *(int32_t*)eqParam->data = EQ_PARAM_CENTER_FREQ; *((uint16_t*)((int32_t*)eqParam->data + 1)) = i; EXPECT_EQ(0, equalizer->getParameter(eqParam)); EXPECT_EQ(0, eqParam->status); float cfreq = *((int32_t*)eqParam->data + 2) / 1000; // milli hz // pick frequency close to bin center frequency auto [bin_index, bin_freq] = roundToFreqCenteredToFftBin(kBinWidth, cfreq); centerFrequencies.push_back(bin_freq); binOffsets.push_back(bin_index); } // input for effect module auto input = pffft::AlignedVector(totalFrameCount); generateMultiTone(centerFrequencies, kSamplingFrequency, kPlayBackDurationSec, kDefAmplitude, input.data(), totalFrameCount); auto fftInput = pffft::AlignedVector(kNPointFFT); pffft::detail::PFFFT_Setup* handle = pffft_new_setup(kNPointFFT, pffft::detail::PFFFT_REAL); pffft_transform_ordered(handle, input.data(), fftInput.data(), nullptr, pffft::detail::PFFFT_FORWARD); pffft_destroy_setup(handle); float inputMag[numBands]; for (auto i = 0; i < numBands; i++) { auto k = binOffsets[i]; inputMag[i] = sqrt((fftInput[k * 2] * fftInput[k * 2]) + (fftInput[k * 2 + 1] * fftInput[k * 2 + 1])); } TemporaryFile tf(kDataTempPath); close(tf.release()); std::ofstream fout(tf.path, std::ios::out | std::ios::binary); fout.write((char*)input.data(), input.size() * sizeof(input[0])); fout.close(); float expGaindB[numBands], actGaindB[numBands]; std::string msg = ""; int numPresetsOk = 0; for (auto preset = 0; preset < numPresets; preset++) { // set preset eqParam->psize = sizeof(uint32_t); eqParam->vsize = sizeof(uint32_t); *(int32_t*)eqParam->data = EQ_PARAM_CUR_PRESET; *((uint16_t*)((int32_t*)eqParam->data + 1)) = preset; EXPECT_EQ(0, equalizer->setParameter(eqParam)); EXPECT_EQ(0, eqParam->status); // get preset gains eqParam->psize = sizeof(uint32_t); eqParam->vsize = (numBands + 1) * sizeof(uint32_t); *(int32_t*)eqParam->data = EQ_PARAM_PROPERTIES; EXPECT_EQ(0, equalizer->getParameter(eqParam)); EXPECT_EQ(0, eqParam->status); t_equalizer_settings* settings = reinterpret_cast((int32_t*)eqParam->data + 1); EXPECT_EQ(preset, settings->curPreset); EXPECT_EQ(numBands, settings->numBands); for (auto i = 0; i < numBands; i++) { expGaindB[i] = ((int16_t)settings->bandLevels[i]) / 100.0f; // gain in milli bels } memset(actGaindB, 0, sizeof(actGaindB)); ASSERT_NO_FATAL_FAILURE(computeFilterGainsAtTones(kCaptureDurationSec, kNPointFFT, binOffsets, inputMag, actGaindB, tf.path, sessionId)); bool isOk = true; for (auto i = 0; i < numBands - 1; i++) { auto diffA = expGaindB[i] - expGaindB[i + 1]; auto diffB = actGaindB[i] - actGaindB[i + 1]; if (diffA == 0 && fabs(diffA - diffB) > 1.0f) { msg += (android::base::StringPrintf( "For eq preset : %d, between bands %d and %d, expected relative gain is : " "%f, got relative gain is : %f, error : %f \n", preset, i, i + 1, diffA, diffB, diffA - diffB)); isOk = false; } else if (diffA * diffB < 0) { msg += (android::base::StringPrintf( "For eq preset : %d, between bands %d and %d, expected relative gain and " "seen relative gain are of opposite signs \n. Expected relative gain is : " "%f, seen relative gain is : %f \n", preset, i, i + 1, diffA, diffB)); isOk = false; } } if (isOk) numPresetsOk++; } EXPECT_EQ(numPresetsOk, numPresets) << msg; } TEST(AudioEffectTest, CheckBassBoostEffect) { audio_session_t sessionId = (audio_session_t)AudioSystem::newAudioUniqueId(AUDIO_UNIQUE_ID_USE_SESSION); sp bassboost = createEffect(SL_IID_BASSBOOST, sessionId); ASSERT_EQ(OK, bassboost->initCheck()); ASSERT_EQ(NO_ERROR, bassboost->setEnabled(true)); if ((bassboost->descriptor().flags & EFFECT_FLAG_HW_ACC_MASK) != 0) { GTEST_SKIP() << "effect processed output inaccessible, skipping test"; } int32_t buf32[sizeof(effect_param_t) / sizeof(int32_t) + MAX_PARAMS]; effect_param_t* bbParam = (effect_param_t*)(&buf32); bbParam->psize = sizeof(int32_t); bbParam->vsize = sizeof(int32_t); *(int32_t*)bbParam->data = BASSBOOST_PARAM_STRENGTH_SUPPORTED; EXPECT_EQ(0, bassboost->getParameter(bbParam)); EXPECT_EQ(0, bbParam->status); bool strengthSupported = *((int32_t*)bbParam->data + 1); const int totalFrameCount = kSamplingFrequency * kPlayBackDurationSec; // selecting bass frequency, speech tone (for relative gain) std::vector testFrequencies{100, 1200}; std::vector binOffsets; for (auto i = 0; i < testFrequencies.size(); i++) { // pick frequency close to bin center frequency auto [bin_index, bin_freq] = roundToFreqCenteredToFftBin(kBinWidth, testFrequencies[i]); testFrequencies[i] = bin_freq; binOffsets.push_back(bin_index); } // input for effect module auto input = pffft::AlignedVector(totalFrameCount); generateMultiTone(testFrequencies, kSamplingFrequency, kPlayBackDurationSec, kDefAmplitude, input.data(), totalFrameCount); auto fftInput = pffft::AlignedVector(kNPointFFT); pffft::detail::PFFFT_Setup* handle = pffft_new_setup(kNPointFFT, pffft::detail::PFFFT_REAL); pffft_transform_ordered(handle, input.data(), fftInput.data(), nullptr, pffft::detail::PFFFT_FORWARD); pffft_destroy_setup(handle); float inputMag[testFrequencies.size()]; for (auto i = 0; i < testFrequencies.size(); i++) { auto k = binOffsets[i]; inputMag[i] = sqrt((fftInput[k * 2] * fftInput[k * 2]) + (fftInput[k * 2 + 1] * fftInput[k * 2 + 1])); } TemporaryFile tf(kDataTempPath); close(tf.release()); std::ofstream fout(tf.path, std::ios::out | std::ios::binary); fout.write((char*)input.data(), input.size() * sizeof(input[0])); fout.close(); float gainWithOutFilter[testFrequencies.size()]; memset(gainWithOutFilter, 0, sizeof(gainWithOutFilter)); ASSERT_NO_FATAL_FAILURE(computeFilterGainsAtTones(kCaptureDurationSec, kNPointFFT, binOffsets, inputMag, gainWithOutFilter, tf.path, AUDIO_SESSION_NONE)); float diffA = gainWithOutFilter[0] - gainWithOutFilter[1]; float prevGain = -100.f; for (auto strength = 150; strength < 1000; strength += strengthSupported ? 150 : 1000) { // configure filter strength if (strengthSupported) { bbParam->psize = sizeof(int32_t); bbParam->vsize = sizeof(int16_t); *(int32_t*)bbParam->data = BASSBOOST_PARAM_STRENGTH; *((int16_t*)((int32_t*)bbParam->data + 1)) = strength; EXPECT_EQ(0, bassboost->setParameter(bbParam)); EXPECT_EQ(0, bbParam->status); } float gainWithFilter[testFrequencies.size()]; memset(gainWithFilter, 0, sizeof(gainWithFilter)); ASSERT_NO_FATAL_FAILURE(computeFilterGainsAtTones(kCaptureDurationSec, kNPointFFT, binOffsets, inputMag, gainWithFilter, tf.path, sessionId)); float diffB = gainWithFilter[0] - gainWithFilter[1]; EXPECT_GT(diffB, diffA) << "bassboost effect not seen"; EXPECT_GE(diffB, prevGain) << "increase in boost strength causing fall in gain"; prevGain = diffB; } } // assert the silent audio session with effect does not override the output audio TEST(AudioEffectTest, SilentAudioEffectSessionNotOverrideOutput) { audio_session_t sessionId = (audio_session_t)AudioSystem::newAudioUniqueId(AUDIO_UNIQUE_ID_USE_SESSION); sp bassboost = createEffect(SL_IID_BASSBOOST, sessionId); if ((bassboost->descriptor().flags & EFFECT_FLAG_HW_ACC_MASK) != 0) { GTEST_SKIP() << "effect processed output inaccessible, skipping test"; } ASSERT_EQ(OK, bassboost->initCheck()); ASSERT_EQ(NO_ERROR, bassboost->setEnabled(true)); const auto bin = roundToFreqCenteredToFftBin(kBinWidth, kTestFrequency); const int binIndex = std::get<0 /* index */>(bin); const int binFrequency = std::get<1 /* freq */>(bin); const int totalFrameCount = kSamplingFrequency * kPlayBackDurationSec; // input for effect module auto silentAudio = pffft::AlignedVector(totalFrameCount); auto input = pffft::AlignedVector(totalFrameCount); generateMultiTone({binFrequency}, kSamplingFrequency, kPlayBackDurationSec, kDefAmplitude, input.data(), totalFrameCount); TemporaryFile tf(kDataTempPath); close(tf.release()); std::ofstream fout(tf.path, std::ios::out | std::ios::binary); fout.write((char*)input.data(), input.size() * sizeof(input[0])); fout.close(); // play non-silent audio file on AUDIO_SESSION_NONE float audioGain, audioPlusSilentEffectGain; ASSERT_NO_FATAL_FAILURE(computeFilterGainsAtTones(kCaptureDurationSec, kNPointFFT, {binIndex}, nullptr, &audioGain, tf.path, AUDIO_SESSION_NONE)); EXPECT_FALSE(std::isinf(audioGain)) << "output gain should not be -inf"; TemporaryFile silentFile(kDataTempPath); close(silentFile.release()); std::ofstream fSilent(silentFile.path, std::ios::out | std::ios::binary); fSilent.write((char*)silentAudio.data(), silentAudio.size() * sizeof(silentAudio[0])); fSilent.close(); // play non-silent audio file on AUDIO_SESSION_NONE and silent audio on sessionId, expect // the new output gain to be almost same as last playback ASSERT_NO_FATAL_FAILURE(computeFilterGainsAtTones( kCaptureDurationSec, kNPointFFT, {binIndex}, nullptr, &audioPlusSilentEffectGain, tf.path, AUDIO_SESSION_NONE, silentFile.path, sessionId)); EXPECT_FALSE(std::isinf(audioPlusSilentEffectGain)) << "output might have been overwritten in effect accumulate mode"; EXPECT_NEAR(audioGain, audioPlusSilentEffectGain, kAudioGainDiffTolerancedB) << " output gain should almost same with one more silent audio stream"; } int main(int argc, char** argv) { android::ProcessState::self()->startThreadPool(); ::testing::InitGoogleTest(&argc, argv); ::testing::UnitTest::GetInstance()->listeners().Append(new TestExecutionTracer()); return RUN_ALL_TESTS(); }