1 /**
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <stdio.h>
18 #include <sys/time.h>
19 // #include <sys/timeb.h>
20 #include <fcntl.h>
21 #include <string.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 #include <thread>
25
26 #include <ImsMediaDefine.h>
27 #include <ImsMediaTrace.h>
28 #include <ImsMediaTimer.h>
29 #include <ImsMediaAudioUtil.h>
30 #include <ImsMediaAudioPlayer.h>
31 #include <utils/Errors.h>
32
33 #define AAUDIO_STATE_TIMEOUT_NANO (100 * 1000000L)
34 #define DEFAULT_SAMPLING_RATE (8000)
35 #define CODEC_TIMEOUT_NANO (100000)
36
37 using namespace android;
38
ImsMediaAudioPlayer()39 ImsMediaAudioPlayer::ImsMediaAudioPlayer()
40 {
41 mAudioStream = nullptr;
42 mCodec = nullptr;
43 mFormat = nullptr;
44 mCodecType = 0;
45 mCodecMode = 0;
46 mSamplingRate = DEFAULT_SAMPLING_RATE;
47 mEvsChAwOffset = 0;
48 mEvsBandwidth = kEvsBandwidthNone;
49 memset(mBuffer, 0, sizeof(mBuffer));
50 mEvsBitRate = 0;
51 mEvsCodecHeaderMode = kRtpPayloadHeaderModeEvsHeaderFull;
52 mIsFirstFrame = false;
53 mIsEvsInitialized = false;
54 mIsOctetAligned = false;
55 mIsDtxEnabled = false;
56 }
57
~ImsMediaAudioPlayer()58 ImsMediaAudioPlayer::~ImsMediaAudioPlayer() {}
59
SetCodec(int32_t type)60 void ImsMediaAudioPlayer::SetCodec(int32_t type)
61 {
62 IMLOGD_PACKET1(IM_PACKET_LOG_AUDIO, "[SetCodec] type[%d]", type);
63 mCodecType = type;
64 }
65
SetEvsBitRate(int32_t bitRate)66 void ImsMediaAudioPlayer::SetEvsBitRate(int32_t bitRate)
67 {
68 mEvsBitRate = bitRate;
69 IMLOGD_PACKET1(IM_PACKET_LOG_AUDIO, "[SetEvsBitRate] mEvsBitRate[%d]", mEvsBitRate);
70 }
71
SetEvsChAwOffset(int32_t offset)72 void ImsMediaAudioPlayer::SetEvsChAwOffset(int32_t offset)
73 {
74 mEvsChAwOffset = offset;
75 }
76
SetSamplingRate(int32_t samplingRate)77 void ImsMediaAudioPlayer::SetSamplingRate(int32_t samplingRate)
78 {
79 mSamplingRate = samplingRate;
80 }
81
SetEvsBandwidth(int32_t evsBandwidth)82 void ImsMediaAudioPlayer::SetEvsBandwidth(int32_t evsBandwidth)
83 {
84 mEvsBandwidth = (kEvsBandwidth)evsBandwidth;
85 }
86
SetEvsPayloadHeaderMode(int32_t EvsPayloadHeaderMode)87 void ImsMediaAudioPlayer::SetEvsPayloadHeaderMode(int32_t EvsPayloadHeaderMode)
88 {
89 mEvsCodecHeaderMode = (kRtpPayloadHeaderMode)EvsPayloadHeaderMode;
90 }
91
SetCodecMode(uint32_t mode)92 void ImsMediaAudioPlayer::SetCodecMode(uint32_t mode)
93 {
94 IMLOGD1("[SetCodecMode] mode[%d]", mode);
95 mCodecMode = mode;
96 }
97
SetDtxEnabled(bool isDtxEnabled)98 void ImsMediaAudioPlayer::SetDtxEnabled(bool isDtxEnabled)
99 {
100 mIsDtxEnabled = isDtxEnabled;
101 }
102
SetOctetAligned(bool isOctetAligned)103 void ImsMediaAudioPlayer::SetOctetAligned(bool isOctetAligned)
104 {
105 mIsOctetAligned = isOctetAligned;
106 }
107
ProcessCmr(const uint32_t cmr)108 void ImsMediaAudioPlayer::ProcessCmr(const uint32_t cmr)
109 {
110 IMLOGD1("[ProcessCmr] cmr[%d]", cmr);
111
112 mCodecMode = cmr;
113 Stop();
114 Start();
115 }
116
Start()117 bool ImsMediaAudioPlayer::Start()
118 {
119 char kMimeType[128] = {'\0'};
120
121 if (mCodecType == kAudioCodecAmr)
122 {
123 sprintf(kMimeType, "audio/3gpp");
124 }
125 else if (mCodecType == kAudioCodecAmrWb)
126 {
127 sprintf(kMimeType, "audio/amr-wb");
128 }
129 else if (mCodecType == kAudioCodecEvs)
130 {
131 // TODO: Integration with libEVS is required.
132 sprintf(kMimeType, "audio/evs");
133 }
134 else
135 {
136 return false;
137 }
138
139 openAudioStream();
140
141 if (mAudioStream == nullptr)
142 {
143 IMLOGE0("[Start] create audio stream failed");
144 return false;
145 }
146
147 IMLOGD1("[Start] Creating codec[%s]", kMimeType);
148
149 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
150 {
151 mFormat = AMediaFormat_new();
152 AMediaFormat_setString(mFormat, AMEDIAFORMAT_KEY_MIME, kMimeType);
153 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_SAMPLE_RATE, mSamplingRate);
154 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_CHANNEL_COUNT, 1);
155
156 mCodec = AMediaCodec_createDecoderByType(kMimeType);
157
158 if (mCodec == nullptr)
159 {
160 IMLOGE1("[Start] unable to create %s codec instance", kMimeType);
161 AMediaFormat_delete(mFormat);
162 mFormat = nullptr;
163 return false;
164 }
165
166 IMLOGD0("[Start] configure codec");
167 media_status_t codecResult = AMediaCodec_configure(mCodec, mFormat, nullptr, nullptr, 0);
168
169 if (codecResult != AMEDIA_OK)
170 {
171 IMLOGE2("[Start] unable to configure[%s] codec - err[%d]", kMimeType, codecResult);
172 AMediaCodec_delete(mCodec);
173 mCodec = nullptr;
174 AMediaFormat_delete(mFormat);
175 mFormat = nullptr;
176 return false;
177 }
178 }
179
180 aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STARTING;
181 aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
182 auto result = AAudioStream_requestStart(mAudioStream);
183
184 if (result != AAUDIO_OK)
185 {
186 IMLOGE1("[Start] Error start stream[%s]", AAudio_convertResultToText(result));
187 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
188 {
189 AMediaCodec_delete(mCodec);
190 mCodec = nullptr;
191 AMediaFormat_delete(mFormat);
192 mFormat = nullptr;
193 }
194 return false;
195 }
196
197 result = AAudioStream_waitForStateChange(
198 mAudioStream, inputState, &nextState, AAUDIO_STATE_TIMEOUT_NANO);
199
200 if (result != AAUDIO_OK)
201 {
202 IMLOGE1("[Start] Error start stream[%s]", AAudio_convertResultToText(result));
203 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
204 {
205 AMediaCodec_delete(mCodec);
206 mCodec = nullptr;
207 AMediaFormat_delete(mFormat);
208 mFormat = nullptr;
209 }
210 return false;
211 }
212
213 IMLOGI1("[Start] start stream state[%s]", AAudio_convertStreamStateToText(nextState));
214
215 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
216 {
217 media_status_t codecResult = AMediaCodec_start(mCodec);
218 if (codecResult != AMEDIA_OK)
219 {
220 IMLOGE1("[Start] unable to start codec - err[%d]", codecResult);
221 AMediaCodec_delete(mCodec);
222 mCodec = nullptr;
223 AMediaFormat_delete(mFormat);
224 mFormat = nullptr;
225 return false;
226 }
227 }
228
229 IMLOGD0("[Start] exit");
230 return true;
231 }
232
Stop()233 void ImsMediaAudioPlayer::Stop()
234 {
235 IMLOGD0("[Stop] enter");
236 std::lock_guard<std::mutex> guard(mMutex);
237 if ((mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb) && (mCodec != nullptr))
238 {
239 AMediaCodec_stop(mCodec);
240 AMediaCodec_delete(mCodec);
241 mCodec = nullptr;
242 }
243
244 if ((mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb) && (mFormat != nullptr))
245 {
246 AMediaFormat_delete(mFormat);
247 mFormat = nullptr;
248 }
249
250 if (mAudioStream != nullptr)
251 {
252 aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STOPPING;
253 aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
254 aaudio_result_t result = AAudioStream_requestStop(mAudioStream);
255
256 if (result != AAUDIO_OK)
257 {
258 IMLOGE1("[Stop] Error stop stream[%s]", AAudio_convertResultToText(result));
259 }
260
261 // TODO: if it causes extra delay in stop, optimize later
262 result = AAudioStream_waitForStateChange(
263 mAudioStream, inputState, &nextState, AAUDIO_STATE_TIMEOUT_NANO);
264
265 if (result != AAUDIO_OK)
266 {
267 IMLOGE1("[Stop] Error stop stream[%s]", AAudio_convertResultToText(result));
268 }
269
270 IMLOGI1("[Stop] stream state[%s]", AAudio_convertStreamStateToText(nextState));
271
272 AAudioStream_close(mAudioStream);
273 mAudioStream = nullptr;
274 }
275
276 IMLOGD0("[Stop] exit ");
277 }
278
onDataFrame(uint8_t * buffer,uint32_t size,FrameType,bool,uint8_t)279 bool ImsMediaAudioPlayer::onDataFrame(uint8_t* buffer, uint32_t size, FrameType /*frameType*/,
280 bool /*hasNextFrame*/, uint8_t /*nextFrameByte*/)
281 {
282 std::lock_guard<std::mutex> guard(mMutex);
283
284 if (size == 0 || buffer == nullptr || mAudioStream == nullptr ||
285 AAudioStream_getState(mAudioStream) != AAUDIO_STREAM_STATE_STARTED)
286 {
287 return false;
288 }
289
290 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
291 {
292 if (mCodec == nullptr)
293 {
294 return false;
295 }
296 else
297 {
298 return decodeAmr(buffer, size);
299 }
300 }
301 else if (mCodecType == kAudioCodecEvs)
302 {
303 // TODO:Integration with libEVS is required.
304 return decodeEvs(buffer, size);
305 }
306 return false;
307 }
308
decodeAmr(uint8_t * buffer,uint32_t size)309 bool ImsMediaAudioPlayer::decodeAmr(uint8_t* buffer, uint32_t size)
310 {
311 auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO);
312
313 if (index >= 0)
314 {
315 size_t bufferSize = 0;
316 uint8_t* inputBuffer = AMediaCodec_getInputBuffer(mCodec, index, &bufferSize);
317 if (inputBuffer != nullptr)
318 {
319 memcpy(inputBuffer, buffer, size);
320 IMLOGD_PACKET2(IM_PACKET_LOG_AUDIO,
321 "[decodeAmr] queue input buffer index[%d], size[%d]", index, size);
322
323 auto err = AMediaCodec_queueInputBuffer(
324 mCodec, index, 0, size, ImsMediaTimer::GetTimeInMicroSeconds(), 0);
325 if (err != AMEDIA_OK)
326 {
327 IMLOGE1("[decodeAmr] Unable to queue input buffers - err[%d]", err);
328 }
329 }
330 }
331 else
332 {
333 IMLOGE1("[decodeAmr] Unable to get input buffers - err[%d]", index);
334 }
335
336 AMediaCodecBufferInfo info;
337 index = AMediaCodec_dequeueOutputBuffer(mCodec, &info, CODEC_TIMEOUT_NANO);
338
339 if (index >= 0)
340 {
341 IMLOGD_PACKET5(IM_PACKET_LOG_AUDIO,
342 "[decodeAmr] index[%d], size[%d], offset[%d], time[%ld], flags[%d]", index,
343 info.size, info.offset, info.presentationTimeUs, info.flags);
344
345 if (info.size > 0)
346 {
347 size_t buffCapacity;
348 uint8_t* buf = AMediaCodec_getOutputBuffer(mCodec, index, &buffCapacity);
349 if (buf != nullptr && buffCapacity > 0)
350 {
351 memcpy(mBuffer, buf, info.size);
352 // call audio write
353 AAudioStream_write(mAudioStream, mBuffer, info.size / 2, 0);
354 }
355 }
356
357 AMediaCodec_releaseOutputBuffer(mCodec, index, false);
358 }
359 else if (index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
360 {
361 IMLOGD0("[decodeAmr] output buffer changed");
362 }
363 else if (index == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
364 {
365 if (mFormat != nullptr)
366 {
367 AMediaFormat_delete(mFormat);
368 }
369
370 mFormat = AMediaCodec_getOutputFormat(mCodec);
371 IMLOGD1("[decodeAmr] format changed, format[%s]", AMediaFormat_toString(mFormat));
372 }
373 else if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
374 {
375 IMLOGD0("[decodeAmr] no output buffer");
376 }
377 else
378 {
379 IMLOGD1("[decodeAmr] unexpected index[%d]", index);
380 }
381
382 return true;
383 }
384
385 // TODO: Integration with libEVS is required.
decodeEvs(uint8_t * buffer,uint32_t size)386 bool ImsMediaAudioPlayer::decodeEvs(uint8_t* buffer, uint32_t size)
387 {
388 uint16_t output[PCM_BUFFER_SIZE];
389 int decodeSize = 0;
390
391 // TODO: Integration with libEVS is required to decode buffer data.
392 (void)buffer;
393 (void)size;
394
395 if (!mIsEvsInitialized)
396 {
397 IMLOGD0("[decodeEvs] Decoder has been initialised");
398 mIsEvsInitialized = true;
399 }
400
401 if (!mIsFirstFrame)
402 {
403 IMLOGD0("[decodeEvs] First frame has been decoded");
404 mIsFirstFrame = true;
405 }
406
407 AAudioStream_write(mAudioStream, output, (decodeSize / 2), 0);
408 memset(output, 0, PCM_BUFFER_SIZE);
409
410 return true;
411 }
412
openAudioStream()413 void ImsMediaAudioPlayer::openAudioStream()
414 {
415 AAudioStreamBuilder* builder = nullptr;
416 aaudio_result_t result = AAudio_createStreamBuilder(&builder);
417
418 if (result != AAUDIO_OK)
419 {
420 IMLOGE1("[openAudioStream] Error creating stream builder[%s]",
421 AAudio_convertResultToText(result));
422 return;
423 }
424
425 // setup builder
426 AAudioStreamBuilder_setInputPreset(builder, AAUDIO_INPUT_PRESET_VOICE_COMMUNICATION);
427 AAudioStreamBuilder_setDirection(builder, AAUDIO_DIRECTION_OUTPUT);
428 AAudioStreamBuilder_setFormat(builder, AAUDIO_FORMAT_PCM_I16);
429 AAudioStreamBuilder_setChannelCount(builder, 1);
430 AAudioStreamBuilder_setSampleRate(builder, mSamplingRate);
431 AAudioStreamBuilder_setSharingMode(builder, AAUDIO_SHARING_MODE_EXCLUSIVE);
432 AAudioStreamBuilder_setPerformanceMode(builder, AAUDIO_PERFORMANCE_MODE_LOW_LATENCY);
433 AAudioStreamBuilder_setUsage(builder, AAUDIO_USAGE_VOICE_COMMUNICATION);
434 AAudioStreamBuilder_setErrorCallback(builder, audioErrorCallback, this);
435
436 // open stream
437 result = AAudioStreamBuilder_openStream(builder, &mAudioStream);
438 AAudioStreamBuilder_delete(builder);
439
440 if (result != AAUDIO_OK)
441 {
442 IMLOGE1("[openAudioStream] Failed to openStream. Error[%s]",
443 AAudio_convertResultToText(result));
444 if (mAudioStream != nullptr)
445 {
446 AAudioStream_close(mAudioStream);
447 }
448 mAudioStream = nullptr;
449 }
450 }
451
restartAudioStream()452 void ImsMediaAudioPlayer::restartAudioStream()
453 {
454 std::lock_guard<std::mutex> guard(mMutex);
455
456 if (mAudioStream == nullptr)
457 {
458 return;
459 }
460
461 AAudioStream_requestStop(mAudioStream);
462 AAudioStream_close(mAudioStream);
463
464 mAudioStream = nullptr;
465 openAudioStream();
466
467 if (mAudioStream == nullptr)
468 {
469 return;
470 }
471
472 aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STARTING;
473 aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
474 aaudio_result_t result = AAudioStream_requestStart(mAudioStream);
475 if (result != AAUDIO_OK)
476 {
477 IMLOGE1("[restartAudioStream] Error start stream[%s]", AAudio_convertResultToText(result));
478 return;
479 }
480
481 result = AAudioStream_waitForStateChange(
482 mAudioStream, inputState, &nextState, 3 * AAUDIO_STATE_TIMEOUT_NANO);
483
484 if (result != AAUDIO_OK)
485 {
486 IMLOGE1("[restartAudioStream] Error start stream[%s]", AAudio_convertResultToText(result));
487 return;
488 }
489
490 IMLOGI1("[restartAudioStream] start stream state[%s]",
491 AAudio_convertStreamStateToText(nextState));
492 }
493
audioErrorCallback(AAudioStream * stream,void * userData,aaudio_result_t error)494 void ImsMediaAudioPlayer::audioErrorCallback(
495 AAudioStream* stream, void* userData, aaudio_result_t error)
496 {
497 if (stream == nullptr || userData == nullptr)
498 {
499 return;
500 }
501
502 aaudio_stream_state_t streamState = AAudioStream_getState(stream);
503 IMLOGW2("[errorCallback] error[%s], state[%d]", AAudio_convertResultToText(error), streamState);
504
505 if (error == AAUDIO_ERROR_DISCONNECTED)
506 {
507 // Handle stream restart on a separate thread
508 std::thread streamRestartThread(&ImsMediaAudioPlayer::restartAudioStream,
509 reinterpret_cast<ImsMediaAudioPlayer*>(userData));
510 streamRestartThread.detach();
511 }
512 }
513