1 /**
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <stdio.h>
18 #include <sys/time.h>
19 // #include <sys/timeb.h>
20 #include <fcntl.h>
21 #include <string.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 #include <thread>
25
26 #include <ImsMediaDefine.h>
27 #include <ImsMediaTrace.h>
28 #include <ImsMediaTimer.h>
29 #include <ImsMediaAudioUtil.h>
30 #include <ImsMediaAudioPlayer.h>
31 #include <utils/Errors.h>
32
33 #define AAUDIO_STATE_TIMEOUT_NANO (100 * 1000000L)
34 #define DEFAULT_SAMPLING_RATE (8000)
35 #define CODEC_TIMEOUT_NANO (100000)
36
37 using namespace android;
38
ImsMediaAudioPlayer()39 ImsMediaAudioPlayer::ImsMediaAudioPlayer()
40 {
41 mAudioStream = nullptr;
42 mCodec = nullptr;
43 mFormat = nullptr;
44 mCodecType = 0;
45 mCodecMode = 0;
46 mSamplingRate = DEFAULT_SAMPLING_RATE;
47 mEvsChAwOffset = 0;
48 mEvsBandwidth = kEvsBandwidthNone;
49 memset(mBuffer, 0, sizeof(mBuffer));
50 mEvsBitRate = 0;
51 mEvsCodecHeaderMode = kRtpPyaloadHeaderModeEvsHeaderFull;
52 mIsFirstFrame = false;
53 mIsEvsInitialized = false;
54 mIsOctetAligned = false;
55 mIsDtxEnabled = false;
56 }
57
~ImsMediaAudioPlayer()58 ImsMediaAudioPlayer::~ImsMediaAudioPlayer() {}
59
SetCodec(int32_t type)60 void ImsMediaAudioPlayer::SetCodec(int32_t type)
61 {
62 IMLOGD_PACKET1(IM_PACKET_LOG_AUDIO, "[SetCodec] type[%d]", type);
63 mCodecType = type;
64 }
65
SetEvsBitRate(int32_t bitRate)66 void ImsMediaAudioPlayer::SetEvsBitRate(int32_t bitRate)
67 {
68 mEvsBitRate = bitRate;
69 }
70
SetEvsChAwOffset(int32_t offset)71 void ImsMediaAudioPlayer::SetEvsChAwOffset(int32_t offset)
72 {
73 mEvsChAwOffset = offset;
74 }
75
SetSamplingRate(int32_t samplingRate)76 void ImsMediaAudioPlayer::SetSamplingRate(int32_t samplingRate)
77 {
78 mSamplingRate = samplingRate;
79 }
80
SetEvsBandwidth(int32_t evsBandwidth)81 void ImsMediaAudioPlayer::SetEvsBandwidth(int32_t evsBandwidth)
82 {
83 mEvsBandwidth = (kEvsBandwidth)evsBandwidth;
84 }
85
SetEvsPayloadHeaderMode(int32_t EvsPayloadHeaderMode)86 void ImsMediaAudioPlayer::SetEvsPayloadHeaderMode(int32_t EvsPayloadHeaderMode)
87 {
88 mEvsCodecHeaderMode = (kRtpPyaloadHeaderMode)EvsPayloadHeaderMode;
89 }
90
SetCodecMode(uint32_t mode)91 void ImsMediaAudioPlayer::SetCodecMode(uint32_t mode)
92 {
93 IMLOGD1("[SetCodecMode] mode[%d]", mode);
94 mCodecMode = mode;
95 }
96
SetDtxEnabled(bool isDtxEnabled)97 void ImsMediaAudioPlayer::SetDtxEnabled(bool isDtxEnabled)
98 {
99 mIsDtxEnabled = isDtxEnabled;
100 }
101
SetOctetAligned(bool isOctetAligned)102 void ImsMediaAudioPlayer::SetOctetAligned(bool isOctetAligned)
103 {
104 mIsOctetAligned = isOctetAligned;
105 }
106
ProcessCmr(const uint32_t cmr)107 void ImsMediaAudioPlayer::ProcessCmr(const uint32_t cmr)
108 {
109 IMLOGD1("[ProcessCmr] cmr[%d]", cmr);
110
111 mCodecMode = cmr;
112 Stop();
113 Start();
114 }
115
Start()116 bool ImsMediaAudioPlayer::Start()
117 {
118 char kMimeType[128] = {'\0'};
119
120 if (mCodecType == kAudioCodecAmr)
121 {
122 sprintf(kMimeType, "audio/3gpp");
123 }
124 else if (mCodecType == kAudioCodecAmrWb)
125 {
126 sprintf(kMimeType, "audio/amr-wb");
127 }
128 else if (mCodecType == kAudioCodecEvs)
129 {
130 // TODO: Integration with libEVS is required.
131 sprintf(kMimeType, "audio/evs");
132 }
133 else
134 {
135 return false;
136 }
137
138 openAudioStream();
139
140 if (mAudioStream == nullptr)
141 {
142 IMLOGE0("[Start] create audio stream failed");
143 return false;
144 }
145
146 IMLOGD1("[Start] Creating codec[%s]", kMimeType);
147
148 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
149 {
150 mFormat = AMediaFormat_new();
151 AMediaFormat_setString(mFormat, AMEDIAFORMAT_KEY_MIME, kMimeType);
152 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_SAMPLE_RATE, mSamplingRate);
153 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_CHANNEL_COUNT, 1);
154
155 mCodec = AMediaCodec_createDecoderByType(kMimeType);
156
157 if (mCodec == nullptr)
158 {
159 IMLOGE1("[Start] unable to create %s codec instance", kMimeType);
160 AMediaFormat_delete(mFormat);
161 mFormat = nullptr;
162 return false;
163 }
164
165 IMLOGD0("[Start] configure codec");
166 media_status_t codecResult = AMediaCodec_configure(mCodec, mFormat, nullptr, nullptr, 0);
167
168 if (codecResult != AMEDIA_OK)
169 {
170 IMLOGE2("[Start] unable to configure[%s] codec - err[%d]", kMimeType, codecResult);
171 AMediaCodec_delete(mCodec);
172 mCodec = nullptr;
173 AMediaFormat_delete(mFormat);
174 mFormat = nullptr;
175 return false;
176 }
177 }
178
179 aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STARTING;
180 aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
181 auto result = AAudioStream_requestStart(mAudioStream);
182
183 if (result != AAUDIO_OK)
184 {
185 IMLOGE1("[Start] Error start stream[%s]", AAudio_convertResultToText(result));
186 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
187 {
188 AMediaCodec_delete(mCodec);
189 mCodec = nullptr;
190 AMediaFormat_delete(mFormat);
191 mFormat = nullptr;
192 }
193 return false;
194 }
195
196 result = AAudioStream_waitForStateChange(
197 mAudioStream, inputState, &nextState, AAUDIO_STATE_TIMEOUT_NANO);
198
199 if (result != AAUDIO_OK)
200 {
201 IMLOGE1("[Start] Error start stream[%s]", AAudio_convertResultToText(result));
202 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
203 {
204 AMediaCodec_delete(mCodec);
205 mCodec = nullptr;
206 AMediaFormat_delete(mFormat);
207 mFormat = nullptr;
208 }
209 return false;
210 }
211
212 IMLOGI1("[Start] start stream state[%s]", AAudio_convertStreamStateToText(nextState));
213
214 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
215 {
216 media_status_t codecResult = AMediaCodec_start(mCodec);
217 if (codecResult != AMEDIA_OK)
218 {
219 IMLOGE1("[Start] unable to start codec - err[%d]", codecResult);
220 AMediaCodec_delete(mCodec);
221 mCodec = nullptr;
222 AMediaFormat_delete(mFormat);
223 mFormat = nullptr;
224 return false;
225 }
226 }
227
228 IMLOGD0("[Start] exit");
229 return true;
230 }
231
Stop()232 void ImsMediaAudioPlayer::Stop()
233 {
234 IMLOGD0("[Stop] enter");
235 std::lock_guard<std::mutex> guard(mMutex);
236 if ((mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb) && (mCodec != nullptr))
237 {
238 AMediaCodec_stop(mCodec);
239 AMediaCodec_delete(mCodec);
240 mCodec = nullptr;
241 }
242
243 if ((mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb) && (mFormat != nullptr))
244 {
245 AMediaFormat_delete(mFormat);
246 mFormat = nullptr;
247 }
248
249 if (mAudioStream != nullptr)
250 {
251 aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STOPPING;
252 aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
253 aaudio_result_t result = AAudioStream_requestStop(mAudioStream);
254
255 if (result != AAUDIO_OK)
256 {
257 IMLOGE1("[Stop] Error stop stream[%s]", AAudio_convertResultToText(result));
258 }
259
260 // TODO: if it causes extra delay in stop, optimize later
261 result = AAudioStream_waitForStateChange(
262 mAudioStream, inputState, &nextState, AAUDIO_STATE_TIMEOUT_NANO);
263
264 if (result != AAUDIO_OK)
265 {
266 IMLOGE1("[Stop] Error stop stream[%s]", AAudio_convertResultToText(result));
267 }
268
269 IMLOGI1("[Stop] stream state[%s]", AAudio_convertStreamStateToText(nextState));
270
271 AAudioStream_close(mAudioStream);
272 mAudioStream = nullptr;
273 }
274
275 IMLOGD0("[Stop] exit ");
276 }
277
onDataFrame(uint8_t * buffer,uint32_t size,FrameType,bool,uint8_t)278 bool ImsMediaAudioPlayer::onDataFrame(uint8_t* buffer, uint32_t size, FrameType /*frameType*/,
279 bool /*hasNextFrame*/, uint8_t /*nextFrameByte*/)
280 {
281 std::lock_guard<std::mutex> guard(mMutex);
282
283 if (size == 0 || buffer == nullptr || mAudioStream == nullptr ||
284 AAudioStream_getState(mAudioStream) != AAUDIO_STREAM_STATE_STARTED)
285 {
286 return false;
287 }
288
289 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
290 {
291 if (mCodec == nullptr)
292 {
293 return false;
294 }
295 else
296 {
297 return decodeAmr(buffer, size);
298 }
299 }
300 else if (mCodecType == kAudioCodecEvs)
301 {
302 // TODO:Integration with libEVS is required.
303 return decodeEvs(buffer, size);
304 }
305 return false;
306 }
307
decodeAmr(uint8_t * buffer,uint32_t size)308 bool ImsMediaAudioPlayer::decodeAmr(uint8_t* buffer, uint32_t size)
309 {
310 auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO);
311
312 if (index >= 0)
313 {
314 size_t bufferSize = 0;
315 uint8_t* inputBuffer = AMediaCodec_getInputBuffer(mCodec, index, &bufferSize);
316 if (inputBuffer != nullptr)
317 {
318 memcpy(inputBuffer, buffer, size);
319 IMLOGD_PACKET2(IM_PACKET_LOG_AUDIO,
320 "[decodeAmr] queue input buffer index[%d], size[%d]", index, size);
321
322 auto err = AMediaCodec_queueInputBuffer(
323 mCodec, index, 0, size, ImsMediaTimer::GetTimeInMicroSeconds(), 0);
324 if (err != AMEDIA_OK)
325 {
326 IMLOGE1("[decodeAmr] Unable to queue input buffers - err[%d]", err);
327 }
328 }
329 }
330 else
331 {
332 IMLOGE1("[decodeAmr] Unable to get input buffers - err[%d]", index);
333 }
334
335 AMediaCodecBufferInfo info;
336 index = AMediaCodec_dequeueOutputBuffer(mCodec, &info, CODEC_TIMEOUT_NANO);
337
338 if (index >= 0)
339 {
340 IMLOGD_PACKET5(IM_PACKET_LOG_AUDIO,
341 "[decodeAmr] index[%d], size[%d], offset[%d], time[%ld], flags[%d]", index,
342 info.size, info.offset, info.presentationTimeUs, info.flags);
343
344 if (info.size > 0)
345 {
346 size_t buffCapacity;
347 uint8_t* buf = AMediaCodec_getOutputBuffer(mCodec, index, &buffCapacity);
348 if (buf != nullptr && buffCapacity > 0)
349 {
350 memcpy(mBuffer, buf, info.size);
351 // call audio write
352 AAudioStream_write(mAudioStream, mBuffer, info.size / 2, 0);
353 }
354 }
355
356 AMediaCodec_releaseOutputBuffer(mCodec, index, false);
357 }
358 else if (index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
359 {
360 IMLOGD0("[decodeAmr] output buffer changed");
361 }
362 else if (index == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
363 {
364 if (mFormat != nullptr)
365 {
366 AMediaFormat_delete(mFormat);
367 }
368
369 mFormat = AMediaCodec_getOutputFormat(mCodec);
370 IMLOGD1("[decodeAmr] format changed, format[%s]", AMediaFormat_toString(mFormat));
371 }
372 else if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
373 {
374 IMLOGD0("[decodeAmr] no output buffer");
375 }
376 else
377 {
378 IMLOGD1("[decodeAmr] unexpected index[%d]", index);
379 }
380
381 return true;
382 }
383
384 // TODO: Integration with libEVS is required.
decodeEvs(uint8_t * buffer,uint32_t size)385 bool ImsMediaAudioPlayer::decodeEvs(uint8_t* buffer, uint32_t size)
386 {
387 uint16_t output[PCM_BUFFER_SIZE];
388 int decodeSize = 0;
389
390 // TODO: Integration with libEVS is required to decode buffer data.
391 (void)buffer;
392 (void)size;
393
394 if (!mIsEvsInitialized)
395 {
396 IMLOGD0("[decodeEvs] Decoder has been initialised");
397 mIsEvsInitialized = true;
398 }
399
400 if (!mIsFirstFrame)
401 {
402 IMLOGD0("[decodeEvs] First frame has been decoded");
403 mIsFirstFrame = true;
404 }
405
406 AAudioStream_write(mAudioStream, output, (decodeSize / 2), 0);
407 memset(output, 0, PCM_BUFFER_SIZE);
408
409 return true;
410 }
411
openAudioStream()412 void ImsMediaAudioPlayer::openAudioStream()
413 {
414 AAudioStreamBuilder* builder = nullptr;
415 aaudio_result_t result = AAudio_createStreamBuilder(&builder);
416
417 if (result != AAUDIO_OK)
418 {
419 IMLOGE1("[openAudioStream] Error creating stream builder[%s]",
420 AAudio_convertResultToText(result));
421 return;
422 }
423
424 // setup builder
425 AAudioStreamBuilder_setInputPreset(builder, AAUDIO_INPUT_PRESET_VOICE_COMMUNICATION);
426 AAudioStreamBuilder_setDirection(builder, AAUDIO_DIRECTION_OUTPUT);
427 AAudioStreamBuilder_setFormat(builder, AAUDIO_FORMAT_PCM_I16);
428 AAudioStreamBuilder_setChannelCount(builder, 1);
429 AAudioStreamBuilder_setSampleRate(builder, mSamplingRate);
430 AAudioStreamBuilder_setSharingMode(builder, AAUDIO_SHARING_MODE_EXCLUSIVE);
431 AAudioStreamBuilder_setPerformanceMode(builder, AAUDIO_PERFORMANCE_MODE_LOW_LATENCY);
432 AAudioStreamBuilder_setUsage(builder, AAUDIO_USAGE_VOICE_COMMUNICATION);
433 AAudioStreamBuilder_setErrorCallback(builder, audioErrorCallback, this);
434
435 // open stream
436 result = AAudioStreamBuilder_openStream(builder, &mAudioStream);
437 AAudioStreamBuilder_delete(builder);
438
439 if (result != AAUDIO_OK)
440 {
441 IMLOGE1("[openAudioStream] Failed to openStream. Error[%s]",
442 AAudio_convertResultToText(result));
443 if (mAudioStream != nullptr)
444 {
445 AAudioStream_close(mAudioStream);
446 }
447 mAudioStream = nullptr;
448 }
449 }
450
restartAudioStream()451 void ImsMediaAudioPlayer::restartAudioStream()
452 {
453 std::lock_guard<std::mutex> guard(mMutex);
454
455 if (mAudioStream == nullptr)
456 {
457 return;
458 }
459
460 AAudioStream_requestStop(mAudioStream);
461 AAudioStream_close(mAudioStream);
462
463 mAudioStream = nullptr;
464 openAudioStream();
465
466 if (mAudioStream == nullptr)
467 {
468 return;
469 }
470
471 aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STARTING;
472 aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
473 aaudio_result_t result = AAudioStream_requestStart(mAudioStream);
474 if (result != AAUDIO_OK)
475 {
476 IMLOGE1("[restartAudioStream] Error start stream[%s]", AAudio_convertResultToText(result));
477 return;
478 }
479
480 result = AAudioStream_waitForStateChange(
481 mAudioStream, inputState, &nextState, 3 * AAUDIO_STATE_TIMEOUT_NANO);
482
483 if (result != AAUDIO_OK)
484 {
485 IMLOGE1("[restartAudioStream] Error start stream[%s]", AAudio_convertResultToText(result));
486 return;
487 }
488
489 IMLOGI1("[restartAudioStream] start stream state[%s]",
490 AAudio_convertStreamStateToText(nextState));
491 }
492
audioErrorCallback(AAudioStream * stream,void * userData,aaudio_result_t error)493 void ImsMediaAudioPlayer::audioErrorCallback(
494 AAudioStream* stream, void* userData, aaudio_result_t error)
495 {
496 if (stream == nullptr || userData == nullptr)
497 {
498 return;
499 }
500
501 aaudio_stream_state_t streamState = AAudioStream_getState(stream);
502 IMLOGW2("[errorCallback] error[%s], state[%d]", AAudio_convertResultToText(error), streamState);
503
504 if (error == AAUDIO_ERROR_DISCONNECTED)
505 {
506 // Handle stream restart on a separate thread
507 std::thread streamRestartThread(&ImsMediaAudioPlayer::restartAudioStream,
508 reinterpret_cast<ImsMediaAudioPlayer*>(userData));
509 streamRestartThread.detach();
510 }
511 }
512