1 /*
2 * Copyright 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <memory>
18 #include <oboe/Definitions.h>
19 #include "FFMpegExtractor.h"
20 #include "utils/logging.h"
21
22 constexpr int kInternalBufferSize = 1152; // Use MP3 block size. https://wiki.hydrogenaud.io/index.php?title=MP3
23
read(void * opaque,uint8_t * buf,int buf_size)24 int read(void *opaque, uint8_t *buf, int buf_size) {
25
26 auto asset = (AAsset *) opaque;
27 int bytesRead = AAsset_read(asset, buf, (size_t)buf_size);
28 return bytesRead;
29 }
30
seek(void * opaque,int64_t offset,int whence)31 int64_t seek(void *opaque, int64_t offset, int whence){
32
33 auto asset = (AAsset*)opaque;
34
35 // See https://www.ffmpeg.org/doxygen/3.0/avio_8h.html#a427ff2a881637b47ee7d7f9e368be63f
36 if (whence == AVSEEK_SIZE) return AAsset_getLength(asset);
37 if (AAsset_seek(asset, offset, whence) == -1){
38 return -1;
39 } else {
40 return 0;
41 }
42 }
43
createAVIOContext(AAsset * asset,uint8_t * buffer,uint32_t bufferSize,AVIOContext ** avioContext)44 bool FFMpegExtractor::createAVIOContext(AAsset *asset, uint8_t *buffer, uint32_t bufferSize,
45 AVIOContext **avioContext) {
46
47 constexpr int isBufferWriteable = 0;
48
49 *avioContext = avio_alloc_context(
50 buffer, // internal buffer for FFmpeg to use
51 bufferSize, // For optimal decoding speed this should be the protocol block size
52 isBufferWriteable,
53 asset, // Will be passed to our callback functions as a (void *)
54 read, // Read callback function
55 nullptr, // Write callback function (not used)
56 seek); // Seek callback function
57
58 if (*avioContext == nullptr){
59 LOGE("Failed to create AVIO context");
60 return false;
61 } else {
62 return true;
63 }
64 }
65
66 bool
createAVFormatContext(AVIOContext * avioContext,AVFormatContext ** avFormatContext)67 FFMpegExtractor::createAVFormatContext(AVIOContext *avioContext, AVFormatContext **avFormatContext) {
68
69 *avFormatContext = avformat_alloc_context();
70 (*avFormatContext)->pb = avioContext;
71
72 if (*avFormatContext == nullptr){
73 LOGE("Failed to create AVFormatContext");
74 return false;
75 } else {
76 return true;
77 }
78 }
79
openAVFormatContext(AVFormatContext * avFormatContext)80 bool FFMpegExtractor::openAVFormatContext(AVFormatContext *avFormatContext) {
81
82 int result = avformat_open_input(&avFormatContext,
83 "", /* URL is left empty because we're providing our own I/O */
84 nullptr /* AVInputFormat *fmt */,
85 nullptr /* AVDictionary **options */
86 );
87
88 if (result == 0) {
89 return true;
90 } else {
91 LOGE("Failed to open file. Error code %s", av_err2str(result));
92 return false;
93 }
94 }
95
getStreamInfo(AVFormatContext * avFormatContext)96 bool FFMpegExtractor::getStreamInfo(AVFormatContext *avFormatContext) {
97
98 int result = avformat_find_stream_info(avFormatContext, nullptr);
99 if (result == 0 ){
100 return true;
101 } else {
102 LOGE("Failed to find stream info. Error code %s", av_err2str(result));
103 return false;
104 }
105 }
106
getBestAudioStream(AVFormatContext * avFormatContext)107 AVStream *FFMpegExtractor::getBestAudioStream(AVFormatContext *avFormatContext) {
108
109 int streamIndex = av_find_best_stream(avFormatContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
110
111 if (streamIndex < 0){
112 LOGE("Could not find stream");
113 return nullptr;
114 } else {
115 return avFormatContext->streams[streamIndex];
116 }
117 }
118
decode(AAsset * asset,uint8_t * targetData,AudioProperties targetProperties)119 int64_t FFMpegExtractor::decode(
120 AAsset *asset,
121 uint8_t *targetData,
122 AudioProperties targetProperties) {
123
124 int returnValue = -1; // -1 indicates error
125
126 // Create a buffer for FFmpeg to use for decoding (freed in the custom deleter below)
127 auto buffer = reinterpret_cast<uint8_t*>(av_malloc(kInternalBufferSize));
128
129 // Create an AVIOContext with a custom deleter
130 std::unique_ptr<AVIOContext, void(*)(AVIOContext *)> ioContext {
131 nullptr,
132 [](AVIOContext *c) {
133 av_free(c->buffer);
134 avio_context_free(&c);
135 }
136 };
137 {
138 AVIOContext *tmp = nullptr;
139 if (!createAVIOContext(asset, buffer, kInternalBufferSize, &tmp)){
140 LOGE("Could not create an AVIOContext");
141 return returnValue;
142 }
143 ioContext.reset(tmp);
144 }
145
146 // Create an AVFormatContext using the avformat_free_context as the deleter function
147 std::unique_ptr<AVFormatContext, decltype(&avformat_free_context)> formatContext {
148 nullptr,
149 &avformat_free_context
150 };
151 {
152 AVFormatContext *tmp;
153 if (!createAVFormatContext(ioContext.get(), &tmp)) return returnValue;
154 formatContext.reset(tmp);
155 }
156
157 if (!openAVFormatContext(formatContext.get())) return returnValue;
158
159 if (!getStreamInfo(formatContext.get())) return returnValue;
160
161 // Obtain the best audio stream to decode
162 AVStream *stream = getBestAudioStream(formatContext.get());
163 if (stream == nullptr || stream->codecpar == nullptr){
164 LOGE("Could not find a suitable audio stream to decode");
165 return returnValue;
166 }
167
168 printCodecParameters(stream->codecpar);
169
170 // Find the codec to decode this stream
171 AVCodec *codec = avcodec_find_decoder(stream->codecpar->codec_id);
172 if (!codec){
173 LOGE("Could not find codec with ID: %d", stream->codecpar->codec_id);
174 return returnValue;
175 }
176
177 // Create the codec context, specifying the deleter function
178 std::unique_ptr<AVCodecContext, void(*)(AVCodecContext *)> codecContext {
179 nullptr,
180 [](AVCodecContext *c) { avcodec_free_context(&c); }
181 };
182 {
183 AVCodecContext *tmp = avcodec_alloc_context3(codec);
184 if (!tmp){
185 LOGE("Failed to allocate codec context");
186 return returnValue;
187 }
188 codecContext.reset(tmp);
189 }
190
191 // Copy the codec parameters into the context
192 if (avcodec_parameters_to_context(codecContext.get(), stream->codecpar) < 0){
193 LOGE("Failed to copy codec parameters to codec context");
194 return returnValue;
195 }
196
197 // Open the codec
198 if (avcodec_open2(codecContext.get(), codec, nullptr) < 0){
199 LOGE("Could not open codec");
200 return returnValue;
201 }
202
203 // prepare resampler
204 int32_t outChannelLayout = (1 << targetProperties.channelCount) - 1;
205 LOGD("Channel layout %d", outChannelLayout);
206
207 SwrContext *swr = swr_alloc();
208 av_opt_set_int(swr, "in_channel_count", stream->codecpar->channels, 0);
209 av_opt_set_int(swr, "out_channel_count", targetProperties.channelCount, 0);
210 av_opt_set_int(swr, "in_channel_layout", stream->codecpar->channel_layout, 0);
211 av_opt_set_int(swr, "out_channel_layout", outChannelLayout, 0);
212 av_opt_set_int(swr, "in_sample_rate", stream->codecpar->sample_rate, 0);
213 av_opt_set_int(swr, "out_sample_rate", targetProperties.sampleRate, 0);
214 av_opt_set_int(swr, "in_sample_fmt", stream->codecpar->format, 0);
215 av_opt_set_sample_fmt(swr, "out_sample_fmt", AV_SAMPLE_FMT_FLT, 0);
216 av_opt_set_int(swr, "force_resampling", 1, 0);
217
218 // Check that resampler has been inited
219 int result = swr_init(swr);
220 if (result != 0){
221 LOGE("swr_init failed. Error: %s", av_err2str(result));
222 return returnValue;
223 };
224 if (!swr_is_initialized(swr)) {
225 LOGE("swr_is_initialized is false\n");
226 return returnValue;
227 }
228
229 // Prepare to read data
230 int bytesWritten = 0;
231 AVPacket avPacket; // Stores compressed audio data
232 av_init_packet(&avPacket);
233 AVFrame *decodedFrame = av_frame_alloc(); // Stores raw audio data
234 int bytesPerSample = av_get_bytes_per_sample((AVSampleFormat)stream->codecpar->format);
235
236 LOGD("Bytes per sample %d", bytesPerSample);
237
238 LOGD("DECODE START");
239
240 // While there is more data to read, read it into the avPacket
241 while (av_read_frame(formatContext.get(), &avPacket) == 0){
242
243 if (avPacket.stream_index == stream->index && avPacket.size > 0) {
244
245 // Pass our compressed data into the codec
246 result = avcodec_send_packet(codecContext.get(), &avPacket);
247 if (result != 0) {
248 LOGE("avcodec_send_packet error: %s", av_err2str(result));
249 goto cleanup;
250 }
251
252 // Retrieve our raw data from the codec
253 result = avcodec_receive_frame(codecContext.get(), decodedFrame);
254 if (result == AVERROR(EAGAIN)) {
255 // The codec needs more data before it can decode
256 LOGI("avcodec_receive_frame returned EAGAIN");
257 avPacket.size = 0;
258 avPacket.data = nullptr;
259 continue;
260 } else if (result != 0) {
261 LOGE("avcodec_receive_frame error: %s", av_err2str(result));
262 goto cleanup;
263 }
264
265 // DO RESAMPLING
266 auto dst_nb_samples = (int32_t) av_rescale_rnd(
267 swr_get_delay(swr, decodedFrame->sample_rate) + decodedFrame->nb_samples,
268 targetProperties.sampleRate,
269 decodedFrame->sample_rate,
270 AV_ROUND_UP);
271
272 short *buffer1;
273 av_samples_alloc(
274 (uint8_t **) &buffer1,
275 nullptr,
276 targetProperties.channelCount,
277 dst_nb_samples,
278 AV_SAMPLE_FMT_FLT,
279 0);
280 int frame_count = swr_convert(
281 swr,
282 (uint8_t **) &buffer1,
283 dst_nb_samples,
284 (const uint8_t **) decodedFrame->data,
285 decodedFrame->nb_samples);
286
287 int64_t bytesToWrite = frame_count * sizeof(float) * targetProperties.channelCount;
288 memcpy(targetData + bytesWritten, buffer1, (size_t)bytesToWrite);
289 bytesWritten += bytesToWrite;
290 av_freep(&buffer1);
291
292 avPacket.size = 0;
293 avPacket.data = nullptr;
294 }
295 }
296
297 av_frame_free(&decodedFrame);
298 LOGD("DECODE END");
299
300 returnValue = bytesWritten;
301
302 cleanup:
303 return returnValue;
304 }
305
printCodecParameters(AVCodecParameters * params)306 void FFMpegExtractor::printCodecParameters(AVCodecParameters *params) {
307
308 LOGD("Stream properties");
309 LOGD("Channels: %d", params->channels);
310 LOGD("Channel layout: %" PRId64, params->channel_layout);
311 LOGD("Sample rate: %d", params->sample_rate);
312 LOGD("Format: %s", av_get_sample_fmt_name((AVSampleFormat)params->format));
313 LOGD("Frame size: %d", params->frame_size);
314 }
315
316
317
318
319