1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 /* Original code copied from NDK Native-media sample code */
18
19 //#define LOG_NDEBUG 0
20 #define LOG_TAG "NativeMedia"
21 #include <log/log.h>
22
23 #include <assert.h>
24 #include <jni.h>
25 #include <mutex>
26 #include <queue>
27 #include <stdio.h>
28 #include <string.h>
29 #include <unistd.h>
30
31 #include <android/native_window_jni.h>
32
33 #include "media/NdkMediaExtractor.h"
34 #include "media/NdkMediaCodec.h"
35 #include "media/NdkMediaDataSource.h"
36 #include "media/NdkMediaFormat.h"
37 template <class T>
38 class simplevector {
39 T *storage;
40 int capacity;
41 int numfilled;
42 public:
simplevector()43 simplevector() {
44 capacity = 16;
45 numfilled = 0;
46 storage = new T[capacity];
47 }
~simplevector()48 ~simplevector() {
49 delete[] storage;
50 }
51
add(T item)52 void add(T item) {
53 if (numfilled == capacity) {
54 T *old = storage;
55 capacity *= 2;
56 storage = new T[capacity];
57 for (int i = 0; i < numfilled; i++) {
58 storage[i] = old[i];
59 }
60 delete[] old;
61 }
62 storage[numfilled] = item;
63 numfilled++;
64 }
65
size()66 int size() {
67 return numfilled;
68 }
69
data()70 T* data() {
71 return storage;
72 }
73 };
74
75 struct FdDataSource {
76
FdDataSourceFdDataSource77 FdDataSource(int fd, jlong offset, jlong size)
78 : mFd(dup(fd)),
79 mOffset(offset),
80 mSize(size) {
81 }
82
readAtFdDataSource83 ssize_t readAt(off64_t offset, void *data, size_t size) {
84 ssize_t ssize = size;
85 if (!data || offset < 0 || offset + ssize < offset) {
86 return -1;
87 }
88 if (offset >= mSize) {
89 return 0; // EOS
90 }
91 if (offset + ssize > mSize) {
92 ssize = mSize - offset;
93 }
94 if (lseek(mFd, mOffset + offset, SEEK_SET) < 0) {
95 return -1;
96 }
97 return read(mFd, data, ssize);
98 }
99
getSizeFdDataSource100 ssize_t getSize() {
101 return mSize;
102 }
103
closeFdDataSource104 void close() {
105 ::close(mFd);
106 }
107
108 private:
109
110 int mFd;
111 off64_t mOffset;
112 int64_t mSize;
113
114 };
115
FdSourceReadAt(void * userdata,off64_t offset,void * data,size_t size)116 static ssize_t FdSourceReadAt(void *userdata, off64_t offset, void *data, size_t size) {
117 FdDataSource *src = (FdDataSource*) userdata;
118 return src->readAt(offset, data, size);
119 }
120
FdSourceGetSize(void * userdata)121 static ssize_t FdSourceGetSize(void *userdata) {
122 FdDataSource *src = (FdDataSource*) userdata;
123 return src->getSize();
124 }
125
FdSourceClose(void * userdata)126 static void FdSourceClose(void *userdata) {
127 FdDataSource *src = (FdDataSource*) userdata;
128 src->close();
129 }
130
131 class CallbackData {
132 std::mutex mMutex;
133 std::queue<int32_t> mInputBufferIds;
134 std::queue<int32_t> mOutputBufferIds;
135 std::queue<AMediaCodecBufferInfo> mOutputBufferInfos;
136 std::queue<AMediaFormat*> mFormats;
137
138 public:
CallbackData()139 CallbackData() { }
140
~CallbackData()141 ~CallbackData() {
142 mMutex.lock();
143 while (!mFormats.empty()) {
144 AMediaFormat* format = mFormats.front();
145 mFormats.pop();
146 AMediaFormat_delete(format);
147 }
148 mMutex.unlock();
149 }
150
addInputBufferId(int32_t index)151 void addInputBufferId(int32_t index) {
152 mMutex.lock();
153 mInputBufferIds.push(index);
154 mMutex.unlock();
155 }
156
getInputBufferId()157 int32_t getInputBufferId() {
158 int32_t id = -1;
159 mMutex.lock();
160 if (!mInputBufferIds.empty()) {
161 id = mInputBufferIds.front();
162 mInputBufferIds.pop();
163 }
164 mMutex.unlock();
165 return id;
166 }
167
addOutputBuffer(int32_t index,AMediaCodecBufferInfo * bufferInfo)168 void addOutputBuffer(int32_t index, AMediaCodecBufferInfo *bufferInfo) {
169 mMutex.lock();
170 mOutputBufferIds.push(index);
171 mOutputBufferInfos.push(*bufferInfo);
172 mMutex.unlock();
173 }
174
addOutputFormat(AMediaFormat * format)175 void addOutputFormat(AMediaFormat *format) {
176 mMutex.lock();
177 mOutputBufferIds.push(AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED);
178 mFormats.push(format);
179 mMutex.unlock();
180 }
181
getOutput(AMediaCodecBufferInfo * bufferInfo,AMediaFormat ** format)182 int32_t getOutput(AMediaCodecBufferInfo *bufferInfo, AMediaFormat **format) {
183 int32_t id = AMEDIACODEC_INFO_TRY_AGAIN_LATER;
184 mMutex.lock();
185 if (!mOutputBufferIds.empty()) {
186 id = mOutputBufferIds.front();
187 mOutputBufferIds.pop();
188
189 if (id >= 0) {
190 *bufferInfo = mOutputBufferInfos.front();
191 mOutputBufferInfos.pop();
192 } else { // AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED
193 *format = mFormats.front();
194 mFormats.pop();
195 }
196 }
197 mMutex.unlock();
198 return id;
199 }
200 };
201
OnInputAvailableCB(AMediaCodec *,void * userdata,int32_t index)202 static void OnInputAvailableCB(
203 AMediaCodec * /* aMediaCodec */,
204 void *userdata,
205 int32_t index) {
206 ALOGV("OnInputAvailableCB: index(%d)", index);
207 CallbackData *callbackData = (CallbackData *)userdata;
208 callbackData->addInputBufferId(index);
209 }
210
OnOutputAvailableCB(AMediaCodec *,void * userdata,int32_t index,AMediaCodecBufferInfo * bufferInfo)211 static void OnOutputAvailableCB(
212 AMediaCodec * /* aMediaCodec */,
213 void *userdata,
214 int32_t index,
215 AMediaCodecBufferInfo *bufferInfo) {
216 ALOGV("OnOutputAvailableCB: index(%d), (%d, %d, %lld, 0x%x)",
217 index, bufferInfo->offset, bufferInfo->size,
218 (long long)bufferInfo->presentationTimeUs, bufferInfo->flags);
219 CallbackData *callbackData = (CallbackData *)userdata;
220 callbackData->addOutputBuffer(index, bufferInfo);
221 }
222
OnFormatChangedCB(AMediaCodec *,void * userdata,AMediaFormat * format)223 static void OnFormatChangedCB(
224 AMediaCodec * /* aMediaCodec */,
225 void *userdata,
226 AMediaFormat *format) {
227 ALOGV("OnFormatChangedCB: format(%s)", AMediaFormat_toString(format));
228 CallbackData *callbackData = (CallbackData *)userdata;
229 callbackData->addOutputFormat(format);
230 }
231
OnErrorCB(AMediaCodec *,void *,media_status_t err,int32_t actionCode,const char * detail)232 static void OnErrorCB(
233 AMediaCodec * /* aMediaCodec */,
234 void * /* userdata */,
235 media_status_t err,
236 int32_t actionCode,
237 const char *detail) {
238 ALOGV("OnErrorCB: err(%d), actionCode(%d), detail(%s)", err, actionCode, detail);
239 }
240
adler32(const uint8_t * input,int len)241 static int adler32(const uint8_t *input, int len) {
242
243 int a = 1;
244 int b = 0;
245 for (int i = 0; i < len; i++) {
246 a += input[i];
247 b += a;
248 a = a % 65521;
249 b = b % 65521;
250 }
251 int ret = b * 65536 + a;
252 ALOGV("adler %d/%d", len, ret);
253 return ret;
254 }
255
checksum(const uint8_t * in,int len,AMediaFormat * format)256 static int checksum(const uint8_t *in, int len, AMediaFormat *format) {
257 int width, stride, height;
258 if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width)) {
259 width = len;
260 }
261 if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_STRIDE, &stride)) {
262 stride = width;
263 }
264 if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height)) {
265 height = 1;
266 }
267 uint8_t *bb = new uint8_t[width * height];
268 for (int i = 0; i < height; i++) {
269 memcpy(bb + i * width, in + i * stride, width);
270 }
271 // bb is filled with data
272 int sum = adler32(bb, width * height);
273 delete[] bb;
274 return sum;
275 }
276
Java_android_media_decoder_cts_NativeDecoderTest_getDecodedDataNative(JNIEnv * env,jclass,int fd,jlong offset,jlong size,jboolean wrapFd,jboolean useCallback)277 extern "C" jobject Java_android_media_decoder_cts_NativeDecoderTest_getDecodedDataNative(
278 JNIEnv *env, jclass /*clazz*/, int fd, jlong offset, jlong size, jboolean wrapFd,
279 jboolean useCallback) {
280 ALOGV("getDecodedDataNative");
281
282 FdDataSource fdSrc(fd, offset, size);
283 AMediaExtractor *ex = AMediaExtractor_new();
284 AMediaDataSource *ndkSrc = AMediaDataSource_new();
285
286 int err;
287 if (wrapFd) {
288 AMediaDataSource_setUserdata(ndkSrc, &fdSrc);
289 AMediaDataSource_setReadAt(ndkSrc, FdSourceReadAt);
290 AMediaDataSource_setGetSize(ndkSrc, FdSourceGetSize);
291 AMediaDataSource_setClose(ndkSrc, FdSourceClose);
292 err = AMediaExtractor_setDataSourceCustom(ex, ndkSrc);
293 } else {
294 err = AMediaExtractor_setDataSourceFd(ex, fd, offset, size);
295 }
296 if (err != 0) {
297 ALOGE("setDataSource error: %d", err);
298 return NULL;
299 }
300
301 int numtracks = AMediaExtractor_getTrackCount(ex);
302
303 AMediaCodec **codec = new AMediaCodec*[numtracks];
304 AMediaFormat **format = new AMediaFormat*[numtracks];
305 memset(format, 0, sizeof(AMediaFormat*) * numtracks);
306 bool *sawInputEOS = new bool[numtracks];
307 bool *sawOutputEOS = new bool[numtracks];
308 simplevector<int> *sizes = new simplevector<int>[numtracks];
309 CallbackData *callbackData = new CallbackData[numtracks];
310
311 ALOGV("input has %d tracks", numtracks);
312 for (int i = 0; i < numtracks; i++) {
313 AMediaFormat *format = AMediaExtractor_getTrackFormat(ex, i);
314 const char *s = AMediaFormat_toString(format);
315 ALOGI("track %d format: %s", i, s);
316 const char *mime;
317 if (!AMediaFormat_getString(format, AMEDIAFORMAT_KEY_MIME, &mime)) {
318 ALOGE("no mime type");
319 return NULL;
320 } else if (!strncmp(mime, "audio/", 6) || !strncmp(mime, "video/", 6)) {
321 codec[i] = AMediaCodec_createDecoderByType(mime);
322 AMediaCodec_configure(codec[i], format, NULL /* surface */, NULL /* crypto */, 0);
323 if (useCallback) {
324 AMediaCodecOnAsyncNotifyCallback aCB = {
325 OnInputAvailableCB,
326 OnOutputAvailableCB,
327 OnFormatChangedCB,
328 OnErrorCB
329 };
330 AMediaCodec_setAsyncNotifyCallback(codec[i], aCB, &callbackData[i]);
331 }
332 AMediaCodec_start(codec[i]);
333 sawInputEOS[i] = false;
334 sawOutputEOS[i] = false;
335 } else {
336 ALOGE("expected audio or video mime type, got %s", mime);
337 return NULL;
338 }
339 AMediaFormat_delete(format);
340 AMediaExtractor_selectTrack(ex, i);
341 }
342 int eosCount = 0;
343 while(eosCount < numtracks) {
344 int t = AMediaExtractor_getSampleTrackIndex(ex);
345 if (t >=0) {
346 ssize_t bufidx;
347 if (useCallback) {
348 bufidx = callbackData[t].getInputBufferId();
349 } else {
350 bufidx = AMediaCodec_dequeueInputBuffer(codec[t], 5000);
351 }
352 ALOGV("track %d, input buffer %zd", t, bufidx);
353 if (bufidx >= 0) {
354 size_t bufsize;
355 uint8_t *buf = AMediaCodec_getInputBuffer(codec[t], bufidx, &bufsize);
356 int sampleSize = AMediaExtractor_readSampleData(ex, buf, bufsize);
357 ALOGV("read %d", sampleSize);
358 if (sampleSize < 0) {
359 sampleSize = 0;
360 sawInputEOS[t] = true;
361 ALOGV("EOS");
362 //break;
363 }
364 int64_t presentationTimeUs = AMediaExtractor_getSampleTime(ex);
365
366 AMediaCodec_queueInputBuffer(codec[t], bufidx, 0, sampleSize, presentationTimeUs,
367 sawInputEOS[t] ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0);
368 AMediaExtractor_advance(ex);
369 }
370 } else {
371 ALOGV("@@@@ no more input samples");
372 for (int tt = 0; tt < numtracks; tt++) {
373 if (!sawInputEOS[tt]) {
374 // we ran out of samples without ever signaling EOS to the codec,
375 // so do that now
376 int bufidx;
377 if (useCallback) {
378 bufidx = callbackData[tt].getInputBufferId();
379 } else {
380 bufidx = AMediaCodec_dequeueInputBuffer(codec[tt], 5000);
381 }
382 if (bufidx >= 0) {
383 AMediaCodec_queueInputBuffer(codec[tt], bufidx, 0, 0, 0,
384 AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM);
385 sawInputEOS[tt] = true;
386 }
387 }
388 }
389 }
390
391 // check all codecs for available data
392 AMediaCodecBufferInfo info;
393 AMediaFormat *outputFormat;
394 for (int tt = 0; tt < numtracks; tt++) {
395 if (!sawOutputEOS[tt]) {
396 int status;
397 if (useCallback) {
398 status = callbackData[tt].getOutput(&info, &outputFormat);
399 } else {
400 status = AMediaCodec_dequeueOutputBuffer(codec[tt], &info, 1);
401 }
402 ALOGV("dequeueoutput on track %d: %d", tt, status);
403 if (status >= 0) {
404 if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
405 ALOGV("EOS on track %d", tt);
406 sawOutputEOS[tt] = true;
407 eosCount++;
408 }
409 ALOGV("got decoded buffer for track %d, size %d", tt, info.size);
410 if (info.size > 0) {
411 size_t bufsize;
412 uint8_t *buf = AMediaCodec_getOutputBuffer(codec[tt], status, &bufsize);
413 int adler = checksum(buf, info.size, format[tt]);
414 sizes[tt].add(adler);
415 }
416 AMediaCodec_releaseOutputBuffer(codec[tt], status, false);
417 } else if (status == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
418 ALOGV("output buffers changed for track %d", tt);
419 } else if (status == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
420 if (format[tt] != NULL) {
421 AMediaFormat_delete(format[tt]);
422 }
423 if (useCallback) {
424 format[tt] = outputFormat;
425 } else {
426 format[tt] = AMediaCodec_getOutputFormat(codec[tt]);
427 }
428 ALOGV("format changed for track %d: %s", tt, AMediaFormat_toString(format[tt]));
429 } else if (status == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
430 ALOGV("no output buffer right now for track %d", tt);
431 } else {
432 ALOGV("unexpected info code for track %d : %d", tt, status);
433 }
434 } else {
435 ALOGV("already at EOS on track %d", tt);
436 }
437 }
438 }
439 ALOGV("decoding loop done");
440
441 // allocate java int array for result and return it
442 int numsamples = 0;
443 for (int i = 0; i < numtracks; i++) {
444 numsamples += sizes[i].size();
445 }
446 ALOGV("checksums: %d", numsamples);
447 jintArray ret = env->NewIntArray(numsamples);
448 jboolean isCopy;
449 jint *org = env->GetIntArrayElements(ret, &isCopy);
450 jint *dst = org;
451 for (int i = 0; i < numtracks; i++) {
452 int *data = sizes[i].data();
453 int len = sizes[i].size();
454 ALOGV("copying %d", len);
455 for (int j = 0; j < len; j++) {
456 *dst++ = data[j];
457 }
458 }
459 env->ReleaseIntArrayElements(ret, org, 0);
460
461 delete[] callbackData;
462 delete[] sizes;
463 delete[] sawOutputEOS;
464 delete[] sawInputEOS;
465 for (int i = 0; i < numtracks; i++) {
466 AMediaFormat_delete(format[i]);
467 AMediaCodec_stop(codec[i]);
468 AMediaCodec_delete(codec[i]);
469 }
470 delete[] format;
471 delete[] codec;
472 AMediaExtractor_delete(ex);
473 AMediaDataSource_delete(ndkSrc);
474 return ret;
475 }
476