1 /*
2 * Copyright 2015, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "MediaSync-JNI"
19 #include <utils/Log.h>
20
21 #include "android_media_MediaSync.h"
22
23 #include "android_media_AudioTrack.h"
24 #include "android_media_PlaybackParams.h"
25 #include "android_media_SyncParams.h"
26 #include "android_runtime/AndroidRuntime.h"
27 #include "android_runtime/android_view_Surface.h"
28 #include "jni.h"
29 #include <nativehelper/JNIHelp.h>
30
31 #include <gui/Surface.h>
32
33 #include <media/AudioResamplerPublic.h>
34 #include <media/AudioTrack.h>
35 #include <media/stagefright/MediaClock.h>
36 #include <media/stagefright/MediaSync.h>
37 #include <media/stagefright/foundation/ADebug.h>
38 #include <media/stagefright/foundation/AString.h>
39
40 #include <nativehelper/ScopedLocalRef.h>
41
42 namespace android {
43
44 struct fields_t {
45 jfieldID context;
46 jfieldID mediaTimestampMediaTimeUsID;
47 jfieldID mediaTimestampNanoTimeID;
48 jfieldID mediaTimestampClockRateID;
49 };
50
51 static fields_t gFields;
52 static PlaybackParams::fields_t gPlaybackParamsFields;
53 static SyncParams::fields_t gSyncParamsFields;
54
55 ////////////////////////////////////////////////////////////////////////////////
56
JMediaSync()57 JMediaSync::JMediaSync() {
58 mSync = MediaSync::create();
59 }
60
~JMediaSync()61 JMediaSync::~JMediaSync() {
62 }
63
setSurface(const sp<IGraphicBufferProducer> & bufferProducer)64 status_t JMediaSync::setSurface(const sp<IGraphicBufferProducer> &bufferProducer) {
65 return mSync->setSurface(bufferProducer);
66 }
67
setAudioTrack(const sp<AudioTrack> & audioTrack)68 status_t JMediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
69 return mSync->setAudioTrack(audioTrack);
70 }
71
createInputSurface(sp<IGraphicBufferProducer> * bufferProducer)72 status_t JMediaSync::createInputSurface(
73 sp<IGraphicBufferProducer>* bufferProducer) {
74 return mSync->createInputSurface(bufferProducer);
75 }
76
getMediaClock()77 sp<const MediaClock> JMediaSync::getMediaClock() {
78 return mSync->getMediaClock();
79 }
80
setPlaybackParams(const AudioPlaybackRate & rate)81 status_t JMediaSync::setPlaybackParams(const AudioPlaybackRate& rate) {
82 return mSync->setPlaybackSettings(rate);
83 }
84
getPlaybackParams(AudioPlaybackRate * rate)85 void JMediaSync::getPlaybackParams(AudioPlaybackRate* rate /* nonnull */) {
86 mSync->getPlaybackSettings(rate);
87 }
88
setSyncParams(const AVSyncSettings & syncParams)89 status_t JMediaSync::setSyncParams(const AVSyncSettings& syncParams) {
90 return mSync->setSyncSettings(syncParams);
91 }
92
getSyncParams(AVSyncSettings * syncParams)93 void JMediaSync::getSyncParams(AVSyncSettings* syncParams /* nonnull */) {
94 mSync->getSyncSettings(syncParams);
95 }
96
setVideoFrameRateHint(float rate)97 status_t JMediaSync::setVideoFrameRateHint(float rate) {
98 return mSync->setVideoFrameRateHint(rate);
99 }
100
getVideoFrameRate()101 float JMediaSync::getVideoFrameRate() {
102 return mSync->getVideoFrameRate();
103 }
104
flush()105 void JMediaSync::flush() {
106 mSync->flush();
107 }
108
updateQueuedAudioData(int sizeInBytes,int64_t presentationTimeUs)109 status_t JMediaSync::updateQueuedAudioData(
110 int sizeInBytes, int64_t presentationTimeUs) {
111 return mSync->updateQueuedAudioData(sizeInBytes, presentationTimeUs);
112 }
113
getPlayTimeForPendingAudioFrames(int64_t * outTimeUs)114 status_t JMediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
115 return mSync->getPlayTimeForPendingAudioFrames(outTimeUs);
116 }
117
118 } // namespace android
119
120 ////////////////////////////////////////////////////////////////////////////////
121
122 using namespace android;
123
setMediaSync(JNIEnv * env,jobject thiz,const sp<JMediaSync> & sync)124 static sp<JMediaSync> setMediaSync(JNIEnv *env, jobject thiz, const sp<JMediaSync> &sync) {
125 sp<JMediaSync> old = (JMediaSync *)env->GetLongField(thiz, gFields.context);
126 if (sync != NULL) {
127 sync->incStrong(thiz);
128 }
129 if (old != NULL) {
130 old->decStrong(thiz);
131 }
132
133 env->SetLongField(thiz, gFields.context, (jlong)sync.get());
134
135 return old;
136 }
137
getMediaSync(JNIEnv * env,jobject thiz)138 static sp<JMediaSync> getMediaSync(JNIEnv *env, jobject thiz) {
139 return (JMediaSync *)env->GetLongField(thiz, gFields.context);
140 }
141
android_media_MediaSync_release(JNIEnv * env,jobject thiz)142 static void android_media_MediaSync_release(JNIEnv *env, jobject thiz) {
143 setMediaSync(env, thiz, NULL);
144 }
145
throwExceptionAsNecessary(JNIEnv * env,status_t err,const char * msg=NULL)146 static void throwExceptionAsNecessary(
147 JNIEnv *env, status_t err, const char *msg = NULL) {
148 switch (err) {
149 case NO_ERROR:
150 break;
151
152 case BAD_VALUE:
153 jniThrowException(env, "java/lang/IllegalArgumentException", msg);
154 break;
155
156 case NO_INIT:
157 case INVALID_OPERATION:
158 default:
159 if (err > 0) {
160 break;
161 }
162 AString msgWithErrorCode(msg == NULL ? "" : msg);
163 msgWithErrorCode.append(" error:");
164 msgWithErrorCode.append(err);
165 jniThrowException(env, "java/lang/IllegalStateException", msgWithErrorCode.c_str());
166 break;
167 }
168 }
169
android_media_MediaSync_native_setSurface(JNIEnv * env,jobject thiz,jobject jsurface)170 static void android_media_MediaSync_native_setSurface(
171 JNIEnv *env, jobject thiz, jobject jsurface) {
172 ALOGV("android_media_MediaSync_setSurface");
173
174 sp<JMediaSync> sync = getMediaSync(env, thiz);
175 if (sync == NULL) {
176 throwExceptionAsNecessary(env, INVALID_OPERATION);
177 return;
178 }
179
180 sp<IGraphicBufferProducer> bufferProducer;
181 if (jsurface != NULL) {
182 sp<Surface> surface(android_view_Surface_getSurface(env, jsurface));
183 if (surface != NULL) {
184 bufferProducer = surface->getIGraphicBufferProducer();
185 } else {
186 throwExceptionAsNecessary(env, BAD_VALUE, "The surface has been released");
187 return;
188 }
189 }
190
191 status_t err = sync->setSurface(bufferProducer);
192
193 if (err == INVALID_OPERATION) {
194 throwExceptionAsNecessary(
195 env, INVALID_OPERATION, "Surface has already been configured");
196 } if (err != NO_ERROR) {
197 AString msg("Failed to connect to surface with error ");
198 msg.append(err);
199 throwExceptionAsNecessary(env, BAD_VALUE, msg.c_str());
200 }
201 }
202
android_media_MediaSync_native_setAudioTrack(JNIEnv * env,jobject thiz,jobject jaudioTrack)203 static void android_media_MediaSync_native_setAudioTrack(
204 JNIEnv *env, jobject thiz, jobject jaudioTrack) {
205 ALOGV("android_media_MediaSync_setAudioTrack");
206
207 sp<JMediaSync> sync = getMediaSync(env, thiz);
208 if (sync == NULL) {
209 throwExceptionAsNecessary(env, INVALID_OPERATION);
210 return;
211 }
212
213 sp<AudioTrack> audioTrack;
214 if (jaudioTrack != NULL) {
215 audioTrack = android_media_AudioTrack_getAudioTrack(env, jaudioTrack);
216 if (audioTrack == NULL) {
217 throwExceptionAsNecessary(env, BAD_VALUE, "The audio track has been released");
218 return;
219 }
220 }
221
222 status_t err = sync->setAudioTrack(audioTrack);
223
224 if (err == INVALID_OPERATION) {
225 throwExceptionAsNecessary(
226 env, INVALID_OPERATION, "Audio track has already been configured");
227 } if (err != NO_ERROR) {
228 AString msg("Failed to configure audio track with error ");
229 msg.append(err);
230 throwExceptionAsNecessary(env, BAD_VALUE, msg.c_str());
231 }
232 }
233
android_media_MediaSync_createInputSurface(JNIEnv * env,jobject thiz)234 static jobject android_media_MediaSync_createInputSurface(
235 JNIEnv* env, jobject thiz) {
236 ALOGV("android_media_MediaSync_createInputSurface");
237
238 sp<JMediaSync> sync = getMediaSync(env, thiz);
239 if (sync == NULL) {
240 throwExceptionAsNecessary(env, INVALID_OPERATION);
241 return NULL;
242 }
243
244 // Tell the MediaSync that we want to use a Surface as input.
245 sp<IGraphicBufferProducer> bufferProducer;
246 status_t err = sync->createInputSurface(&bufferProducer);
247 if (err != NO_ERROR) {
248 throwExceptionAsNecessary(env, INVALID_OPERATION);
249 return NULL;
250 }
251
252 // Wrap the IGBP in a Java-language Surface.
253 return android_view_Surface_createFromIGraphicBufferProducer(env,
254 bufferProducer);
255 }
256
android_media_MediaSync_native_updateQueuedAudioData(JNIEnv * env,jobject thiz,jint sizeInBytes,jlong presentationTimeUs)257 static void android_media_MediaSync_native_updateQueuedAudioData(
258 JNIEnv *env, jobject thiz, jint sizeInBytes, jlong presentationTimeUs) {
259 sp<JMediaSync> sync = getMediaSync(env, thiz);
260 if (sync == NULL) {
261 throwExceptionAsNecessary(env, INVALID_OPERATION);
262 return;
263 }
264
265 status_t err = sync->updateQueuedAudioData(sizeInBytes, presentationTimeUs);
266 if (err != NO_ERROR) {
267 throwExceptionAsNecessary(env, err);
268 return;
269 }
270 }
271
android_media_MediaSync_native_getTimestamp(JNIEnv * env,jobject thiz,jobject timestamp)272 static jboolean android_media_MediaSync_native_getTimestamp(
273 JNIEnv *env, jobject thiz, jobject timestamp) {
274 sp<JMediaSync> sync = getMediaSync(env, thiz);
275 if (sync == NULL) {
276 throwExceptionAsNecessary(env, INVALID_OPERATION);
277 return JNI_FALSE;
278 }
279
280 sp<const MediaClock> mediaClock = sync->getMediaClock();
281 if (mediaClock == NULL) {
282 return JNI_FALSE;
283 }
284
285 int64_t nowUs = ALooper::GetNowUs();
286 int64_t mediaUs = 0;
287 if (mediaClock->getMediaTime(nowUs, &mediaUs) != OK) {
288 return JNI_FALSE;
289 }
290
291 env->SetLongField(timestamp, gFields.mediaTimestampMediaTimeUsID,
292 (jlong)mediaUs);
293 env->SetLongField(timestamp, gFields.mediaTimestampNanoTimeID,
294 (jlong)(nowUs * 1000));
295 env->SetFloatField(timestamp, gFields.mediaTimestampClockRateID,
296 (jfloat)mediaClock->getPlaybackRate());
297 return JNI_TRUE;
298 }
299
android_media_MediaSync_native_getPlayTimeForPendingAudioFrames(JNIEnv * env,jobject thiz)300 static jlong android_media_MediaSync_native_getPlayTimeForPendingAudioFrames(
301 JNIEnv *env, jobject thiz) {
302 sp<JMediaSync> sync = getMediaSync(env, thiz);
303 if (sync == NULL) {
304 throwExceptionAsNecessary(env, INVALID_OPERATION);
305 }
306
307 int64_t playTimeUs = 0;
308 status_t err = sync->getPlayTimeForPendingAudioFrames(&playTimeUs);
309 if (err != NO_ERROR) {
310 throwExceptionAsNecessary(env, err);
311 }
312 return (jlong)playTimeUs;
313 }
314
android_media_MediaSync_setPlaybackParams(JNIEnv * env,jobject thiz,jobject params)315 static jfloat android_media_MediaSync_setPlaybackParams(
316 JNIEnv *env, jobject thiz, jobject params) {
317 sp<JMediaSync> sync = getMediaSync(env, thiz);
318 if (sync == NULL) {
319 throwExceptionAsNecessary(env, INVALID_OPERATION);
320 return (jfloat)0.f;
321 }
322
323 PlaybackParams pbs;
324 pbs.fillFromJobject(env, gPlaybackParamsFields, params);
325 ALOGV("setPlaybackParams: %d:%f %d:%f %d:%u %d:%u",
326 pbs.speedSet, pbs.audioRate.mSpeed,
327 pbs.pitchSet, pbs.audioRate.mPitch,
328 pbs.audioFallbackModeSet, pbs.audioRate.mFallbackMode,
329 pbs.audioStretchModeSet, pbs.audioRate.mStretchMode);
330
331 AudioPlaybackRate rate;
332 sync->getPlaybackParams(&rate);
333 bool updatedRate = false;
334 if (pbs.speedSet) {
335 rate.mSpeed = pbs.audioRate.mSpeed;
336 updatedRate = true;
337 }
338 if (pbs.pitchSet) {
339 rate.mPitch = pbs.audioRate.mPitch;
340 updatedRate = true;
341 }
342 if (pbs.audioFallbackModeSet) {
343 rate.mFallbackMode = pbs.audioRate.mFallbackMode;
344 updatedRate = true;
345 }
346 if (pbs.audioStretchModeSet) {
347 rate.mStretchMode = pbs.audioRate.mStretchMode;
348 updatedRate = true;
349 }
350 if (updatedRate) {
351 status_t err = sync->setPlaybackParams(rate);
352 if (err != OK) {
353 throwExceptionAsNecessary(env, err);
354 return (jfloat)0.f;
355 }
356 }
357
358 sp<const MediaClock> mediaClock = sync->getMediaClock();
359 if (mediaClock == NULL) {
360 return (jfloat)0.f;
361 }
362
363 return (jfloat)mediaClock->getPlaybackRate();
364 }
365
android_media_MediaSync_getPlaybackParams(JNIEnv * env,jobject thiz)366 static jobject android_media_MediaSync_getPlaybackParams(
367 JNIEnv *env, jobject thiz) {
368 sp<JMediaSync> sync = getMediaSync(env, thiz);
369 if (sync == NULL) {
370 throwExceptionAsNecessary(env, INVALID_OPERATION);
371 return NULL;
372 }
373
374 PlaybackParams pbs;
375 AudioPlaybackRate &audioRate = pbs.audioRate;
376 sync->getPlaybackParams(&audioRate);
377 ALOGV("getPlaybackParams: %f %f %d %d",
378 audioRate.mSpeed, audioRate.mPitch, audioRate.mFallbackMode, audioRate.mStretchMode);
379
380 pbs.speedSet = true;
381 pbs.pitchSet = true;
382 pbs.audioFallbackModeSet = true;
383 pbs.audioStretchModeSet = true;
384
385 return pbs.asJobject(env, gPlaybackParamsFields);
386 }
387
android_media_MediaSync_setSyncParams(JNIEnv * env,jobject thiz,jobject params)388 static jfloat android_media_MediaSync_setSyncParams(
389 JNIEnv *env, jobject thiz, jobject params) {
390 sp<JMediaSync> sync = getMediaSync(env, thiz);
391 if (sync == NULL) {
392 throwExceptionAsNecessary(env, INVALID_OPERATION);
393 return (jfloat)0.f;
394 }
395
396 SyncParams scs;
397 scs.fillFromJobject(env, gSyncParamsFields, params);
398 ALOGV("setSyncParams: %d:%d %d:%d %d:%f %d:%f",
399 scs.syncSourceSet, scs.sync.mSource,
400 scs.audioAdjustModeSet, scs.sync.mAudioAdjustMode,
401 scs.toleranceSet, scs.sync.mTolerance,
402 scs.frameRateSet, scs.frameRate);
403
404 AVSyncSettings avsync;
405 sync->getSyncParams(&avsync);
406 bool updatedSync = false;
407 status_t err = OK;
408 if (scs.syncSourceSet) {
409 avsync.mSource = scs.sync.mSource;
410 updatedSync = true;
411 }
412 if (scs.audioAdjustModeSet) {
413 avsync.mAudioAdjustMode = scs.sync.mAudioAdjustMode;
414 updatedSync = true;
415 }
416 if (scs.toleranceSet) {
417 avsync.mTolerance = scs.sync.mTolerance;
418 updatedSync = true;
419 }
420 if (updatedSync) {
421 err = sync->setSyncParams(avsync);
422 }
423
424 if (scs.frameRateSet && err == OK) {
425 err = sync->setVideoFrameRateHint(scs.frameRate);
426 }
427 if (err != OK) {
428 throwExceptionAsNecessary(env, err);
429 return (jfloat)0.f;
430 }
431
432 sp<const MediaClock> mediaClock = sync->getMediaClock();
433 if (mediaClock == NULL) {
434 return (jfloat)0.f;
435 }
436
437 return (jfloat)mediaClock->getPlaybackRate();
438 }
439
android_media_MediaSync_getSyncParams(JNIEnv * env,jobject thiz)440 static jobject android_media_MediaSync_getSyncParams(JNIEnv *env, jobject thiz) {
441 sp<JMediaSync> sync = getMediaSync(env, thiz);
442 if (sync == NULL) {
443 throwExceptionAsNecessary(env, INVALID_OPERATION);
444 return NULL;
445 }
446
447 SyncParams scs;
448 sync->getSyncParams(&scs.sync);
449 scs.frameRate = sync->getVideoFrameRate();
450
451 ALOGV("getSyncParams: %d %d %f %f",
452 scs.sync.mSource, scs.sync.mAudioAdjustMode, scs.sync.mTolerance, scs.frameRate);
453
454 // check params
455 if (scs.sync.mSource >= AVSYNC_SOURCE_MAX
456 || scs.sync.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
457 || scs.sync.mTolerance < 0.f
458 || scs.sync.mTolerance >= AVSYNC_TOLERANCE_MAX) {
459 throwExceptionAsNecessary(env, INVALID_OPERATION);
460 return NULL;
461 }
462
463 scs.syncSourceSet = true;
464 scs.audioAdjustModeSet = true;
465 scs.toleranceSet = true;
466 scs.frameRateSet = scs.frameRate >= 0.f;
467
468 return scs.asJobject(env, gSyncParamsFields);
469 }
470
android_media_MediaSync_native_flush(JNIEnv * env,jobject thiz)471 static void android_media_MediaSync_native_flush(JNIEnv *env, jobject thiz) {
472 sp<JMediaSync> sync = getMediaSync(env, thiz);
473 if (sync == NULL) {
474 throwExceptionAsNecessary(env, INVALID_OPERATION);
475 return;
476 }
477
478 sync->flush();
479 }
480
android_media_MediaSync_native_init(JNIEnv * env)481 static void android_media_MediaSync_native_init(JNIEnv *env) {
482 ScopedLocalRef<jclass> clazz(env, env->FindClass("android/media/MediaSync"));
483 CHECK(clazz.get() != NULL);
484
485 gFields.context = env->GetFieldID(clazz.get(), "mNativeContext", "J");
486 CHECK(gFields.context != NULL);
487
488 clazz.reset(env->FindClass("android/media/MediaTimestamp"));
489 CHECK(clazz.get() != NULL);
490
491 gFields.mediaTimestampMediaTimeUsID =
492 env->GetFieldID(clazz.get(), "mediaTimeUs", "J");
493 CHECK(gFields.mediaTimestampMediaTimeUsID != NULL);
494
495 gFields.mediaTimestampNanoTimeID =
496 env->GetFieldID(clazz.get(), "nanoTime", "J");
497 CHECK(gFields.mediaTimestampNanoTimeID != NULL);
498
499 gFields.mediaTimestampClockRateID =
500 env->GetFieldID(clazz.get(), "clockRate", "F");
501 CHECK(gFields.mediaTimestampClockRateID != NULL);
502
503 gSyncParamsFields.init(env);
504 gPlaybackParamsFields.init(env);
505 }
506
android_media_MediaSync_native_setup(JNIEnv * env,jobject thiz)507 static void android_media_MediaSync_native_setup(JNIEnv *env, jobject thiz) {
508 sp<JMediaSync> sync = new JMediaSync();
509
510 setMediaSync(env, thiz, sync);
511 }
512
android_media_MediaSync_native_finalize(JNIEnv * env,jobject thiz)513 static void android_media_MediaSync_native_finalize(JNIEnv *env, jobject thiz) {
514 android_media_MediaSync_release(env, thiz);
515 }
516
517 static JNINativeMethod gMethods[] = {
518 { "native_setSurface",
519 "(Landroid/view/Surface;)V",
520 (void *)android_media_MediaSync_native_setSurface },
521
522 { "native_setAudioTrack",
523 "(Landroid/media/AudioTrack;)V",
524 (void *)android_media_MediaSync_native_setAudioTrack },
525
526 { "createInputSurface", "()Landroid/view/Surface;",
527 (void *)android_media_MediaSync_createInputSurface },
528
529 { "native_updateQueuedAudioData",
530 "(IJ)V",
531 (void *)android_media_MediaSync_native_updateQueuedAudioData },
532
533 { "native_getTimestamp",
534 "(Landroid/media/MediaTimestamp;)Z",
535 (void *)android_media_MediaSync_native_getTimestamp },
536
537 { "native_getPlayTimeForPendingAudioFrames",
538 "()J",
539 (void *)android_media_MediaSync_native_getPlayTimeForPendingAudioFrames },
540
541 { "native_flush", "()V", (void *)android_media_MediaSync_native_flush },
542
543 { "native_init", "()V", (void *)android_media_MediaSync_native_init },
544
545 { "native_setup", "()V", (void *)android_media_MediaSync_native_setup },
546
547 { "native_release", "()V", (void *)android_media_MediaSync_release },
548
549 { "native_setPlaybackParams", "(Landroid/media/PlaybackParams;)F",
550 (void *)android_media_MediaSync_setPlaybackParams },
551
552 { "getPlaybackParams", "()Landroid/media/PlaybackParams;",
553 (void *)android_media_MediaSync_getPlaybackParams },
554
555 { "native_setSyncParams", "(Landroid/media/SyncParams;)F",
556 (void *)android_media_MediaSync_setSyncParams },
557
558 { "getSyncParams", "()Landroid/media/SyncParams;",
559 (void *)android_media_MediaSync_getSyncParams },
560
561 { "native_finalize", "()V", (void *)android_media_MediaSync_native_finalize },
562 };
563
register_android_media_MediaSync(JNIEnv * env)564 int register_android_media_MediaSync(JNIEnv *env) {
565 return AndroidRuntime::registerNativeMethods(
566 env, "android/media/MediaSync", gMethods, NELEM(gMethods));
567 }
568