1 /* 2 * Copyright 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package androidx.camera.integration.extensions; 17 18 import static android.os.Environment.getExternalStoragePublicDirectory; 19 20 import static androidx.camera.core.ImageCapture.ERROR_CAMERA_CLOSED; 21 import static androidx.camera.core.ImageCapture.ERROR_CAPTURE_FAILED; 22 import static androidx.camera.core.ImageCapture.ERROR_FILE_IO; 23 import static androidx.camera.core.ImageCapture.ERROR_INVALID_CAMERA; 24 import static androidx.camera.core.ImageCapture.ERROR_UNKNOWN; 25 import static androidx.camera.core.ImageCapture.OUTPUT_FORMAT_JPEG; 26 import static androidx.camera.core.ImageCapture.OUTPUT_FORMAT_JPEG_ULTRA_HDR; 27 import static androidx.camera.core.ImageCapture.OUTPUT_FORMAT_RAW; 28 import static androidx.camera.core.ImageCapture.OUTPUT_FORMAT_RAW_JPEG; 29 import static androidx.camera.core.ImageCapture.getImageCaptureCapabilities; 30 import static androidx.camera.integration.extensions.CameraDirection.BACKWARD; 31 import static androidx.camera.integration.extensions.CameraDirection.FORWARD; 32 import static androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_CAMERA_IMPLEMENTATION; 33 import static androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_CAMERA_DIRECTION; 34 import static androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_CAMERA_ID; 35 import static androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_DELETE_CAPTURED_IMAGE; 36 import static androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_EXTENSION_MODE; 37 import static androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_OUTPUT_FORMAT; 38 import static androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_VIDEO_CAPTURE_ENABLED; 39 import static androidx.camera.integration.extensions.utils.PermissionUtil.setupPermissions; 40 import static androidx.camera.video.VideoRecordEvent.Finalize.ERROR_DURATION_LIMIT_REACHED; 41 import static androidx.camera.video.VideoRecordEvent.Finalize.ERROR_FILE_SIZE_LIMIT_REACHED; 42 import static androidx.camera.video.VideoRecordEvent.Finalize.ERROR_INSUFFICIENT_STORAGE; 43 import static androidx.camera.video.VideoRecordEvent.Finalize.ERROR_NONE; 44 import static androidx.camera.video.VideoRecordEvent.Finalize.ERROR_SOURCE_INACTIVE; 45 import static androidx.core.util.Preconditions.checkNotNull; 46 47 import static java.util.concurrent.TimeUnit.NANOSECONDS; 48 49 import android.annotation.SuppressLint; 50 import android.content.ContentValues; 51 import android.content.Intent; 52 import android.content.pm.PackageManager; 53 import android.graphics.Bitmap; 54 import android.hardware.camera2.CameraCaptureSession; 55 import android.hardware.camera2.CaptureRequest; 56 import android.net.Uri; 57 import android.os.Build; 58 import android.os.Bundle; 59 import android.os.Environment; 60 import android.os.StrictMode; 61 import android.os.SystemClock; 62 import android.provider.MediaStore; 63 import android.util.Log; 64 import android.util.Pair; 65 import android.view.Menu; 66 import android.view.MenuInflater; 67 import android.view.MenuItem; 68 import android.view.MotionEvent; 69 import android.view.ScaleGestureDetector; 70 import android.view.View; 71 import android.view.ViewStub; 72 import android.widget.Button; 73 import android.widget.PopupMenu; 74 import android.widget.TextView; 75 import android.widget.Toast; 76 import android.widget.ToggleButton; 77 78 import androidx.annotation.OptIn; 79 import androidx.annotation.VisibleForTesting; 80 import androidx.appcompat.app.AppCompatActivity; 81 import androidx.camera.camera2.interop.Camera2Interop; 82 import androidx.camera.camera2.interop.ExperimentalCamera2Interop; 83 import androidx.camera.camera2.pipe.integration.CameraPipeConfig; 84 import androidx.camera.core.Camera; 85 import androidx.camera.core.CameraControl; 86 import androidx.camera.core.CameraInfo; 87 import androidx.camera.core.CameraSelector; 88 import androidx.camera.core.FocusMeteringAction; 89 import androidx.camera.core.FocusMeteringResult; 90 import androidx.camera.core.ImageCapture; 91 import androidx.camera.core.ImageCaptureCapabilities; 92 import androidx.camera.core.ImageCaptureException; 93 import androidx.camera.core.MeteringPoint; 94 import androidx.camera.core.Preview; 95 import androidx.camera.core.UseCaseGroup; 96 import androidx.camera.core.impl.utils.executor.CameraXExecutors; 97 import androidx.camera.extensions.ExtensionMode; 98 import androidx.camera.extensions.ExtensionsManager; 99 import androidx.camera.integration.extensions.utils.CameraSelectorUtil; 100 import androidx.camera.integration.extensions.utils.ExtensionModeUtil; 101 import androidx.camera.integration.extensions.utils.FpsRecorder; 102 import androidx.camera.integration.extensions.validation.CameraValidationResultActivity; 103 import androidx.camera.lifecycle.ProcessCameraProvider; 104 import androidx.camera.video.MediaStoreOutputOptions; 105 import androidx.camera.video.PendingRecording; 106 import androidx.camera.video.Recorder; 107 import androidx.camera.video.Recording; 108 import androidx.camera.video.RecordingStats; 109 import androidx.camera.video.VideoCapture; 110 import androidx.camera.video.VideoRecordEvent; 111 import androidx.camera.view.PreviewView; 112 import androidx.concurrent.futures.CallbackToFutureAdapter; 113 import androidx.core.app.ActivityCompat; 114 import androidx.core.content.ContextCompat; 115 import androidx.core.math.MathUtils; 116 import androidx.core.util.Consumer; 117 import androidx.lifecycle.Lifecycle; 118 import androidx.test.espresso.idling.CountingIdlingResource; 119 120 import com.google.common.base.Preconditions; 121 import com.google.common.util.concurrent.FutureCallback; 122 import com.google.common.util.concurrent.Futures; 123 import com.google.common.util.concurrent.ListenableFuture; 124 125 import org.jspecify.annotations.NonNull; 126 import org.jspecify.annotations.Nullable; 127 128 import java.io.File; 129 import java.text.Format; 130 import java.text.SimpleDateFormat; 131 import java.util.ArrayList; 132 import java.util.Calendar; 133 import java.util.HashMap; 134 import java.util.List; 135 import java.util.Locale; 136 import java.util.Map; 137 import java.util.Set; 138 139 /** An activity that shows off how extensions can be applied */ 140 public class CameraExtensionsActivity extends AppCompatActivity 141 implements ActivityCompat.OnRequestPermissionsResultCallback { 142 143 private static final String TAG = "CameraExtensionActivity"; 144 private static final int PERMISSIONS_REQUEST_CODE = 42; 145 public static final String CAMERA2_IMPLEMENTATION_OPTION = "camera2"; 146 public static final String CAMERA_PIPE_IMPLEMENTATION_OPTION = "camera_pipe"; 147 148 private CameraSelector mCurrentCameraSelector = CameraSelector.DEFAULT_BACK_CAMERA; 149 150 boolean mPermissionsGranted = false; 151 private CallbackToFutureAdapter.Completer<Boolean> mPermissionCompleter; 152 153 private @Nullable Preview mPreview; 154 155 private @Nullable ImageCapture mImageCapture; 156 157 private @Nullable VideoCapture<Recorder> mVideoCapture = null; 158 159 private @Nullable Recording mActiveRecording = null; 160 161 @ExtensionMode.Mode 162 private int mCurrentExtensionMode = ExtensionMode.BOKEH; 163 164 // Espresso testing variables 165 private final CountingIdlingResource mInitializationIdlingResource = new CountingIdlingResource( 166 "Initialization"); 167 168 private final CountingIdlingResource mTakePictureIdlingResource = new CountingIdlingResource( 169 "TakePicture"); 170 171 private final CountingIdlingResource mPostviewIdlingResource = new CountingIdlingResource( 172 "Postview"); 173 174 private final CountingIdlingResource mPreviewViewStreamingStateIdlingResource = 175 new CountingIdlingResource("PreviewView-Streaming"); 176 177 private final CountingIdlingResource mPreviewViewIdleStateIdlingResource = 178 new CountingIdlingResource("PreviewView-Idle"); 179 180 private PreviewView mPreviewView; 181 182 ProcessCameraProvider mCameraProvider; 183 184 Camera mCamera; 185 186 ExtensionsManager mExtensionsManager; 187 188 boolean mDeleteCapturedImage = false; 189 190 // < Sensor timestamp, current timestamp > 191 Map<Long, Long> mFrameTimestampMap = new HashMap<>(); 192 193 @Nullable String mFrameInfo = null; 194 195 @Nullable String mRecordingInfo = null; 196 197 String mCurrentCameraId = null; 198 199 private ToggleButton mToggleVideoCapture; 200 201 /** 202 * Saves the error message of the last take picture action if any error occurs. This will be 203 * null which means no error occurs. 204 */ 205 private @Nullable String mLastTakePictureErrorMessage = null; 206 207 private PreviewView.StreamState mCurrentStreamState = null; 208 209 private @ImageCapture.OutputFormat int mImageOutputFormat = OUTPUT_FORMAT_JPEG; 210 private Button mButtonImageOutputFormat; 211 setupButtons()212 void setupButtons() { 213 Button btnToggleMode = findViewById(R.id.PhotoToggle); 214 Button btnSwitchCamera = findViewById(R.id.Switch); 215 btnToggleMode.setOnClickListener(view -> bindUseCasesWithNextExtensionMode()); 216 btnSwitchCamera.setOnClickListener(view -> switchCameras()); 217 218 // Setup video capture related buttons. 219 mToggleVideoCapture.setVisibility(View.VISIBLE); 220 mToggleVideoCapture.setOnCheckedChangeListener( 221 (button, isChecked) -> { 222 updateRecordingButton(); 223 bindUseCasesWithCurrentExtensionMode(); 224 } 225 ); 226 Button btnRecord = findViewById(R.id.record); 227 btnRecord.setOnClickListener(view -> { 228 if (mActiveRecording != null) { 229 stopRecording(); 230 } else { 231 startRecording(); 232 } 233 }); 234 } 235 switchCameras()236 void switchCameras() { 237 mCameraProvider.unbindAll(); 238 if (mCurrentCameraId != null) { 239 String nextCameraId = CameraSelectorUtil.findNextSupportedCameraId( 240 this, mExtensionsManager, mCurrentCameraId, mCurrentExtensionMode); 241 if (nextCameraId == null) { 242 Log.e(TAG, "Cannot find next camera id that supports the extensions mode"); 243 return; 244 } 245 mCurrentCameraSelector = CameraSelectorUtil.createCameraSelectorById(mCurrentCameraId); 246 } else { 247 mCurrentCameraSelector = (mCurrentCameraSelector == CameraSelector.DEFAULT_BACK_CAMERA) 248 ? CameraSelector.DEFAULT_FRONT_CAMERA : CameraSelector.DEFAULT_BACK_CAMERA; 249 } 250 if (!bindUseCasesWithCurrentExtensionMode()) { 251 bindUseCasesWithNextExtensionMode(); 252 } 253 } 254 255 @VisibleForTesting switchToExtensionMode(@xtensionMode.Mode int extensionMode)256 boolean switchToExtensionMode(@ExtensionMode.Mode int extensionMode) { 257 if (mCamera == null || mExtensionsManager == null) { 258 return false; 259 } 260 261 if (!mExtensionsManager.isExtensionAvailable(mCurrentCameraSelector, extensionMode)) { 262 return false; 263 } 264 265 mCurrentExtensionMode = extensionMode; 266 bindUseCasesWithCurrentExtensionMode(); 267 268 return true; 269 } 270 bindUseCasesWithNextExtensionMode()271 void bindUseCasesWithNextExtensionMode() { 272 do { 273 mCurrentExtensionMode = getNextExtensionMode(mCurrentExtensionMode); 274 } while (!bindUseCasesWithCurrentExtensionMode()); 275 } 276 277 @OptIn(markerClass = ExperimentalCamera2Interop.class) bindUseCasesWithCurrentExtensionMode()278 boolean bindUseCasesWithCurrentExtensionMode() { 279 if (!mExtensionsManager.isExtensionAvailable(mCurrentCameraSelector, 280 mCurrentExtensionMode)) { 281 return false; 282 } 283 284 mCameraProvider.unbindAll(); 285 286 CameraSelector cameraSelector = mExtensionsManager.getExtensionEnabledCameraSelector( 287 mCurrentCameraSelector, mCurrentExtensionMode); 288 289 mCamera = mCameraProvider.bindToLifecycle(this, cameraSelector); 290 291 // Reset to the default JPEG output format if the format set previously is not supported by 292 // the new extensions mode or the different lens facing camera. 293 if (!ImageCapture.getImageCaptureCapabilities( 294 mCamera.getCameraInfo()).getSupportedOutputFormats().contains(mImageOutputFormat)) { 295 mImageOutputFormat = OUTPUT_FORMAT_JPEG; 296 } 297 setUpImageOutputFormatButton(); 298 299 final boolean isPostviewSupported = ImageCapture.getImageCaptureCapabilities( 300 mCamera.getCameraInfo()).isPostviewSupported(); 301 302 resetPreviewViewStreamingStateIdlingResource(); 303 resetPreviewViewIdleStateIdlingResource(); 304 305 ImageCapture.Builder imageCaptureBuilder = new ImageCapture.Builder() 306 .setTargetName("ImageCapture") 307 .setOutputFormat(mImageOutputFormat) 308 .setPostviewEnabled(isPostviewSupported); 309 mImageCapture = imageCaptureBuilder.build(); 310 311 mFrameTimestampMap.clear(); 312 Preview.Builder previewBuilder = new Preview.Builder().setTargetName("Preview"); 313 314 new Camera2Interop.Extender<>(previewBuilder) 315 .setSessionCaptureCallback(new CameraCaptureSession.CaptureCallback() { 316 @Override 317 public void onCaptureStarted(@NonNull CameraCaptureSession session, 318 @NonNull CaptureRequest request, long timestamp, long frameNumber) { 319 mFrameTimestampMap.put(timestamp, SystemClock.elapsedRealtimeNanos()); 320 } 321 }); 322 mPreview = previewBuilder.build(); 323 mCurrentStreamState = null; 324 mPreview.setSurfaceProvider(mPreviewView.getSurfaceProvider()); 325 326 // Observes the stream state for the unit tests to know the preview status. 327 mPreviewView.getPreviewStreamState().removeObservers(this); 328 mPreviewView.getPreviewStreamState().observeForever(streamState -> { 329 mCurrentStreamState = streamState; 330 if (streamState == PreviewView.StreamState.STREAMING 331 && !mPreviewViewStreamingStateIdlingResource.isIdleNow()) { 332 mPreviewViewStreamingStateIdlingResource.decrement(); 333 } else if (streamState == PreviewView.StreamState.IDLE 334 && !mPreviewViewIdleStateIdlingResource.isIdleNow()) { 335 mPreviewViewIdleStateIdlingResource.decrement(); 336 } 337 }); 338 339 FpsRecorder fpsRecorder = new FpsRecorder(10 /* sample count */); 340 // Calls internal API PreviewView#setFrameUpdateListener to calculate the frame latency. 341 // Remove this if you copy this sample app to your project. 342 mPreviewView.setFrameUpdateListener(CameraXExecutors.directExecutor(), (timestamp) -> { 343 Long frameCapturedTime = mFrameTimestampMap.remove(timestamp); 344 if (frameCapturedTime == null) { 345 Log.e(TAG, "Cannot find frame with timestamp: " + timestamp); 346 return; 347 } 348 349 long latency = (SystemClock.elapsedRealtimeNanos() - frameCapturedTime) / 1000000L; 350 double fps = fpsRecorder.recordTimestamp(SystemClock.elapsedRealtimeNanos()); 351 String fpsText = String.format("%1$s", 352 (Double.isNaN(fps) || Double.isInfinite(fps)) ? "---" : 353 String.format(Locale.US, "%.0f", fps)); 354 mFrameInfo = "Latency:" + latency + " ms\n" + "FPS: " + fpsText; 355 updateInfoBlock(); 356 }); 357 358 UseCaseGroup.Builder useCaseGroupBuilder = 359 new UseCaseGroup.Builder() 360 .addUseCase(mPreview) 361 .addUseCase(mImageCapture); 362 363 // Setup VideoCapture. 364 stopRecording(); 365 mVideoCapture = null; 366 if (mToggleVideoCapture.isChecked()) { 367 Recorder recorder = new Recorder.Builder().build(); 368 mVideoCapture = VideoCapture.withOutput(recorder); 369 useCaseGroupBuilder.addUseCase(checkNotNull(mVideoCapture)); 370 } 371 372 mCamera = mCameraProvider.bindToLifecycle(this, cameraSelector, 373 useCaseGroupBuilder.build()); 374 375 // Update the UI and save location for ImageCapture 376 Button toggleButton = findViewById(R.id.PhotoToggle); 377 String extensionModeString = 378 ExtensionModeUtil.getExtensionModeStringFromId(mCurrentExtensionMode); 379 toggleButton.setText(extensionModeString); 380 381 Button captureButton = findViewById(R.id.Picture); 382 383 Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS", Locale.US); 384 File dir = new File( 385 getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES), 386 "ExtensionsPictures"); 387 388 captureButton.setOnClickListener((view) -> { 389 resetTakePictureIdlingResource(); 390 resetPostviewIdlingResource(); 391 392 // Makes the postview idling resource idle when it is not supported. 393 if (!isPostviewSupported && !mPostviewIdlingResource.isIdleNow()) { 394 mPostviewIdlingResource.decrement(); 395 } 396 397 String fileName = "[" + formatter.format(Calendar.getInstance().getTime()) 398 + "][CameraX]" + extensionModeString + ".jpg"; 399 File saveFile = new File(dir, fileName); 400 ImageCapture.OutputFileOptions outputFileOptions; 401 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { 402 ContentValues contentValues = new ContentValues(); 403 contentValues.put(MediaStore.MediaColumns.DISPLAY_NAME, fileName); 404 contentValues.put(MediaStore.MediaColumns.MIME_TYPE, "image/jpeg"); 405 contentValues.put(MediaStore.MediaColumns.RELATIVE_PATH, 406 "Pictures/ExtensionsPictures"); 407 outputFileOptions = new ImageCapture.OutputFileOptions.Builder( 408 getContentResolver(), 409 MediaStore.Images.Media.EXTERNAL_CONTENT_URI, 410 contentValues).build(); 411 } else { 412 if (!dir.exists()) { 413 dir.mkdirs(); 414 } 415 outputFileOptions = new ImageCapture.OutputFileOptions.Builder(saveFile).build(); 416 } 417 418 mImageCapture.takePicture( 419 outputFileOptions, 420 ContextCompat.getMainExecutor(CameraExtensionsActivity.this), 421 new ImageCapture.OnImageSavedCallback() { 422 @Override 423 public void onImageSaved( 424 ImageCapture.@NonNull OutputFileResults outputFileResults) { 425 Log.d(TAG, "Saved image to " + saveFile); 426 427 mLastTakePictureErrorMessage = null; 428 429 if (!mTakePictureIdlingResource.isIdleNow()) { 430 mTakePictureIdlingResource.decrement(); 431 } 432 433 Uri outputUri = outputFileResults.getSavedUri(); 434 435 if (mDeleteCapturedImage) { 436 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { 437 try { 438 getContentResolver().delete(outputUri, null, null); 439 } catch (RuntimeException e) { 440 Log.w(TAG, "Failed to delete uri: " + outputUri); 441 } 442 } else { 443 if (!saveFile.delete()) { 444 Log.w(TAG, "Failed to delete file: " + saveFile); 445 } 446 } 447 } else { 448 // Trigger MediaScanner to scan the file 449 // The output Uri is already inserted into media store if the 450 // device API level is equal to or larger than Android Q(29)." 451 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { 452 Intent intent = new Intent( 453 Intent.ACTION_MEDIA_SCANNER_SCAN_FILE); 454 intent.setData(Uri.fromFile(saveFile)); 455 sendBroadcast(intent); 456 } 457 458 Toast.makeText(CameraExtensionsActivity.this, 459 "Saved image to " + fileName, 460 Toast.LENGTH_LONG).show(); 461 } 462 } 463 464 @Override 465 public void onError(@NonNull ImageCaptureException exception) { 466 Log.e(TAG, "Failed to save image - " + exception.getMessage(), 467 exception.getCause()); 468 469 mLastTakePictureErrorMessage = getImageCaptureErrorMessage(exception); 470 if (!mTakePictureIdlingResource.isIdleNow()) { 471 mTakePictureIdlingResource.decrement(); 472 } 473 } 474 475 @Override 476 public void onPostviewBitmapAvailable(@NonNull Bitmap bitmap) { 477 if (!mPostviewIdlingResource.isIdleNow()) { 478 mPostviewIdlingResource.decrement(); 479 } 480 } 481 }); 482 }); 483 484 return true; 485 } 486 updateRecordingButton()487 private void updateRecordingButton() { 488 Button btnRecord = findViewById(R.id.record); 489 if (mToggleVideoCapture.isChecked()) { 490 btnRecord.setVisibility(View.VISIBLE); 491 if (mActiveRecording != null) { 492 btnRecord.setText(R.string.button_record_stop); 493 } else { 494 btnRecord.setText(R.string.button_record_start); 495 } 496 } else { 497 mRecordingInfo = null; 498 updateInfoBlock(); 499 btnRecord.setVisibility(View.GONE); 500 } 501 } 502 503 @SuppressLint("MissingPermission") startRecording()504 private void startRecording() { 505 if (mVideoCapture != null) { 506 Recorder recorder = mVideoCapture.getOutput(); 507 mActiveRecording = prepareRecording(recorder).withAudioEnabled().start( 508 ContextCompat.getMainExecutor(this), 509 generateVideoRecordEventListener() 510 ); 511 } 512 updateRecordingButton(); 513 } 514 stopRecording()515 private void stopRecording() { 516 if (mActiveRecording != null) { 517 mActiveRecording.stop(); 518 mActiveRecording = null; 519 } 520 updateRecordingButton(); 521 } 522 prepareRecording(@onNull Recorder recorder)523 private @NonNull PendingRecording prepareRecording(@NonNull Recorder recorder) { 524 return recorder.prepareRecording(this, generateVideoMediaStoreOptions()); 525 } 526 generateVideoMediaStoreOptions()527 private @NonNull MediaStoreOutputOptions generateVideoMediaStoreOptions() { 528 return new MediaStoreOutputOptions.Builder(getContentResolver(), 529 MediaStore.Video.Media.EXTERNAL_CONTENT_URI) 530 .setContentValues(generateVideoContentValues()) 531 .build(); 532 } 533 generateVideoContentValues()534 private ContentValues generateVideoContentValues() { 535 String fileName = "video_" + System.currentTimeMillis(); 536 ContentValues contentValues = new ContentValues(); 537 contentValues.put(MediaStore.MediaColumns.MIME_TYPE, "video/mp4"); 538 contentValues.put(MediaStore.Video.Media.TITLE, fileName); 539 contentValues.put(MediaStore.Video.Media.DISPLAY_NAME, fileName); 540 contentValues.put(MediaStore.Video.Media.DATE_ADDED, System.currentTimeMillis() / 1000); 541 contentValues.put(MediaStore.Video.Media.DATE_TAKEN, System.currentTimeMillis()); 542 return contentValues; 543 } 544 generateVideoRecordEventListener()545 private Consumer<VideoRecordEvent> generateVideoRecordEventListener() { 546 return event -> { 547 updateRecordingStats(event.getRecordingStats()); 548 if (event instanceof VideoRecordEvent.Finalize) { 549 VideoRecordEvent.Finalize finalizeEvent = (VideoRecordEvent.Finalize) event; 550 Uri uri = finalizeEvent.getOutputResults().getOutputUri(); 551 String message; 552 switch (finalizeEvent.getError()) { 553 case ERROR_NONE: 554 case ERROR_FILE_SIZE_LIMIT_REACHED: 555 case ERROR_DURATION_LIMIT_REACHED: 556 case ERROR_INSUFFICIENT_STORAGE: 557 case ERROR_SOURCE_INACTIVE: 558 message = "Video saved to: " + uri; 559 break; 560 default: 561 message = "Failed to save video: uri " + uri + " with code (" 562 + finalizeEvent.getError() + ")"; 563 break; 564 } 565 Toast.makeText(CameraExtensionsActivity.this, message, Toast.LENGTH_LONG).show(); 566 } 567 }; 568 } 569 updateRecordingStats(@onNull RecordingStats stats)570 private void updateRecordingStats(@NonNull RecordingStats stats) { 571 double durationSec = NANOSECONDS.toMillis(stats.getRecordedDurationNanos()) / 1000d; 572 double sizeMb = stats.getNumBytesRecorded() / (1000d * 1000d); 573 mRecordingInfo = String.format("Duration: %.2f s\nSize: %.2f MB", durationSec, sizeMb); 574 575 updateInfoBlock(); 576 } 577 updateInfoBlock()578 private void updateInfoBlock() { 579 List<String> infoToDisplay = new ArrayList<>(); 580 if (mFrameInfo != null) { 581 infoToDisplay.add(mFrameInfo); 582 } 583 if (mRecordingInfo != null) { 584 infoToDisplay.add(mRecordingInfo); 585 } 586 587 TextView infoBlock = findViewById(R.id.infoBlock); 588 infoBlock.setText(String.join("\n", infoToDisplay)); 589 } 590 591 @SuppressWarnings("UnstableApiUsage") 592 @Override onCreate(@ullable Bundle savedInstanceState)593 protected void onCreate(@Nullable Bundle savedInstanceState) { 594 super.onCreate(savedInstanceState); 595 setContentView(R.layout.activity_camera_extensions); 596 setTitle(R.string.camerax_extensions); 597 598 mInitializationIdlingResource.increment(); 599 600 mCurrentCameraId = getIntent().getStringExtra(INTENT_EXTRA_KEY_CAMERA_ID); 601 if (mCurrentCameraId != null) { 602 mCurrentCameraSelector = CameraSelectorUtil.createCameraSelectorById(mCurrentCameraId); 603 } 604 605 // Get params from adb extra string for the e2e test cases. 606 String cameraDirection = getIntent().getStringExtra(INTENT_EXTRA_KEY_CAMERA_DIRECTION); 607 if (cameraDirection != null) { 608 if (cameraDirection.equals(BACKWARD)) { 609 mCurrentCameraSelector = CameraSelector.DEFAULT_BACK_CAMERA; 610 } else if (cameraDirection.equals(FORWARD)) { 611 mCurrentCameraSelector = CameraSelector.DEFAULT_FRONT_CAMERA; 612 } else { 613 throw new IllegalArgumentException( 614 "The camera " + cameraDirection + " is unavailable."); 615 } 616 } 617 mCurrentExtensionMode = getIntent().getIntExtra(INTENT_EXTRA_KEY_EXTENSION_MODE, 618 mCurrentExtensionMode); 619 mImageOutputFormat = getIntent().getIntExtra(INTENT_EXTRA_KEY_OUTPUT_FORMAT, 620 mImageOutputFormat); 621 622 mDeleteCapturedImage = getIntent().getBooleanExtra(INTENT_EXTRA_KEY_DELETE_CAPTURED_IMAGE, 623 mDeleteCapturedImage); 624 625 mToggleVideoCapture = findViewById(R.id.videoToggle); 626 boolean videoCaptureEnabled = mToggleVideoCapture.isChecked(); 627 mToggleVideoCapture.setChecked( 628 getIntent().getBooleanExtra(INTENT_EXTRA_KEY_VIDEO_CAPTURE_ENABLED, 629 videoCaptureEnabled)); 630 631 StrictMode.VmPolicy policy = 632 new StrictMode.VmPolicy.Builder().detectAll().penaltyLog().build(); 633 StrictMode.setVmPolicy(policy); 634 ViewStub viewFinderStub = findViewById(R.id.viewFinderStub); 635 viewFinderStub.setLayoutResource(R.layout.full_previewview); 636 mPreviewView = (PreviewView) viewFinderStub.inflate(); 637 mPreviewView.setImplementationMode(PreviewView.ImplementationMode.COMPATIBLE); 638 setupPinchToZoomAndTapToFocus(mPreviewView); 639 String cameraImplementation = 640 getIntent().getStringExtra(INTENT_EXTRA_CAMERA_IMPLEMENTATION); 641 Pair<ListenableFuture<Boolean>, CallbackToFutureAdapter.Completer<Boolean>> 642 futureCompleter = setupPermissions(this); 643 mPermissionCompleter = futureCompleter.second; 644 Futures.addCallback(futureCompleter.first, new FutureCallback<Boolean>() { 645 @Override 646 public void onSuccess(@Nullable Boolean result) { 647 mPermissionsGranted = Preconditions.checkNotNull(result); 648 649 if (!mPermissionsGranted) { 650 Log.d(TAG, "Required permissions are not all granted!"); 651 Toast.makeText(CameraExtensionsActivity.this, "Required permissions are not " 652 + "all granted!", Toast.LENGTH_LONG).show(); 653 finish(); 654 return; 655 } 656 657 if (cameraImplementation != null 658 && cameraImplementation.equals(CAMERA_PIPE_IMPLEMENTATION_OPTION)) { 659 ((ExtensionsApplication) getApplication()).setCameraXConfig( 660 CameraPipeConfig.defaultConfig()); 661 } 662 ListenableFuture<ProcessCameraProvider> cameraProviderFuture = 663 ProcessCameraProvider.getInstance(CameraExtensionsActivity.this); 664 665 Futures.addCallback(cameraProviderFuture, 666 new FutureCallback<ProcessCameraProvider>() { 667 @Override 668 public void onSuccess(@Nullable ProcessCameraProvider result) { 669 mCameraProvider = result; 670 setupCamera(); 671 } 672 673 @Override 674 public void onFailure(@NonNull Throwable t) { 675 throw new RuntimeException("Failed to get camera provider", t); 676 } 677 }, ContextCompat.getMainExecutor(CameraExtensionsActivity.this)); 678 } 679 680 @Override 681 public void onFailure(@NonNull Throwable t) { 682 throw new RuntimeException("Failed to get permissions", t); 683 } 684 }, ContextCompat.getMainExecutor(this)); 685 } 686 687 @Override onCreateOptionsMenu(@ullable Menu menu)688 public boolean onCreateOptionsMenu(@Nullable Menu menu) { 689 if (menu != null) { 690 MenuInflater inflater = getMenuInflater(); 691 inflater.inflate(R.menu.main_menu, menu); 692 693 // Remove Camera2Extensions implementation entry if the device API level is less than 32 694 if (Build.VERSION.SDK_INT < 31) { 695 menu.removeItem(R.id.menu_camera2_extensions); 696 } 697 } 698 return true; 699 } 700 701 @Override onOptionsItemSelected(@onNull MenuItem item)702 public boolean onOptionsItemSelected(@NonNull MenuItem item) { 703 Intent intent = new Intent(); 704 intent.setFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK | Intent.FLAG_ACTIVITY_NEW_TASK); 705 int itemId = item.getItemId(); 706 if (itemId == R.id.menu_camera2_extensions) { 707 if (Build.VERSION.SDK_INT >= 31) { 708 mCameraProvider.unbindAll(); 709 intent.setClassName(this, Camera2ExtensionsActivity.class.getName()); 710 startActivity(intent); 711 finish(); 712 } 713 return true; 714 } else if (itemId == R.id.menu_validation_tool) { 715 intent.setClassName(this, CameraValidationResultActivity.class.getName()); 716 startActivity(intent); 717 finish(); 718 return true; 719 } 720 721 return super.onOptionsItemSelected(item); 722 } 723 setupCamera()724 void setupCamera() { 725 if (!mPermissionsGranted) { 726 Log.d(TAG, "Permissions denied."); 727 return; 728 } 729 if (isDestroyed()) { 730 Log.d(TAG, "Activity is destroyed, not to create LifecycleCamera."); 731 return; 732 } 733 734 mCamera = mCameraProvider.bindToLifecycle(this, mCurrentCameraSelector); 735 ListenableFuture<ExtensionsManager> extensionsManagerFuture = 736 ExtensionsManager.getInstanceAsync(getApplicationContext(), mCameraProvider); 737 738 Futures.addCallback(extensionsManagerFuture, 739 new FutureCallback<ExtensionsManager>() { 740 @Override 741 public void onSuccess(@Nullable ExtensionsManager extensionsManager) { 742 // There might be timing issue that the activity has been destroyed when 743 // the onSuccess callback is received. Skips the afterward flow when the 744 // situation happens. 745 if (CameraExtensionsActivity.this.getLifecycle().getCurrentState() 746 == Lifecycle.State.DESTROYED) { 747 return; 748 } 749 mExtensionsManager = extensionsManager; 750 if (!bindUseCasesWithCurrentExtensionMode()) { 751 bindUseCasesWithNextExtensionMode(); 752 } 753 setupButtons(); 754 mInitializationIdlingResource.decrement(); 755 } 756 757 @Override 758 public void onFailure(@NonNull Throwable throwable) { 759 } 760 }, 761 ContextCompat.getMainExecutor(CameraExtensionsActivity.this) 762 ); 763 } 764 765 ScaleGestureDetector.SimpleOnScaleGestureListener mScaleGestureListener = 766 new ScaleGestureDetector.SimpleOnScaleGestureListener() { 767 @Override 768 public boolean onScale(@NonNull ScaleGestureDetector detector) { 769 if (mCamera == null) { 770 return true; 771 } 772 773 CameraInfo cameraInfo = mCamera.getCameraInfo(); 774 CameraControl cameraControl = mCamera.getCameraControl(); 775 float newZoom = 776 cameraInfo.getZoomState().getValue().getZoomRatio() 777 * detector.getScaleFactor(); 778 float clampedNewZoom = MathUtils.clamp(newZoom, 779 cameraInfo.getZoomState().getValue().getMinZoomRatio(), 780 cameraInfo.getZoomState().getValue().getMaxZoomRatio()); 781 782 ListenableFuture<Void> listenableFuture = cameraControl.setZoomRatio( 783 clampedNewZoom); 784 Futures.addCallback(listenableFuture, new FutureCallback<Void>() { 785 @Override 786 public void onSuccess(@Nullable Void result) { 787 Log.d(TAG, "setZoomRatio onSuccess: " + clampedNewZoom); 788 } 789 790 @Override 791 public void onFailure(@NonNull Throwable t) { 792 Log.d(TAG, "setZoomRatio failed, " + t); 793 } 794 }, ContextCompat.getMainExecutor(CameraExtensionsActivity.this)); 795 return true; 796 } 797 }; 798 setupPinchToZoomAndTapToFocus(PreviewView previewView)799 private void setupPinchToZoomAndTapToFocus(PreviewView previewView) { 800 ScaleGestureDetector scaleDetector = new ScaleGestureDetector(this, mScaleGestureListener); 801 802 previewView.setOnTouchListener((view, motionEvent) -> { 803 scaleDetector.onTouchEvent(motionEvent); 804 805 if (motionEvent.getAction() == MotionEvent.ACTION_UP) { 806 if (mCamera == null) { 807 return true; 808 } 809 MeteringPoint point = 810 previewView.getMeteringPointFactory().createPoint( 811 motionEvent.getX(), motionEvent.getY()); 812 813 Futures.addCallback( 814 mCamera.getCameraControl().startFocusAndMetering( 815 new FocusMeteringAction.Builder(point).build()), 816 new FutureCallback<FocusMeteringResult>() { 817 @Override 818 public void onSuccess(FocusMeteringResult result) { 819 Log.d(TAG, "Focus and metering succeeded."); 820 } 821 822 @Override 823 public void onFailure(@NonNull Throwable t) { 824 Log.e(TAG, "Focus and metering failed.", t); 825 } 826 }, 827 ContextCompat.getMainExecutor(CameraExtensionsActivity.this)); 828 } 829 return true; 830 }); 831 } 832 getPreview()833 public @Nullable Preview getPreview() { 834 return mPreview; 835 } 836 getImageCapture()837 public @Nullable ImageCapture getImageCapture() { 838 return mImageCapture; 839 } 840 841 @Override onRequestPermissionsResult( int requestCode, String @NonNull [] permissions, int @NonNull [] grantResults)842 public void onRequestPermissionsResult( 843 int requestCode, String @NonNull [] permissions, int @NonNull [] grantResults) { 844 super.onRequestPermissionsResult(requestCode, permissions, grantResults); 845 846 if (requestCode != PERMISSIONS_REQUEST_CODE) { 847 return; 848 } 849 850 // If request is cancelled, the result arrays are empty. 851 if (grantResults.length == 0) { 852 mPermissionCompleter.set(false); 853 return; 854 } 855 856 boolean allPermissionGranted = true; 857 858 for (int grantResult : grantResults) { 859 if (grantResult != PackageManager.PERMISSION_GRANTED) { 860 allPermissionGranted = false; 861 break; 862 } 863 } 864 865 Log.d(TAG, "All permissions granted: " + allPermissionGranted); 866 mPermissionCompleter.set(allPermissionGranted); 867 } 868 869 @ExtensionMode.Mode getNextExtensionMode(@xtensionMode.Mode int extensionMode)870 private int getNextExtensionMode(@ExtensionMode.Mode int extensionMode) { 871 switch (extensionMode) { 872 case ExtensionMode.NONE: 873 return ExtensionMode.BOKEH; 874 case ExtensionMode.BOKEH: 875 return ExtensionMode.HDR; 876 case ExtensionMode.HDR: 877 return ExtensionMode.NIGHT; 878 case ExtensionMode.NIGHT: 879 return ExtensionMode.FACE_RETOUCH; 880 case ExtensionMode.FACE_RETOUCH: 881 return ExtensionMode.AUTO; 882 case ExtensionMode.AUTO: 883 return ExtensionMode.NONE; 884 default: 885 throw new IllegalStateException("Unexpected value: " + extensionMode); 886 } 887 } 888 889 @VisibleForTesting isExtensionModeSupported(@onNull String cameraId, @ExtensionMode.Mode int mode)890 boolean isExtensionModeSupported(@NonNull String cameraId, @ExtensionMode.Mode int mode) { 891 CameraSelector cameraSelector = CameraSelectorUtil.createCameraSelectorById(cameraId); 892 893 return mExtensionsManager.isExtensionAvailable(cameraSelector, mode); 894 } 895 896 @VisibleForTesting isExtensionModeSupported(@onNull CameraSelector cameraSelector, @ExtensionMode.Mode int mode)897 boolean isExtensionModeSupported(@NonNull CameraSelector cameraSelector, 898 @ExtensionMode.Mode int mode) { 899 return mExtensionsManager.isExtensionAvailable(cameraSelector, mode); 900 } 901 902 @VisibleForTesting 903 @ExtensionMode.Mode getCurrentExtensionMode()904 public int getCurrentExtensionMode() { 905 return mCurrentExtensionMode; 906 } 907 908 @VisibleForTesting getInitializationIdlingResource()909 public @NonNull CountingIdlingResource getInitializationIdlingResource() { 910 return mInitializationIdlingResource; 911 } 912 913 @VisibleForTesting getPreviewViewStreamingStateIdlingResource()914 public @NonNull CountingIdlingResource getPreviewViewStreamingStateIdlingResource() { 915 return mPreviewViewStreamingStateIdlingResource; 916 } 917 918 @VisibleForTesting getPreviewViewIdleStateIdlingResource()919 public @NonNull CountingIdlingResource getPreviewViewIdleStateIdlingResource() { 920 return mPreviewViewIdleStateIdlingResource; 921 } 922 923 @VisibleForTesting getTakePictureIdlingResource()924 public @NonNull CountingIdlingResource getTakePictureIdlingResource() { 925 return mTakePictureIdlingResource; 926 } 927 928 @VisibleForTesting getPostviewIdlingResource()929 public @NonNull CountingIdlingResource getPostviewIdlingResource() { 930 return mPostviewIdlingResource; 931 } 932 933 @VisibleForTesting resetPreviewViewStreamingStateIdlingResource()934 public void resetPreviewViewStreamingStateIdlingResource() { 935 if (mPreviewViewStreamingStateIdlingResource.isIdleNow()) { 936 mPreviewViewStreamingStateIdlingResource.increment(); 937 } 938 } 939 940 @VisibleForTesting resetPreviewViewIdleStateIdlingResource()941 public void resetPreviewViewIdleStateIdlingResource() { 942 if (mPreviewViewIdleStateIdlingResource.isIdleNow()) { 943 mPreviewViewIdleStateIdlingResource.increment(); 944 } 945 } 946 947 @VisibleForTesting resetTakePictureIdlingResource()948 void resetTakePictureIdlingResource() { 949 if (mTakePictureIdlingResource.isIdleNow()) { 950 mTakePictureIdlingResource.increment(); 951 } 952 } 953 954 @VisibleForTesting resetPostviewIdlingResource()955 void resetPostviewIdlingResource() { 956 if (mPostviewIdlingResource.isIdleNow()) { 957 mPostviewIdlingResource.increment(); 958 } 959 } 960 961 /** 962 * Returns the error message of the last take picture action if any error occurs. Returns 963 * null if no error occurs. 964 */ 965 @VisibleForTesting getLastTakePictureErrorMessage()966 public @Nullable String getLastTakePictureErrorMessage() { 967 return mLastTakePictureErrorMessage; 968 } 969 970 /** 971 * Returns current stream state value. 972 */ 973 @VisibleForTesting getCurrentStreamState()974 public PreviewView.@Nullable StreamState getCurrentStreamState() { 975 return mCurrentStreamState; 976 } 977 getImageCaptureErrorMessage(@onNull ImageCaptureException exception)978 private String getImageCaptureErrorMessage(@NonNull ImageCaptureException exception) { 979 String errorCodeString; 980 int errorCode = exception.getImageCaptureError(); 981 982 switch (errorCode) { 983 case ERROR_UNKNOWN: 984 errorCodeString = "ImageCaptureErrorCode: ERROR_UNKNOWN"; 985 break; 986 case ERROR_FILE_IO: 987 errorCodeString = "ImageCaptureErrorCode: ERROR_FILE_IO"; 988 break; 989 case ERROR_CAPTURE_FAILED: 990 errorCodeString = "ImageCaptureErrorCode: ERROR_CAPTURE_FAILED"; 991 break; 992 case ERROR_CAMERA_CLOSED: 993 errorCodeString = "ImageCaptureErrorCode: ERROR_CAMERA_CLOSED"; 994 break; 995 case ERROR_INVALID_CAMERA: 996 errorCodeString = "ImageCaptureErrorCode: ERROR_INVALID_CAMERA"; 997 break; 998 default: 999 errorCodeString = "ImageCaptureErrorCode: " + errorCode; 1000 break; 1001 } 1002 1003 return errorCodeString + ", Message: " + exception.getMessage() + ", Cause: " 1004 + exception.getCause(); 1005 } 1006 setUpImageOutputFormatButton()1007 private void setUpImageOutputFormatButton() { 1008 mButtonImageOutputFormat = findViewById(R.id.image_output_format); 1009 mButtonImageOutputFormat.setText(getImageOutputFormatMenuItemName(mImageOutputFormat)); 1010 mButtonImageOutputFormat.setOnClickListener(view -> { 1011 PopupMenu popup = new PopupMenu(this, view); 1012 Menu menu = popup.getMenu(); 1013 final int groupId = Menu.NONE; 1014 1015 // Add device supported output formats. 1016 ImageCaptureCapabilities capabilities = getImageCaptureCapabilities( 1017 mCamera.getCameraInfo()); 1018 Set<Integer> supportedOutputFormats = capabilities.getSupportedOutputFormats(); 1019 for (int supportedOutputFormat : supportedOutputFormats) { 1020 // Add output format item to menu. 1021 final int menuItemId = imageOutputFormatToItemId(supportedOutputFormat); 1022 final int order = menu.size(); 1023 final String menuItemName = getImageOutputFormatMenuItemName(supportedOutputFormat); 1024 1025 menu.add(groupId, menuItemId, order, menuItemName); 1026 if (mImageOutputFormat == supportedOutputFormat) { 1027 menu.findItem(menuItemId).setChecked(true); 1028 } 1029 } 1030 1031 // Make menu single checkable. 1032 menu.setGroupCheckable(groupId, true, true); 1033 1034 // Set item click listener. 1035 popup.setOnMenuItemClickListener(item -> { 1036 int outputFormat = itemIdToImageOutputFormat(item.getItemId()); 1037 if (outputFormat != mImageOutputFormat) { 1038 mImageOutputFormat = outputFormat; 1039 final String newIconName = getImageOutputFormatMenuItemName(mImageOutputFormat); 1040 mButtonImageOutputFormat.setText(newIconName); 1041 1042 // Output format changed, rebind UseCases. 1043 bindUseCasesWithCurrentExtensionMode(); 1044 } 1045 return true; 1046 }); 1047 1048 popup.show(); 1049 }); 1050 mButtonImageOutputFormat.setVisibility(View.VISIBLE); 1051 } 1052 getImageOutputFormatMenuItemName( @mageCapture.OutputFormat int format)1053 private static @NonNull String getImageOutputFormatMenuItemName( 1054 @ImageCapture.OutputFormat int format) { 1055 switch (format) { 1056 case OUTPUT_FORMAT_JPEG: 1057 return "Jpeg"; 1058 case OUTPUT_FORMAT_JPEG_ULTRA_HDR: 1059 return "Jpeg Ultra HDR"; 1060 case OUTPUT_FORMAT_RAW: 1061 return "Raw"; 1062 case OUTPUT_FORMAT_RAW_JPEG: 1063 return "Raw + Jpeg"; 1064 default: 1065 return "Unknown format"; 1066 } 1067 } 1068 imageOutputFormatToItemId(@mageCapture.OutputFormat int format)1069 private static int imageOutputFormatToItemId(@ImageCapture.OutputFormat int format) { 1070 switch (format) { 1071 case OUTPUT_FORMAT_JPEG: 1072 return 0; 1073 case OUTPUT_FORMAT_JPEG_ULTRA_HDR: 1074 return 1; 1075 default: 1076 throw new IllegalArgumentException("Undefined output format: " + format); 1077 } 1078 } 1079 1080 @ImageCapture.OutputFormat itemIdToImageOutputFormat(int itemId)1081 private static int itemIdToImageOutputFormat(int itemId) { 1082 switch (itemId) { 1083 case 0: 1084 return OUTPUT_FORMAT_JPEG; 1085 case 1: 1086 return OUTPUT_FORMAT_JPEG_ULTRA_HDR; 1087 default: 1088 throw new IllegalArgumentException("Undefined item id: " + itemId); 1089 } 1090 } 1091 } 1092