1 /* 2 * Copyright 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package androidx.camera.integration.view; 18 19 import static androidx.camera.core.impl.utils.executor.CameraXExecutors.mainThreadExecutor; 20 import static androidx.camera.video.VideoRecordEvent.Finalize.ERROR_NONE; 21 22 import android.Manifest; 23 import android.content.ContentResolver; 24 import android.content.ContentValues; 25 import android.graphics.Color; 26 import android.net.Uri; 27 import android.os.Bundle; 28 import android.provider.MediaStore; 29 import android.util.Log; 30 import android.view.LayoutInflater; 31 import android.view.View; 32 import android.view.ViewGroup; 33 import android.widget.Button; 34 import android.widget.CheckBox; 35 import android.widget.CompoundButton; 36 import android.widget.FrameLayout; 37 import android.widget.ImageView; 38 import android.widget.SeekBar; 39 import android.widget.TextView; 40 import android.widget.Toast; 41 import android.widget.ToggleButton; 42 43 import androidx.annotation.MainThread; 44 import androidx.annotation.RequiresPermission; 45 import androidx.annotation.VisibleForTesting; 46 import androidx.camera.core.CameraSelector; 47 import androidx.camera.core.ImageAnalysis; 48 import androidx.camera.core.ImageCapture; 49 import androidx.camera.core.Logger; 50 import androidx.camera.core.ZoomState; 51 import androidx.camera.core.impl.utils.futures.FutureCallback; 52 import androidx.camera.core.impl.utils.futures.Futures; 53 import androidx.camera.integration.view.util.CaptureUtilsKt; 54 import androidx.camera.video.MediaStoreOutputOptions; 55 import androidx.camera.video.Recording; 56 import androidx.camera.video.VideoRecordEvent; 57 import androidx.camera.view.CameraController; 58 import androidx.camera.view.LifecycleCameraController; 59 import androidx.camera.view.PreviewView; 60 import androidx.camera.view.RotationProvider; 61 import androidx.camera.view.TapToFocusInfo; 62 import androidx.camera.view.video.AudioConfig; 63 import androidx.core.util.Consumer; 64 import androidx.fragment.app.Fragment; 65 import androidx.fragment.app.FragmentActivity; 66 import androidx.lifecycle.LiveData; 67 68 import com.google.common.util.concurrent.ListenableFuture; 69 70 import org.jspecify.annotations.NonNull; 71 import org.jspecify.annotations.Nullable; 72 73 import java.text.SimpleDateFormat; 74 import java.util.Date; 75 import java.util.Locale; 76 import java.util.concurrent.ExecutorService; 77 import java.util.concurrent.Executors; 78 79 /** 80 * {@link Fragment} for testing {@link LifecycleCameraController}. 81 */ 82 public class CameraControllerFragment extends Fragment { 83 84 private static final String TAG = "CameraCtrlFragment"; 85 86 // Synthetic access 87 @SuppressWarnings("WeakerAccess") 88 LifecycleCameraController mCameraController; 89 90 @VisibleForTesting 91 PreviewView mPreviewView; 92 private FrameLayout mContainer; 93 private Button mFlashMode; 94 private ToggleButton mCameraToggle; 95 private ExecutorService mExecutorService; 96 private ToggleButton mCaptureEnabledToggle; 97 private ToggleButton mAnalysisEnabledToggle; 98 private ToggleButton mVideoEnabledToggle; 99 private ToggleButton mPinchToZoomToggle; 100 private ToggleButton mTapToFocusToggle; 101 private TextView mZoomStateText; 102 private TextView mFocusResultText; 103 private TextView mTorchStateText; 104 private TextView mLuminance; 105 private CheckBox mOnDisk; 106 private ImageView mFocusOnTapCircle; 107 private boolean mIsAnalyzerSet = true; 108 // Listen to accelerometer rotation change and pass it to tests. 109 private RotationProvider mRotationProvider; 110 private int mRotation; 111 private final RotationProvider.Listener mRotationListener = rotation -> mRotation = rotation; 112 private @Nullable Recording mActiveRecording = null; 113 private final Consumer<VideoRecordEvent> mVideoRecordEventListener = videoRecordEvent -> { 114 if (videoRecordEvent instanceof VideoRecordEvent.Finalize) { 115 VideoRecordEvent.Finalize finalize = (VideoRecordEvent.Finalize) videoRecordEvent; 116 Uri uri = finalize.getOutputResults().getOutputUri(); 117 118 if (finalize.getError() == ERROR_NONE) { 119 toast("Video saved to: " + uri); 120 } else { 121 String msg = "Saved uri " + uri; 122 msg += " with code (" + finalize.getError() + ")"; 123 toast("Failed to save video: " + msg); 124 } 125 } 126 }; 127 128 // Wrapped analyzer for tests to receive callbacks. 129 private ImageAnalysis.@Nullable Analyzer mWrappedAnalyzer; 130 131 private final ImageAnalysis.Analyzer mAnalyzer = image -> { 132 byte[] bytes = new byte[image.getPlanes()[0].getBuffer().remaining()]; 133 image.getPlanes()[0].getBuffer().get(bytes); 134 int total = 0; 135 for (byte value : bytes) { 136 total += value & 0xFF; 137 } 138 if (bytes.length != 0) { 139 final int luminance = total / bytes.length; 140 mLuminance.post(() -> mLuminance.setText(String.valueOf(luminance))); 141 } 142 // Forward the call to wrapped analyzer if set. 143 if (mWrappedAnalyzer != null) { 144 mWrappedAnalyzer.analyze(image); 145 } 146 image.close(); 147 }; 148 getNewVideoOutputMediaStoreOptions()149 private @NonNull MediaStoreOutputOptions getNewVideoOutputMediaStoreOptions() { 150 String videoFileName = "video_" + System.currentTimeMillis(); 151 ContentResolver resolver = requireContext().getContentResolver(); 152 ContentValues contentValues = new ContentValues(); 153 contentValues.put(MediaStore.MediaColumns.MIME_TYPE, "video/mp4"); 154 contentValues.put(MediaStore.Video.Media.TITLE, videoFileName); 155 contentValues.put(MediaStore.Video.Media.DISPLAY_NAME, videoFileName); 156 return new MediaStoreOutputOptions 157 .Builder(resolver, MediaStore.Video.Media.EXTERNAL_CONTENT_URI) 158 .setContentValues(contentValues) 159 .build(); 160 } 161 162 @RequiresPermission(Manifest.permission.RECORD_AUDIO) 163 @Override onCreateView( @onNull LayoutInflater inflater, @Nullable ViewGroup container, @Nullable Bundle savedInstanceState)164 public @NonNull View onCreateView( 165 @NonNull LayoutInflater inflater, 166 @Nullable ViewGroup container, 167 @Nullable Bundle savedInstanceState) { 168 mExecutorService = Executors.newSingleThreadExecutor(); 169 mRotationProvider = new RotationProvider(requireContext()); 170 boolean canDetectRotation = mRotationProvider.addListener( 171 mainThreadExecutor(), mRotationListener); 172 if (!canDetectRotation) { 173 Logger.e(TAG, "The device cannot detect rotation with motion sensor."); 174 } 175 mCameraController = new LifecycleCameraController(requireContext()); 176 checkFailedFuture(mCameraController.getInitializationFuture()); 177 runSafely(() -> mCameraController.bindToLifecycle(getViewLifecycleOwner())); 178 179 180 View view = inflater.inflate(R.layout.camera_controller_view, container, false); 181 mPreviewView = view.findViewById(R.id.preview_view); 182 // Use compatible mode so StreamState is accurate. 183 mPreviewView.setImplementationMode(PreviewView.ImplementationMode.COMPATIBLE); 184 mPreviewView.setController(mCameraController); 185 mPreviewView.setScreenFlashWindow(requireActivity().getWindow()); 186 187 // Set up the button to add and remove the PreviewView 188 mContainer = view.findViewById(R.id.container); 189 view.findViewById(R.id.remove_or_add).setOnClickListener(v -> { 190 if (mContainer.getChildCount() == 0) { 191 mContainer.addView(mPreviewView); 192 } else { 193 mContainer.removeView(mPreviewView); 194 } 195 }); 196 197 // Set up the button to change the PreviewView's size. 198 view.findViewById(R.id.shrink).setOnClickListener(v -> { 199 // Shrinks PreviewView by 10% each time it's clicked. 200 mPreviewView.setLayoutParams(new FrameLayout.LayoutParams(mPreviewView.getWidth(), 201 (int) (mPreviewView.getHeight() * 0.9))); 202 }); 203 204 // Set up the front/back camera toggle. 205 mCameraToggle = view.findViewById(R.id.camera_toggle); 206 mCameraToggle.setOnCheckedChangeListener( 207 (compoundButton, value) -> 208 runSafely(() -> { 209 if (value) { 210 mCameraController.setImageCaptureFlashMode( 211 ImageCapture.FLASH_MODE_OFF); 212 updateUiText(); 213 } 214 215 mCameraController.setCameraSelector(value 216 ? CameraSelector.DEFAULT_BACK_CAMERA 217 : CameraSelector.DEFAULT_FRONT_CAMERA); 218 })); 219 220 // Image Capture enable switch. 221 mCaptureEnabledToggle = view.findViewById(R.id.capture_enabled); 222 mCaptureEnabledToggle.setOnCheckedChangeListener(this::onUseCaseToggled); 223 mCaptureEnabledToggle.setChecked(mCameraController.isImageCaptureEnabled()); 224 225 // Flash mode for image capture. 226 mFlashMode = view.findViewById(R.id.flash_mode); 227 mFlashMode.setOnClickListener(v -> { 228 switch (mCameraController.getImageCaptureFlashMode()) { 229 case ImageCapture.FLASH_MODE_AUTO: 230 mCameraController.setImageCaptureFlashMode(ImageCapture.FLASH_MODE_ON); 231 break; 232 case ImageCapture.FLASH_MODE_ON: 233 if (!mCameraToggle.isChecked()) { 234 mCameraController.setImageCaptureFlashMode(ImageCapture.FLASH_MODE_SCREEN); 235 } else { 236 mCameraController.setImageCaptureFlashMode(ImageCapture.FLASH_MODE_OFF); 237 } 238 break; 239 case ImageCapture.FLASH_MODE_SCREEN: 240 mCameraController.setImageCaptureFlashMode(ImageCapture.FLASH_MODE_OFF); 241 break; 242 case ImageCapture.FLASH_MODE_OFF: 243 mCameraController.setImageCaptureFlashMode(ImageCapture.FLASH_MODE_AUTO); 244 break; 245 default: 246 throw new IllegalStateException("Invalid flash mode: " 247 + mCameraController.getImageCaptureFlashMode()); 248 } 249 updateUiText(); 250 }); 251 252 mOnDisk = view.findViewById(R.id.on_disk); 253 // Take picture button. 254 view.findViewById(R.id.capture).setOnClickListener( 255 v -> CaptureUtilsKt.takePicture(mCameraController, requireContext(), 256 mExecutorService, this::toast, mOnDisk::isChecked)); 257 258 // Set up analysis UI. 259 mAnalysisEnabledToggle = view.findViewById(R.id.analysis_enabled); 260 mAnalysisEnabledToggle.setOnCheckedChangeListener( 261 this::onUseCaseToggled); 262 mAnalysisEnabledToggle.setChecked(mCameraController.isImageAnalysisEnabled()); 263 264 ToggleButton analyzerSet = view.findViewById(R.id.analyzer_set); 265 analyzerSet.setOnCheckedChangeListener( 266 (compoundButton, value) -> { 267 mIsAnalyzerSet = value; 268 updateControllerAnalyzer(); 269 }); 270 analyzerSet.setChecked(mIsAnalyzerSet); 271 updateControllerAnalyzer(); 272 273 mLuminance = view.findViewById(R.id.luminance); 274 275 // Set up video UI. 276 mVideoEnabledToggle = view.findViewById(R.id.video_enabled); 277 mVideoEnabledToggle.setOnCheckedChangeListener( 278 (compoundButton, checked) -> { 279 onUseCaseToggled(compoundButton, checked); 280 updateUiText(); 281 }); 282 283 view.findViewById(R.id.video_record).setOnClickListener(v -> { 284 try { 285 startRecording(mVideoRecordEventListener); 286 } catch (RuntimeException exception) { 287 toast("Failed to record video: " + exception.getMessage()); 288 } 289 updateUiText(); 290 }); 291 view.findViewById(R.id.video_stop_recording).setOnClickListener( 292 v -> { 293 stopRecording(); 294 updateUiText(); 295 }); 296 297 mPinchToZoomToggle = view.findViewById(R.id.pinch_to_zoom_toggle); 298 mPinchToZoomToggle.setOnCheckedChangeListener( 299 (compoundButton, checked) -> mCameraController.setPinchToZoomEnabled(checked)); 300 301 mTapToFocusToggle = view.findViewById(R.id.tap_to_focus_toggle); 302 mTapToFocusToggle.setOnCheckedChangeListener( 303 (compoundButton, checked) -> mCameraController.setTapToFocusEnabled(checked)); 304 305 ((ToggleButton) view.findViewById(R.id.torch_toggle)).setOnCheckedChangeListener( 306 (compoundButton, checked) -> checkFailedFuture( 307 mCameraController.enableTorch(checked))); 308 309 ((SeekBar) view.findViewById(R.id.linear_zoom_slider)).setOnSeekBarChangeListener( 310 new SeekBar.OnSeekBarChangeListener() { 311 @Override 312 public void onProgressChanged(SeekBar seekBar, int progress, boolean b) { 313 checkFailedFuture(mCameraController.setLinearZoom( 314 (float) progress / seekBar.getMax())); 315 } 316 317 @Override 318 public void onStartTrackingTouch(SeekBar seekBar) { 319 320 } 321 322 @Override 323 public void onStopTrackingTouch(SeekBar seekBar) { 324 325 } 326 }); 327 328 mZoomStateText = view.findViewById(R.id.zoom_state_text); 329 updateZoomStateText(mCameraController.getZoomState().getValue()); 330 mCameraController.getZoomState().observe(getViewLifecycleOwner(), 331 this::updateZoomStateText); 332 333 mFocusOnTapCircle = view.findViewById(R.id.focus_on_tap_circle); 334 mFocusResultText = view.findViewById(R.id.focus_result_text); 335 LiveData<TapToFocusInfo> focusOnTapState = mCameraController.getTapToFocusInfoState(); 336 focusOnTapState.observe(getViewLifecycleOwner(), this::applyFocusOnTap); 337 338 mTorchStateText = view.findViewById(R.id.torch_state_text); 339 updateTorchStateText(mCameraController.getTorchState().getValue()); 340 mCameraController.getTorchState().observe(getViewLifecycleOwner(), 341 this::updateTorchStateText); 342 343 updateUiText(); 344 return view; 345 } 346 347 @Override onDestroyView()348 public void onDestroyView() { 349 super.onDestroyView(); 350 if (mExecutorService != null) { 351 mExecutorService.shutdown(); 352 } 353 mRotationProvider.removeListener(mRotationListener); 354 } 355 checkFailedFuture(ListenableFuture<Void> voidFuture)356 void checkFailedFuture(ListenableFuture<Void> voidFuture) { 357 Futures.addCallback(voidFuture, new FutureCallback<Void>() { 358 359 @Override 360 public void onSuccess(@Nullable Void result) { 361 362 } 363 364 @Override 365 public void onFailure(@NonNull Throwable t) { 366 toast(t.toString()); 367 } 368 }, mainThreadExecutor()); 369 } 370 371 // Synthetic access 372 @SuppressWarnings("WeakerAccess") toast(String message)373 void toast(String message) { 374 FragmentActivity activity = getActivity(); 375 if (activity != null) { 376 activity.runOnUiThread(() -> { 377 if (isAdded()) { 378 Toast.makeText(getContext(), message, Toast.LENGTH_SHORT).show(); 379 } 380 }); 381 } 382 Log.d(TAG, message); 383 } 384 updateZoomStateText(@ullable ZoomState zoomState)385 private void updateZoomStateText(@Nullable ZoomState zoomState) { 386 if (zoomState == null) { 387 mZoomStateText.setText("Null"); 388 } else { 389 mZoomStateText.setText(zoomState.toString()); 390 } 391 } 392 applyFocusOnTap(@onNull TapToFocusInfo tapToFocusInfo)393 private void applyFocusOnTap(@NonNull TapToFocusInfo tapToFocusInfo) { 394 if (mFocusOnTapCircle == null) { 395 return; 396 } 397 398 switch (tapToFocusInfo.getFocusState()) { 399 case CameraController.TAP_TO_FOCUS_NOT_STARTED: 400 case CameraController.TAP_TO_FOCUS_NOT_FOCUSED: 401 case CameraController.TAP_TO_FOCUS_FAILED: 402 mFocusOnTapCircle.setVisibility(View.INVISIBLE); 403 break; 404 case CameraController.TAP_TO_FOCUS_STARTED: 405 mFocusOnTapCircle.setVisibility(View.VISIBLE); 406 mFocusOnTapCircle.setColorFilter(Color.GRAY); 407 updateFocusOnTapCirclePosition(tapToFocusInfo); 408 break; 409 case CameraController.TAP_TO_FOCUS_FOCUSED: 410 mFocusOnTapCircle.setVisibility(View.VISIBLE); 411 mFocusOnTapCircle.setColorFilter(Color.WHITE); 412 updateFocusOnTapCirclePosition(tapToFocusInfo); 413 break; 414 } 415 416 updateFocusStateText(tapToFocusInfo.getFocusState()); 417 } 418 updateFocusOnTapCirclePosition(@onNull TapToFocusInfo tapToFocusInfo)419 private void updateFocusOnTapCirclePosition(@NonNull TapToFocusInfo tapToFocusInfo) { 420 if (tapToFocusInfo.getTapPoint() != null) { 421 mFocusOnTapCircle.setX( 422 tapToFocusInfo.getTapPoint().x - (float) mFocusOnTapCircle.getWidth() / 2); 423 mFocusOnTapCircle.setY( 424 tapToFocusInfo.getTapPoint().y - (float) mFocusOnTapCircle.getHeight() / 2); 425 } 426 } 427 updateFocusStateText(@onNull Integer tapToFocusState)428 private void updateFocusStateText(@NonNull Integer tapToFocusState) { 429 SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss", Locale.getDefault()); 430 String text = ""; 431 switch (tapToFocusState) { 432 case CameraController.TAP_TO_FOCUS_NOT_STARTED: 433 text = "not started"; 434 break; 435 case CameraController.TAP_TO_FOCUS_STARTED: 436 text = "started"; 437 break; 438 case CameraController.TAP_TO_FOCUS_FOCUSED: 439 text = "successful"; 440 break; 441 case CameraController.TAP_TO_FOCUS_NOT_FOCUSED: 442 text = "unsuccessful"; 443 break; 444 case CameraController.TAP_TO_FOCUS_FAILED: 445 text = "failed"; 446 break; 447 } 448 mFocusResultText.setText( 449 "Focus state: " + text + " time: " + dateFormat.format(new Date())); 450 } 451 updateTorchStateText(@ullable Integer torchState)452 private void updateTorchStateText(@Nullable Integer torchState) { 453 if (torchState == null) { 454 mTorchStateText.setText("Torch state null"); 455 } else { 456 mTorchStateText.setText("Torch state: " + torchState); 457 } 458 } 459 460 /** 461 * Updates UI text based on the state of {@link #mCameraController}. 462 */ updateUiText()463 private void updateUiText() { 464 mFlashMode.setText(getFlashModeTextResId()); 465 final Integer lensFacing = mCameraController.getCameraSelector().getLensFacing(); 466 mCameraToggle.setChecked( 467 lensFacing != null && lensFacing == CameraSelector.LENS_FACING_BACK); 468 mVideoEnabledToggle.setChecked(mCameraController.isVideoCaptureEnabled()); 469 mPinchToZoomToggle.setChecked(mCameraController.isPinchToZoomEnabled()); 470 mTapToFocusToggle.setChecked(mCameraController.isTapToFocusEnabled()); 471 } 472 getFlashModeTextResId()473 private int getFlashModeTextResId() { 474 switch (mCameraController.getImageCaptureFlashMode()) { 475 case ImageCapture.FLASH_MODE_AUTO: 476 return R.string.flash_mode_auto; 477 case ImageCapture.FLASH_MODE_ON: 478 return R.string.flash_mode_on; 479 case ImageCapture.FLASH_MODE_SCREEN: 480 return R.string.flash_mode_screen; 481 case ImageCapture.FLASH_MODE_OFF: 482 return R.string.flash_mode_off; 483 default: 484 throw new IllegalStateException("Invalid flash mode: " 485 + mCameraController.getImageCaptureFlashMode()); 486 } 487 } 488 489 /** 490 * Sets or clears analyzer based on current state. 491 */ updateControllerAnalyzer()492 private void updateControllerAnalyzer() { 493 if (mIsAnalyzerSet) { 494 mCameraController.setImageAnalysisAnalyzer(mExecutorService, mAnalyzer); 495 } else { 496 mCameraController.clearImageAnalysisAnalyzer(); 497 } 498 } 499 500 /** 501 * Executes the runnable and catches {@link IllegalStateException}. 502 */ runSafely(@onNull Runnable runnable)503 private void runSafely(@NonNull Runnable runnable) { 504 try { 505 runnable.run(); 506 } catch (IllegalStateException ex) { 507 toast("Failed to bind use cases."); 508 } 509 } 510 onUseCaseToggled(CompoundButton compoundButton, boolean value)511 private void onUseCaseToggled(CompoundButton compoundButton, boolean value) { 512 if (mCaptureEnabledToggle == null || mAnalysisEnabledToggle == null 513 || mVideoEnabledToggle == null) { 514 return; 515 } 516 int useCaseEnabledFlags = 0; 517 if (mCaptureEnabledToggle.isChecked()) { 518 useCaseEnabledFlags = useCaseEnabledFlags | CameraController.IMAGE_CAPTURE; 519 } 520 if (mAnalysisEnabledToggle.isChecked()) { 521 useCaseEnabledFlags = useCaseEnabledFlags | CameraController.IMAGE_ANALYSIS; 522 } 523 if (mVideoEnabledToggle.isChecked()) { 524 useCaseEnabledFlags = useCaseEnabledFlags | CameraController.VIDEO_CAPTURE; 525 } 526 final int finalUseCaseEnabledFlags = useCaseEnabledFlags; 527 runSafely(() -> mCameraController.setEnabledUseCases(finalUseCaseEnabledFlags)); 528 } 529 530 // ----------------- 531 // For testing 532 // ----------------- 533 534 /** 535 */ 536 @VisibleForTesting getCameraController()537 LifecycleCameraController getCameraController() { 538 return mCameraController; 539 } 540 541 @VisibleForTesting getExecutorService()542 ExecutorService getExecutorService() { 543 return mExecutorService; 544 } 545 546 /** 547 */ 548 @VisibleForTesting setWrappedAnalyzer(ImageAnalysis.@ullable Analyzer analyzer)549 void setWrappedAnalyzer(ImageAnalysis.@Nullable Analyzer analyzer) { 550 mWrappedAnalyzer = analyzer; 551 } 552 553 /** 554 */ 555 @VisibleForTesting getPreviewView()556 PreviewView getPreviewView() { 557 return mPreviewView; 558 } 559 560 /** 561 */ 562 @VisibleForTesting getSensorRotation()563 int getSensorRotation() { 564 return mRotation; 565 } 566 567 @RequiresPermission(Manifest.permission.RECORD_AUDIO) 568 @VisibleForTesting 569 @MainThread startRecording(Consumer<VideoRecordEvent> listener)570 void startRecording(Consumer<VideoRecordEvent> listener) { 571 MediaStoreOutputOptions outputOptions = getNewVideoOutputMediaStoreOptions(); 572 AudioConfig audioConfig = AudioConfig.create(true); 573 mActiveRecording = mCameraController.startRecording(outputOptions, audioConfig, 574 mExecutorService, listener); 575 } 576 577 @VisibleForTesting 578 @MainThread stopRecording()579 void stopRecording() { 580 if (mActiveRecording != null) { 581 mActiveRecording.stop(); 582 } 583 } 584 585 } 586