1 /* 2 * Copyright 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package androidx.camera.integration.core; 18 19 import static android.view.Display.HdrCapabilities.HDR_TYPE_DOLBY_VISION; 20 import static android.view.Display.HdrCapabilities.HDR_TYPE_HDR10; 21 import static android.view.Display.HdrCapabilities.HDR_TYPE_HDR10_PLUS; 22 import static android.view.Display.HdrCapabilities.HDR_TYPE_HLG; 23 24 import android.Manifest; 25 import android.content.Context; 26 import android.content.ContextWrapper; 27 import android.content.pm.PackageManager; 28 import android.hardware.display.DisplayManager; 29 import android.hardware.display.DisplayManager.DisplayListener; 30 import android.os.Build; 31 import android.os.Bundle; 32 import android.os.Handler; 33 import android.os.Looper; 34 import android.util.Log; 35 import android.view.Display; 36 import android.view.Surface; 37 import android.view.View; 38 import android.view.ViewStub; 39 import android.view.WindowManager; 40 import android.widget.TextView; 41 import android.widget.Toast; 42 43 import androidx.activity.result.ActivityResultCallback; 44 import androidx.activity.result.ActivityResultLauncher; 45 import androidx.activity.result.contract.ActivityResultContracts; 46 import androidx.annotation.RequiresApi; 47 import androidx.appcompat.app.AppCompatActivity; 48 import androidx.camera.core.AspectRatio; 49 import androidx.camera.core.CameraSelector; 50 import androidx.camera.core.DynamicRange; 51 import androidx.camera.core.Preview; 52 import androidx.camera.lifecycle.ProcessCameraProvider; 53 import androidx.core.content.ContextCompat; 54 import androidx.lifecycle.ViewModelProvider; 55 56 import org.jspecify.annotations.NonNull; 57 import org.jspecify.annotations.Nullable; 58 59 import java.util.Arrays; 60 import java.util.Collections; 61 import java.util.HashMap; 62 import java.util.HashSet; 63 import java.util.Locale; 64 import java.util.Map; 65 import java.util.Objects; 66 import java.util.Set; 67 import java.util.stream.Collectors; 68 69 /** Activity which runs the camera preview with opengl processing */ 70 public class OpenGLActivity extends AppCompatActivity { 71 private static final String TAG = "OpenGLActivity"; 72 73 /** 74 * Intent Extra string for choosing which Camera implementation to use. 75 */ 76 public static final String INTENT_EXTRA_CAMERA_IMPLEMENTATION = "camera_implementation"; 77 78 /** 79 * Intent Extra string for choosing which type of render surface to use to display Preview. 80 */ 81 public static final String INTENT_EXTRA_RENDER_SURFACE_TYPE = "render_surface_type"; 82 /** 83 * TextureView render surface for {@link OpenGLActivity#INTENT_EXTRA_RENDER_SURFACE_TYPE}. 84 * This is the default render surface. 85 */ 86 public static final String RENDER_SURFACE_TYPE_TEXTUREVIEW = "textureview"; 87 /** 88 * SurfaceView render surface for {@link OpenGLActivity#INTENT_EXTRA_RENDER_SURFACE_TYPE}. 89 * This type will block the main thread while detaching it's {@link Surface} from the OpenGL 90 * renderer to avoid compatibility issues on some devices. 91 */ 92 public static final String RENDER_SURFACE_TYPE_SURFACEVIEW = "surfaceview"; 93 /** 94 * SurfaceView render surface (in non-blocking mode) for 95 * {@link OpenGLActivity#INTENT_EXTRA_RENDER_SURFACE_TYPE}. This type will NOT 96 * block the main thread while detaching it's {@link Surface} from the OpenGL 97 * renderer, but some devices may crash due to their OpenGL/EGL implementation not being 98 * thread-safe. On API 30+, {@link android.view.SurfaceControl} is used to allow releasing of 99 * the surface off the main thread. 100 */ 101 public static final String RENDER_SURFACE_TYPE_SURFACEVIEW_NONBLOCKING = 102 "surfaceview_nonblocking"; 103 104 private static final String DEFAULT_RENDER_SURFACE_TYPE; 105 106 static { 107 // By default we choose TextureView to maximize compatibility. On devices that are API 108 // level 33 and above, we choose SurfaceView by default since SurfaceView has been proven 109 // to be stable on this API level, and we are able to push releasing of the surface off 110 // the main thread via SurfaceControl. 111 if (Build.VERSION.SDK_INT >= 33) { 112 DEFAULT_RENDER_SURFACE_TYPE = RENDER_SURFACE_TYPE_SURFACEVIEW_NONBLOCKING; 113 } else { 114 DEFAULT_RENDER_SURFACE_TYPE = RENDER_SURFACE_TYPE_TEXTUREVIEW; 115 } 116 117 } 118 119 private static final String[] REQUIRED_PERMISSIONS = 120 new String[]{ 121 Manifest.permission.CAMERA, 122 }; 123 124 private static final int FPS_NUM_SAMPLES = 10; 125 private OpenGLRenderer mRenderer; 126 private DisplayManager.DisplayListener mDisplayListener; 127 private ProcessCameraProvider mCameraProvider; 128 129 @Override onCreate(Bundle savedInstanceState)130 public void onCreate(Bundle savedInstanceState) { 131 super.onCreate(savedInstanceState); 132 133 setContentView(R.layout.opengl_activity); 134 135 Display display = null; 136 if (Build.VERSION.SDK_INT >= 30) { 137 display = Api30Impl.getDisplay(this); 138 } 139 OpenGLRenderer renderer = mRenderer = new OpenGLRenderer( 140 getHighDynamicRangesSupportedByDisplay(display)); 141 ViewStub viewFinderStub = findViewById(R.id.viewFinderStub); 142 View viewFinder = OpenGLActivity.chooseViewFinder(getIntent().getExtras(), viewFinderStub, 143 renderer); 144 145 // Add a frame update listener to display FPS 146 FpsRecorder fpsRecorder = new FpsRecorder(FPS_NUM_SAMPLES); 147 TextView fpsCounterView = findViewById(R.id.fps_counter); 148 renderer.setFrameUpdateListener(ContextCompat.getMainExecutor(this), timestamp -> { 149 double fps = fpsRecorder.recordTimestamp(timestamp); 150 fpsCounterView.setText(getString(R.string.fps_counter_template, 151 (Double.isNaN(fps) || Double.isInfinite(fps)) ? "---" : String.format(Locale.US, 152 "%.0f", fps))); 153 }); 154 155 // A display listener is needed when the phone rotates 180 degrees without stopping at a 156 // 90 degree increment. In these cases, onCreate() isn't triggered, so we need to ensure 157 // the output surface uses the correct orientation. 158 mDisplayListener = 159 new DisplayListener() { 160 @Override 161 public void onDisplayAdded(int displayId) { 162 } 163 164 @Override 165 public void onDisplayRemoved(int displayId) { 166 } 167 168 @Override 169 public void onDisplayChanged(int displayId) { 170 Display viewFinderDisplay = viewFinder.getDisplay(); 171 if (viewFinderDisplay != null 172 && viewFinderDisplay.getDisplayId() == displayId) { 173 renderer.invalidateSurface(Surfaces.toSurfaceRotationDegrees( 174 viewFinderDisplay.getRotation())); 175 } 176 } 177 }; 178 179 DisplayManager dpyMgr = 180 Objects.requireNonNull((DisplayManager) getSystemService(Context.DISPLAY_SERVICE)); 181 dpyMgr.registerDisplayListener(mDisplayListener, new Handler(Looper.getMainLooper())); 182 183 Bundle bundle = this.getIntent().getExtras(); 184 if (bundle != null) { 185 String cameraImplementation = bundle.getString(INTENT_EXTRA_CAMERA_IMPLEMENTATION); 186 if (cameraImplementation != null) { 187 CameraXViewModel.configureCameraProvider(cameraImplementation); 188 } 189 } 190 191 CameraXViewModel viewModel = new ViewModelProvider(this).get(CameraXViewModel.class); 192 viewModel 193 .getCameraProvider() 194 .observe( 195 this, 196 cameraProviderResult -> { 197 if (cameraProviderResult.hasProvider()) { 198 mCameraProvider = cameraProviderResult.getProvider(); 199 if (allPermissionsGranted()) { 200 startCamera(); 201 } 202 } else { 203 Log.e(TAG, "Failed to retrieve ProcessCameraProvider", 204 cameraProviderResult.getError()); 205 Toast.makeText(getApplicationContext(), 206 "Unable to initialize CameraX. See logs " 207 + "for details.", Toast.LENGTH_LONG).show(); 208 } 209 }); 210 211 if (!allPermissionsGranted()) { 212 mRequestPermissions.launch(REQUIRED_PERMISSIONS); 213 } 214 } 215 216 @Override onDestroy()217 public void onDestroy() { 218 super.onDestroy(); 219 DisplayManager dpyMgr = Objects.requireNonNull( 220 (DisplayManager) getSystemService(Context.DISPLAY_SERVICE)); 221 dpyMgr.unregisterDisplayListener(mDisplayListener); 222 mRenderer.shutdown(); 223 } 224 225 /** 226 * Chooses the type of view to use for the viewfinder based on intent extras. 227 * 228 * @param intentExtras Optional extras which can contain an extra with key 229 * {@link #INTENT_EXTRA_RENDER_SURFACE_TYPE}. Possible values are one of 230 * {@link #RENDER_SURFACE_TYPE_TEXTUREVIEW}, 231 * {@link #RENDER_SURFACE_TYPE_SURFACEVIEW}, or 232 * {@link #RENDER_SURFACE_TYPE_SURFACEVIEW_NONBLOCKING}. If {@code null}, 233 * or the bundle does not contain a surface type, then 234 * {@link #RENDER_SURFACE_TYPE_TEXTUREVIEW} will be used. 235 * @param viewFinderStub The stub to inflate the chosen viewfinder into. 236 * @param renderer The {@link OpenGLRenderer} which will render frames into the 237 * viewfinder. 238 * @return The inflated viewfinder View. 239 */ chooseViewFinder(@ullable Bundle intentExtras, @NonNull ViewStub viewFinderStub, @NonNull OpenGLRenderer renderer)240 public static @NonNull View chooseViewFinder(@Nullable Bundle intentExtras, 241 @NonNull ViewStub viewFinderStub, 242 @NonNull OpenGLRenderer renderer) { 243 244 String renderSurfaceType = DEFAULT_RENDER_SURFACE_TYPE; 245 if (intentExtras != null) { 246 renderSurfaceType = intentExtras.getString(INTENT_EXTRA_RENDER_SURFACE_TYPE, 247 DEFAULT_RENDER_SURFACE_TYPE); 248 } 249 250 switch (renderSurfaceType) { 251 case RENDER_SURFACE_TYPE_TEXTUREVIEW: 252 Log.d(TAG, "Using TextureView render surface."); 253 return TextureViewRenderSurface.inflateWith(viewFinderStub, renderer); 254 case RENDER_SURFACE_TYPE_SURFACEVIEW: 255 Log.d(TAG, "Using SurfaceView render surface."); 256 return SurfaceViewRenderSurface.inflateWith(viewFinderStub, renderer); 257 case RENDER_SURFACE_TYPE_SURFACEVIEW_NONBLOCKING: 258 Log.d(TAG, "Using SurfaceView (non-blocking) render surface."); 259 return SurfaceViewRenderSurface.inflateNonBlockingWith(viewFinderStub, renderer); 260 default: 261 throw new IllegalArgumentException(String.format(Locale.US, "Unknown render " 262 + "surface type: %s. Supported surface types include: [%s, %s, %s]", 263 renderSurfaceType, RENDER_SURFACE_TYPE_TEXTUREVIEW, 264 RENDER_SURFACE_TYPE_SURFACEVIEW, 265 RENDER_SURFACE_TYPE_SURFACEVIEW_NONBLOCKING)); 266 } 267 } 268 269 /** 270 * Returns a list of HDR dynamic ranges supported by the display. 271 * 272 * <p>The returned HDR dynamic ranges are constants defined by the {@code DynamicRange} class. 273 * The returned list will never contain {@link DynamicRange#SDR}. 274 * 275 * <p>The list may be empty if the display does not support HDR, such as on pre-API 24 devices. 276 */ getHighDynamicRangesSupportedByDisplay( @ullable Display display)277 public static @NonNull Set<DynamicRange> getHighDynamicRangesSupportedByDisplay( 278 @Nullable Display display) { 279 if (display != null && Build.VERSION.SDK_INT >= 24) { 280 return Api24Impl.getHighDynamicRangesSupportedByDisplay(display); 281 } else { 282 return Collections.emptySet(); 283 } 284 } 285 startCamera()286 private void startCamera() { 287 // Keep screen on for this app. This is just for convenience, and is not required. 288 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); 289 290 // Set the aspect ratio of Preview to match the aspect ratio of the view finder (defined 291 // with ConstraintLayout). 292 Preview preview = new Preview.Builder().setTargetAspectRatio(AspectRatio.RATIO_4_3).build(); 293 294 mRenderer.attachInputPreview(preview).addListener(() -> { 295 Log.d(TAG, "OpenGLRenderer get the new surface for the Preview"); 296 }, ContextCompat.getMainExecutor(this)); 297 CameraSelector cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA; 298 299 mCameraProvider.bindToLifecycle(this, cameraSelector, preview); 300 } 301 302 // **************************** Permission handling code start *******************************// 303 private final ActivityResultLauncher<String[]> mRequestPermissions = 304 registerForActivityResult(new ActivityResultContracts.RequestMultiplePermissions(), 305 new ActivityResultCallback<Map<String, Boolean>>() { 306 @Override 307 public void onActivityResult(Map<String, Boolean> result) { 308 for (String permission : REQUIRED_PERMISSIONS) { 309 if (!Objects.requireNonNull(result.get(permission))) { 310 Toast.makeText(OpenGLActivity.this, "Permissions not granted", 311 Toast.LENGTH_SHORT).show(); 312 finish(); 313 } 314 } 315 316 // All permissions granted. 317 if (mCameraProvider != null) { 318 startCamera(); 319 } 320 } 321 }); 322 allPermissionsGranted()323 private boolean allPermissionsGranted() { 324 for (String permission : REQUIRED_PERMISSIONS) { 325 if (ContextCompat.checkSelfPermission(this, permission) 326 != PackageManager.PERMISSION_GRANTED) { 327 return false; 328 } 329 } 330 return true; 331 } 332 // **************************** Permission handling code end *********************************// 333 334 @RequiresApi(24) 335 static class Api24Impl { 336 private static final Map<Integer, Set<DynamicRange>> DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE = 337 new HashMap<>(); 338 339 static { DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE.put(HDR_TYPE_HLG, Collections.singleton(DynamicRange.HLG_10_BIT))340 DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE.put(HDR_TYPE_HLG, 341 Collections.singleton(DynamicRange.HLG_10_BIT)); DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE.put(HDR_TYPE_HDR10, Collections.singleton(DynamicRange.HDR10_10_BIT))342 DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE.put(HDR_TYPE_HDR10, 343 Collections.singleton(DynamicRange.HDR10_10_BIT)); DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE.put(HDR_TYPE_HDR10_PLUS, Collections.singleton(DynamicRange.HDR10_PLUS_10_BIT))344 DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE.put(HDR_TYPE_HDR10_PLUS, 345 Collections.singleton(DynamicRange.HDR10_PLUS_10_BIT)); DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE.put(HDR_TYPE_DOLBY_VISION, new HashSet<>(Arrays.asList( DynamicRange.DOLBY_VISION_8_BIT, DynamicRange.DOLBY_VISION_10_BIT)))346 DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE.put(HDR_TYPE_DOLBY_VISION, 347 new HashSet<>(Arrays.asList( 348 DynamicRange.DOLBY_VISION_8_BIT, DynamicRange.DOLBY_VISION_10_BIT))); 349 } 350 Api24Impl()351 private Api24Impl() { 352 // This class is not instantiable. 353 } 354 getHighDynamicRangesSupportedByDisplay( @onNull Display display)355 static Set<DynamicRange> getHighDynamicRangesSupportedByDisplay( 356 @NonNull Display display) { 357 Display.HdrCapabilities hdrCapabilities = display.getHdrCapabilities(); 358 if (hdrCapabilities == null) { 359 return Collections.emptySet(); 360 } 361 return Arrays.stream(hdrCapabilities.getSupportedHdrTypes()) 362 .boxed() 363 .map(DISPLAY_HDR_TYPE_TO_DYNAMIC_RANGE::get) 364 .flatMap(set -> Objects.requireNonNull(set).stream()) 365 .filter(Objects::nonNull) 366 .collect(Collectors.toSet()); 367 } 368 369 } 370 371 @RequiresApi(30) 372 static class Api30Impl { Api30Impl()373 private Api30Impl() { 374 // This class is not instantiable. 375 } 376 getDisplay(ContextWrapper contextWrapper)377 static Display getDisplay(ContextWrapper contextWrapper) { 378 return contextWrapper.getDisplay(); 379 } 380 381 } 382 } 383