1 /* 2 * Copyright 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package androidx.camera.view; 18 19 import static android.graphics.Paint.ANTI_ALIAS_FLAG; 20 import static android.graphics.Paint.DITHER_FLAG; 21 import static android.graphics.Paint.FILTER_BITMAP_FLAG; 22 23 import static androidx.camera.core.impl.ImageOutputConfig.ROTATION_NOT_SPECIFIED; 24 import static androidx.camera.core.impl.utils.CameraOrientationUtil.surfaceRotationToDegrees; 25 import static androidx.camera.core.impl.utils.TransformUtils.getRectToRect; 26 import static androidx.camera.core.impl.utils.TransformUtils.is90or270; 27 import static androidx.camera.core.impl.utils.TransformUtils.isAspectRatioMatchingWithRoundingError; 28 import static androidx.camera.view.PreviewView.ScaleType.FILL_CENTER; 29 import static androidx.camera.view.PreviewView.ScaleType.FIT_CENTER; 30 import static androidx.camera.view.PreviewView.ScaleType.FIT_END; 31 import static androidx.camera.view.PreviewView.ScaleType.FIT_START; 32 33 import android.graphics.Bitmap; 34 import android.graphics.Canvas; 35 import android.graphics.Matrix; 36 import android.graphics.Paint; 37 import android.graphics.Rect; 38 import android.graphics.RectF; 39 import android.util.LayoutDirection; 40 import android.util.Size; 41 import android.view.Display; 42 import android.view.Surface; 43 import android.view.SurfaceView; 44 import android.view.TextureView; 45 import android.view.View; 46 47 import androidx.annotation.VisibleForTesting; 48 import androidx.camera.core.Logger; 49 import androidx.camera.core.SurfaceRequest; 50 import androidx.camera.core.ViewPort; 51 import androidx.core.util.Preconditions; 52 53 import org.jspecify.annotations.NonNull; 54 import org.jspecify.annotations.Nullable; 55 56 /** 57 * Handles {@link PreviewView} transformation. 58 * 59 * <p> This class transforms the camera output and display it in a {@link PreviewView}. The goal is 60 * to transform it in a way so that the entire area of 61 * {@link SurfaceRequest.TransformationInfo#getCropRect()} is 1) visible to end users, and 2) 62 * displayed as large as possible. 63 * 64 * <p> The inputs for the calculation are 1) the dimension of the Surface, 2) the crop rect, 3) the 65 * dimension of the PreviewView and 4) rotation degrees: 66 * 67 * <pre> 68 * Source: +-----Surface-----+ Destination: +-----PreviewView----+ 69 * | | | | 70 * | +-crop rect-+ | | | 71 * | | | | +--------------------+ 72 * | | | | 73 * | | --> | | Rotation: <-----+ 74 * | | | | 270°| 75 * | | | | | 76 * | +-----------+ | 77 * +-----------------+ 78 * 79 * By mapping the Surface crop rect to match the PreviewView, we have: 80 * 81 * +------transformed Surface-------+ 82 * | | 83 * | +----PreviewView-----+ | 84 * | | ^ | | 85 * | | | | | 86 * | +--------------------+ | 87 * | | 88 * +--------------------------------+ 89 * </pre> 90 * 91 * <p> The transformed Surface is how the PreviewView's inner view should behave, to make the 92 * crop rect matches the PreviewView. 93 */ 94 final class PreviewTransformation { 95 96 private static final String TAG = "PreviewTransform"; 97 98 private static final PreviewView.ScaleType DEFAULT_SCALE_TYPE = FILL_CENTER; 99 100 // SurfaceRequest.getResolution(). 101 private Size mResolution; 102 // This represents the area of the Surface that should be visible to end users. The area is 103 // defined by the Viewport class. 104 private Rect mSurfaceCropRect; 105 // TransformationInfo.getRotationDegrees(). 106 private int mPreviewRotationDegrees; 107 // TransformationInfo.getSensorToBufferTransform(). 108 private Matrix mSensorToBufferTransform; 109 // TransformationInfo.getTargetRotation. 110 private int mTargetRotation; 111 // Whether the preview is using front camera. 112 private boolean mIsFrontCamera; 113 // Whether the Surface contains camera transform. 114 private boolean mHasCameraTransform; 115 116 private PreviewView.ScaleType mScaleType = DEFAULT_SCALE_TYPE; 117 PreviewTransformation()118 PreviewTransformation() { 119 } 120 121 /** 122 * Sets the inputs. 123 * 124 * <p> All the values originally come from a {@link SurfaceRequest}. 125 */ setTransformationInfo(SurfaceRequest.@onNull TransformationInfo transformationInfo, Size resolution, boolean isFrontCamera)126 void setTransformationInfo(SurfaceRequest.@NonNull TransformationInfo transformationInfo, 127 Size resolution, boolean isFrontCamera) { 128 Logger.d(TAG, "Transformation info set: " + transformationInfo + " " + resolution + " " 129 + isFrontCamera); 130 mSurfaceCropRect = transformationInfo.getCropRect(); 131 mPreviewRotationDegrees = transformationInfo.getRotationDegrees(); 132 mTargetRotation = transformationInfo.getTargetRotation(); 133 mResolution = resolution; 134 mIsFrontCamera = isFrontCamera; 135 mHasCameraTransform = transformationInfo.hasCameraTransform(); 136 mSensorToBufferTransform = transformationInfo.getSensorToBufferTransform(); 137 } 138 139 /** 140 * Override with display rotation when Preview does not have a target rotation set. 141 * 142 * TODO: move the PreviewView#updateDisplayRotationIfNeeded logic into PreviewTransformation 143 * so all the transformation logic will be in one place. 144 */ overrideWithDisplayRotation(int rotationDegrees, int displayRotation)145 void overrideWithDisplayRotation(int rotationDegrees, int displayRotation) { 146 if (!mHasCameraTransform) { 147 // When the Surface doesn't have the camera transform, we use mPreviewRotationDegrees 148 // from the core directly. There is no need to override the values. 149 return; 150 } 151 mPreviewRotationDegrees = rotationDegrees; 152 mTargetRotation = displayRotation; 153 } 154 155 /** 156 * Creates a matrix that makes {@link TextureView}'s rotation matches the 157 * {@link #mTargetRotation}. 158 * 159 * <p> The value should be applied by calling {@link TextureView#setTransform(Matrix)}. Usually 160 * {@link #mTargetRotation} is the display rotation. In that case, this 161 * matrix will just make a {@link TextureView} works like a {@link SurfaceView}. If not, then 162 * it will further correct it to the desired rotation. 163 * 164 * <p> This method is also needed in {@link #createTransformedBitmap} to correct the screenshot. 165 */ 166 @VisibleForTesting getTextureViewCorrectionMatrix()167 Matrix getTextureViewCorrectionMatrix() { 168 Preconditions.checkState(isTransformationInfoReady()); 169 RectF surfaceRect = new RectF(0, 0, mResolution.getWidth(), mResolution.getHeight()); 170 int rotationDegrees = getRemainingRotationDegrees(); 171 return getRectToRect(surfaceRect, surfaceRect, rotationDegrees); 172 } 173 174 175 /** 176 * Gets the remaining rotation degrees after the preview is transformed by Android Views. 177 * 178 * <p>Both {@link TextureView} or {@link SurfaceView} uses the camera transform encoded in 179 * the {@link Surface} to correct the output. The remaining rotation degrees depends on 180 * whether the camera transform is present. 181 */ getRemainingRotationDegrees()182 private int getRemainingRotationDegrees() { 183 if (!mHasCameraTransform) { 184 // If the Surface is not connected to the camera, then the SurfaceView/TextureView will 185 // not apply any transformation. In that case, we need to apply the rotation 186 // calculated by CameraX. 187 return mPreviewRotationDegrees; 188 } else { 189 // If the Surface is connected to the camera, then the SurfaceView/TextureView 190 // will be the one to apply the camera orientation. In that case, only the Surface 191 // rotation needs to be applied by PreviewView. 192 return -surfaceRotationToDegrees(mTargetRotation); 193 } 194 } 195 196 /** 197 * Calculates the transformation and applies it to the inner view of {@link PreviewView}. 198 * 199 * <p> The inner view could be {@link SurfaceView} or a {@link TextureView}. 200 * {@link TextureView} needs a preliminary correction since it doesn't handle the 201 * display rotation. 202 */ transformView(Size previewViewSize, int layoutDirection, @NonNull View preview)203 void transformView(Size previewViewSize, int layoutDirection, @NonNull View preview) { 204 if (previewViewSize.getHeight() == 0 || previewViewSize.getWidth() == 0) { 205 Logger.w(TAG, "Transform not applied due to PreviewView size: " + previewViewSize); 206 return; 207 } 208 if (!isTransformationInfoReady()) { 209 return; 210 } 211 212 if (preview instanceof TextureView) { 213 // For TextureView, correct the orientation to match the target rotation. 214 ((TextureView) preview).setTransform(getTextureViewCorrectionMatrix()); 215 } else { 216 // Logs an error if non-display rotation is used with SurfaceView. 217 Display display = preview.getDisplay(); 218 boolean mismatchedDisplayRotation = mHasCameraTransform && display != null 219 && display.getRotation() != mTargetRotation; 220 boolean hasRemainingRotation = 221 !mHasCameraTransform && getRemainingRotationDegrees() != 0; 222 if (mismatchedDisplayRotation || hasRemainingRotation) { 223 Logger.e(TAG, "Custom rotation not supported with SurfaceView/PERFORMANCE mode."); 224 } 225 } 226 227 RectF surfaceRectInPreviewView = getTransformedSurfaceRect(previewViewSize, 228 layoutDirection); 229 preview.setPivotX(0); 230 preview.setPivotY(0); 231 preview.setScaleX(surfaceRectInPreviewView.width() / mResolution.getWidth()); 232 preview.setScaleY(surfaceRectInPreviewView.height() / mResolution.getHeight()); 233 preview.setTranslationX(surfaceRectInPreviewView.left - preview.getLeft()); 234 preview.setTranslationY(surfaceRectInPreviewView.top - preview.getTop()); 235 } 236 237 /** 238 * Sets the {@link PreviewView.ScaleType}. 239 */ setScaleType(PreviewView.ScaleType scaleType)240 void setScaleType(PreviewView.ScaleType scaleType) { 241 mScaleType = scaleType; 242 } 243 244 /** 245 * Gets the {@link PreviewView.ScaleType}. 246 */ getScaleType()247 PreviewView.ScaleType getScaleType() { 248 return mScaleType; 249 } 250 251 /** 252 * Gets the transformed {@link Surface} rect in PreviewView coordinates. 253 * 254 * <p> Returns desired rect of the inner view that once applied, the only part visible to 255 * end users is the crop rect. 256 */ getTransformedSurfaceRect(Size previewViewSize, int layoutDirection)257 private RectF getTransformedSurfaceRect(Size previewViewSize, int layoutDirection) { 258 Preconditions.checkState(isTransformationInfoReady()); 259 Matrix surfaceToPreviewView = 260 getSurfaceToPreviewViewMatrix(previewViewSize, layoutDirection); 261 RectF rect = new RectF(0, 0, mResolution.getWidth(), mResolution.getHeight()); 262 surfaceToPreviewView.mapRect(rect); 263 return rect; 264 } 265 266 /** 267 * Gets the camera sensor to {@link PreviewView} transform. 268 * 269 * <p>Returns null when it's not ready. 270 */ getSensorToViewTransform(@onNull Size previewViewSize, int layoutDirection)271 @Nullable Matrix getSensorToViewTransform(@NonNull Size previewViewSize, int layoutDirection) { 272 if (!isTransformationInfoReady()) { 273 return null; 274 } 275 // The matrix is calculated as the sensor -> buffer transform concatenated with the 276 // buffer -> view transform. 277 Matrix matrix = new Matrix(mSensorToBufferTransform); 278 matrix.postConcat(getSurfaceToPreviewViewMatrix(previewViewSize, layoutDirection)); 279 return matrix; 280 } 281 282 /** 283 * Calculates the transformation from {@link Surface} coordinates to {@link PreviewView} 284 * coordinates. 285 * 286 * <p> The calculation is based on making the crop rect to fill or fit the {@link PreviewView}. 287 */ getSurfaceToPreviewViewMatrix(Size previewViewSize, int layoutDirection)288 Matrix getSurfaceToPreviewViewMatrix(Size previewViewSize, int layoutDirection) { 289 Preconditions.checkState(isTransformationInfoReady()); 290 291 // Get the target of the mapping, the coordinates of the crop rect in PreviewView. 292 RectF previewViewCropRect; 293 if (isViewportAspectRatioMatchPreviewView(previewViewSize)) { 294 // If crop rect has the same aspect ratio as PreviewView, scale the crop rect to fill 295 // the entire PreviewView. This happens if the scale type is FILL_* AND a 296 // PreviewView-based viewport is used. 297 previewViewCropRect = new RectF(0, 0, previewViewSize.getWidth(), 298 previewViewSize.getHeight()); 299 } else { 300 // If the aspect ratios don't match, it could be 1) scale type is FIT_*, 2) the 301 // Viewport is not based on the PreviewView or 3) both. 302 previewViewCropRect = getPreviewViewViewportRectForMismatchedAspectRatios( 303 previewViewSize, layoutDirection); 304 } 305 Matrix matrix = getRectToRect(new RectF(mSurfaceCropRect), previewViewCropRect, 306 mPreviewRotationDegrees); 307 if (mIsFrontCamera && mHasCameraTransform) { 308 // SurfaceView/TextureView automatically mirrors the Surface for front camera, which 309 // needs to be compensated by mirroring the Surface around the upright direction of the 310 // output image. This is only necessary if the stream has camera transform. 311 // Otherwise, an internal GL processor would have mirrored it already. 312 if (is90or270(mPreviewRotationDegrees)) { 313 // If the rotation is 90/270, the Surface should be flipped vertically. 314 // +---+ 90 +---+ 270 +---+ 315 // | ^ | --> | < | | > | 316 // +---+ +---+ +---+ 317 matrix.preScale(1F, -1F, mSurfaceCropRect.centerX(), mSurfaceCropRect.centerY()); 318 } else { 319 // If the rotation is 0/180, the Surface should be flipped horizontally. 320 // +---+ 0 +---+ 180 +---+ 321 // | ^ | --> | ^ | | v | 322 // +---+ +---+ +---+ 323 matrix.preScale(-1F, 1F, mSurfaceCropRect.centerX(), mSurfaceCropRect.centerY()); 324 } 325 } 326 return matrix; 327 } 328 329 /** 330 * Gets the viewport rect in {@link PreviewView} coordinates for the case where viewport's 331 * aspect ratio doesn't match {@link PreviewView}'s aspect ratio. 332 * 333 * <p> When aspect ratios don't match, additional calculation is needed to figure out how to 334 * fit crop rect into the{@link PreviewView}. 335 */ getPreviewViewViewportRectForMismatchedAspectRatios(Size previewViewSize, int layoutDirection)336 RectF getPreviewViewViewportRectForMismatchedAspectRatios(Size previewViewSize, 337 int layoutDirection) { 338 RectF previewViewRect = new RectF(0, 0, previewViewSize.getWidth(), 339 previewViewSize.getHeight()); 340 Size rotatedViewportSize = getRotatedViewportSize(); 341 RectF rotatedViewportRect = new RectF(0, 0, rotatedViewportSize.getWidth(), 342 rotatedViewportSize.getHeight()); 343 Matrix matrix = new Matrix(); 344 setMatrixRectToRect(matrix, rotatedViewportRect, previewViewRect, mScaleType); 345 matrix.mapRect(rotatedViewportRect); 346 if (layoutDirection == LayoutDirection.RTL) { 347 return flipHorizontally(rotatedViewportRect, (float) previewViewSize.getWidth() / 2); 348 } 349 return rotatedViewportRect; 350 } 351 352 /** 353 * Set the matrix that maps the source rectangle to the destination rectangle. 354 * 355 * <p> This static method is an extension of {@link Matrix#setRectToRect} with an additional 356 * support for FILL_* types. 357 */ setMatrixRectToRect(Matrix matrix, RectF source, RectF destination, PreviewView.ScaleType scaleType)358 private static void setMatrixRectToRect(Matrix matrix, RectF source, RectF destination, 359 PreviewView.ScaleType scaleType) { 360 Matrix.ScaleToFit matrixScaleType; 361 switch (scaleType) { 362 case FIT_CENTER: 363 // Fallthrough. 364 case FILL_CENTER: 365 matrixScaleType = Matrix.ScaleToFit.CENTER; 366 break; 367 case FIT_END: 368 // Fallthrough. 369 case FILL_END: 370 matrixScaleType = Matrix.ScaleToFit.END; 371 break; 372 case FIT_START: 373 // Fallthrough. 374 case FILL_START: 375 matrixScaleType = Matrix.ScaleToFit.START; 376 break; 377 default: 378 Logger.e(TAG, "Unexpected crop rect: " + scaleType); 379 matrixScaleType = Matrix.ScaleToFit.FILL; 380 } 381 boolean isFitTypes = 382 scaleType == FIT_CENTER || scaleType == FIT_START || scaleType == FIT_END; 383 if (isFitTypes) { 384 matrix.setRectToRect(source, destination, matrixScaleType); 385 } else { 386 // android.graphics.Matrix doesn't support fill scale types. The workaround is 387 // mapping inversely from destination to source, then invert the matrix. 388 matrix.setRectToRect(destination, source, matrixScaleType); 389 matrix.invert(matrix); 390 } 391 } 392 393 /** 394 * Flips the given rect along a vertical line for RTL layout direction. 395 */ flipHorizontally(RectF original, float flipLineX)396 private static RectF flipHorizontally(RectF original, float flipLineX) { 397 return new RectF( 398 flipLineX + flipLineX - original.right, 399 original.top, 400 flipLineX + flipLineX - original.left, 401 original.bottom); 402 } 403 404 /** 405 * Returns viewport size with target rotation applied. 406 */ getRotatedViewportSize()407 private Size getRotatedViewportSize() { 408 if (is90or270(mPreviewRotationDegrees)) { 409 return new Size(mSurfaceCropRect.height(), mSurfaceCropRect.width()); 410 } 411 return new Size(mSurfaceCropRect.width(), mSurfaceCropRect.height()); 412 } 413 414 /** 415 * Checks if the viewport's aspect ratio matches that of the {@link PreviewView}. 416 * 417 * <p> The mismatch could happen if the {@link ViewPort} is not based on the 418 * {@link PreviewView}, or the {@link PreviewView#getScaleType()} is FIT_*. In this case, we 419 * need to calculate how the crop rect should be fitted. 420 */ 421 @VisibleForTesting isViewportAspectRatioMatchPreviewView(Size previewViewSize)422 boolean isViewportAspectRatioMatchPreviewView(Size previewViewSize) { 423 // Using viewport rect to check if the viewport is based on the PreviewView. 424 Size rotatedViewportSize = getRotatedViewportSize(); 425 return isAspectRatioMatchingWithRoundingError( 426 previewViewSize, /* isAccurate1= */ true, 427 rotatedViewportSize, /* isAccurate2= */ false); 428 } 429 430 /** 431 * Return the crop rect of the preview surface. 432 */ getSurfaceCropRect()433 @Nullable Rect getSurfaceCropRect() { 434 return mSurfaceCropRect; 435 } 436 437 /** 438 * Creates a transformed screenshot of {@link PreviewView}. 439 * 440 * <p> Creates the transformed {@link Bitmap} by applying the same transformation applied to 441 * the inner view. T 442 * 443 * @param original a snapshot of the untransformed inner view. 444 */ createTransformedBitmap(@onNull Bitmap original, Size previewViewSize, int layoutDirection)445 Bitmap createTransformedBitmap(@NonNull Bitmap original, Size previewViewSize, 446 int layoutDirection) { 447 if (!isTransformationInfoReady()) { 448 return original; 449 } 450 Matrix textureViewCorrection = getTextureViewCorrectionMatrix(); 451 RectF surfaceRectInPreviewView = getTransformedSurfaceRect(previewViewSize, 452 layoutDirection); 453 454 Bitmap transformed = Bitmap.createBitmap( 455 previewViewSize.getWidth(), previewViewSize.getHeight(), original.getConfig()); 456 Canvas canvas = new Canvas(transformed); 457 458 Matrix canvasTransform = new Matrix(); 459 canvasTransform.postConcat(textureViewCorrection); 460 canvasTransform.postScale(surfaceRectInPreviewView.width() / mResolution.getWidth(), 461 surfaceRectInPreviewView.height() / mResolution.getHeight()); 462 canvasTransform.postTranslate(surfaceRectInPreviewView.left, surfaceRectInPreviewView.top); 463 464 canvas.drawBitmap(original, canvasTransform, 465 new Paint(ANTI_ALIAS_FLAG | FILTER_BITMAP_FLAG | DITHER_FLAG)); 466 return transformed; 467 } 468 469 /** 470 * Calculates the mapping from a UI touch point (0, 0) - (width, height) to normalized 471 * space (-1, -1) - (1, 1). 472 * 473 * <p> This is used by {@link PreviewViewMeteringPointFactory}. 474 * 475 * @return null if transformation info is not set. 476 */ getPreviewViewToNormalizedSensorMatrix( Size previewViewSize, int layoutDirection, Rect sensorRect)477 @Nullable Matrix getPreviewViewToNormalizedSensorMatrix( 478 Size previewViewSize, int layoutDirection, Rect sensorRect) { 479 if (!isTransformationInfoReady()) { 480 return null; 481 } 482 Matrix matrix = new Matrix(); 483 484 // Map PreviewView coordinates to Surface coordinates. 485 getSensorToViewTransform(previewViewSize, layoutDirection).invert(matrix); 486 487 // Map Surface coordinates to normalized coordinates (-1, -1) - (1, 1). 488 Matrix normalization = new Matrix(); 489 normalization.setRectToRect( 490 new RectF(0, 0, sensorRect.width(), sensorRect.height()), 491 new RectF(0, 0, 1, 1), Matrix.ScaleToFit.FILL); 492 matrix.postConcat(normalization); 493 494 return matrix; 495 } 496 isTransformationInfoReady()497 private boolean isTransformationInfoReady() { 498 // Ignore target rotation if Surface doesn't have camera transform. 499 boolean isTargetRotationSpecified = 500 !mHasCameraTransform || (mTargetRotation != ROTATION_NOT_SPECIFIED); 501 return mSurfaceCropRect != null && mResolution != null 502 && isTargetRotationSpecified; 503 } 504 } 505