1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.wm; 18 19 import static android.app.WindowConfiguration.WINDOWING_MODE_UNDEFINED; 20 import static android.content.Context.MEDIA_PROJECTION_SERVICE; 21 import static android.content.res.Configuration.ORIENTATION_UNDEFINED; 22 import static android.view.ContentRecordingSession.RECORD_CONTENT_DISPLAY; 23 import static android.view.ContentRecordingSession.RECORD_CONTENT_TASK; 24 import static android.view.ViewProtoEnums.DISPLAY_STATE_OFF; 25 26 import static com.android.internal.protolog.WmProtoLogGroups.WM_DEBUG_CONTENT_RECORDING; 27 28 import android.annotation.NonNull; 29 import android.annotation.Nullable; 30 import android.content.res.Configuration; 31 import android.graphics.Point; 32 import android.graphics.PointF; 33 import android.graphics.Rect; 34 import android.media.projection.IMediaProjectionManager; 35 import android.media.projection.StopReason; 36 import android.os.IBinder; 37 import android.os.RemoteException; 38 import android.os.ServiceManager; 39 import android.view.ContentRecordingSession; 40 import android.view.ContentRecordingSession.RecordContent; 41 import android.window.DesktopExperienceFlags; 42 import android.view.Display; 43 import android.view.DisplayInfo; 44 import android.view.SurfaceControl; 45 46 import com.android.internal.annotations.VisibleForTesting; 47 import com.android.internal.protolog.ProtoLog; 48 import com.android.server.display.feature.DisplayManagerFlags; 49 50 /** 51 * Manages content recording for a particular {@link DisplayContent}. 52 */ 53 final class ContentRecorder implements WindowContainerListener { 54 55 /** 56 * Maximum acceptable anisotropy for the output image. 57 * 58 * Necessary to avoid unnecessary scaling when the anisotropy is almost the same, as it is not 59 * exact anyway. For external displays, we expect an anisoptry of about 2% even if the pixels 60 * are, in fact, square due to the imprecision of the display's actual size (rounded to the 61 * nearest cm). 62 */ 63 private static final float MAX_ANISOTROPY = 0.025f; 64 65 /** 66 * The display content this class is handling recording for. 67 */ 68 @NonNull 69 private final DisplayContent mDisplayContent; 70 71 @Nullable private final MediaProjectionManagerWrapper mMediaProjectionManager; 72 73 /** 74 * The session for content recording, or null if this DisplayContent is not being used for 75 * recording. 76 */ 77 private ContentRecordingSession mContentRecordingSession = null; 78 79 /** 80 * The WindowContainer for the level of the hierarchy to record. 81 */ 82 @Nullable private WindowContainer mRecordedWindowContainer = null; 83 84 /** 85 * The surface for recording the contents of this hierarchy, or null if content recording is 86 * temporarily disabled. 87 */ 88 @Nullable private SurfaceControl mRecordedSurface = null; 89 90 /** 91 * The last bounds of the region to record. 92 */ 93 @Nullable private Rect mLastRecordedBounds = null; 94 95 /** 96 * The last size of the surface mirrored out to. 97 */ 98 @Nullable private Point mLastConsumingSurfaceSize = new Point(0, 0); 99 100 /** 101 * The last configuration orientation. 102 */ 103 @Configuration.Orientation 104 private int mLastOrientation = ORIENTATION_UNDEFINED; 105 106 private int mLastWindowingMode = WINDOWING_MODE_UNDEFINED; 107 108 private final boolean mCorrectForAnisotropicPixels; 109 ContentRecorder(@onNull DisplayContent displayContent)110 ContentRecorder(@NonNull DisplayContent displayContent) { 111 this(displayContent, new RemoteMediaProjectionManagerWrapper(displayContent.mDisplayId), 112 !new DisplayManagerFlags().isPixelAnisotropyCorrectionInLogicalDisplayEnabled() 113 && displayContent.getDisplayInfo().type == Display.TYPE_EXTERNAL); 114 } 115 116 @VisibleForTesting ContentRecorder(@onNull DisplayContent displayContent, @NonNull MediaProjectionManagerWrapper mediaProjectionManager, boolean correctForAnisotropicPixels)117 ContentRecorder(@NonNull DisplayContent displayContent, 118 @NonNull MediaProjectionManagerWrapper mediaProjectionManager, 119 boolean correctForAnisotropicPixels) { 120 mDisplayContent = displayContent; 121 mMediaProjectionManager = mediaProjectionManager; 122 mCorrectForAnisotropicPixels = correctForAnisotropicPixels; 123 } 124 125 /** 126 * Sets the incoming recording session. Should only be used when starting to record on 127 * this display; stopping recording is handled separately when the display is destroyed. 128 * 129 * @param session the new session indicating recording will begin on this display. 130 */ setContentRecordingSession(@ullable ContentRecordingSession session)131 void setContentRecordingSession(@Nullable ContentRecordingSession session) { 132 mContentRecordingSession = session; 133 } 134 isContentRecordingSessionSet()135 boolean isContentRecordingSessionSet() { 136 return mContentRecordingSession != null; 137 } 138 139 /** 140 * Returns {@code true} if this DisplayContent is currently recording. 141 */ isCurrentlyRecording()142 boolean isCurrentlyRecording() { 143 return mContentRecordingSession != null && mRecordedSurface != null; 144 } 145 146 /** 147 * Start recording if this DisplayContent no longer has content. Pause recording if it now 148 * has content or the display is not on. 149 */ updateRecording()150 @VisibleForTesting void updateRecording() { 151 if (isCurrentlyRecording() && (mDisplayContent.getLastHasContent() 152 || mDisplayContent.getDisplayInfo().state == Display.STATE_OFF)) { 153 pauseRecording(); 154 } else { 155 // Display no longer has content, or now has a surface to write to, so try to start 156 // recording. 157 startRecordingIfNeeded(); 158 } 159 } 160 onMirrorOutputSurfaceOrientationChanged()161 void onMirrorOutputSurfaceOrientationChanged() { 162 onConfigurationChanged(mLastOrientation, mLastWindowingMode); 163 } 164 165 /** 166 * Handle a configuration change on the display content, and resize recording if needed. 167 * @param lastOrientation the prior orientation of the configuration 168 */ onConfigurationChanged( @onfiguration.Orientation int lastOrientation, int lastWindowingMode)169 void onConfigurationChanged( 170 @Configuration.Orientation int lastOrientation, int lastWindowingMode) { 171 // Update surface for MediaProjection, if this DisplayContent is being used for recording. 172 if (!isCurrentlyRecording() || mLastRecordedBounds == null) { 173 return; 174 } 175 176 // Recording has already begun, but update recording since the display is now on. 177 if (mRecordedWindowContainer == null) { 178 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 179 "Content Recording: Unexpectedly null window container; unable to update " 180 + "recording for display %d", 181 mDisplayContent.getDisplayId()); 182 return; 183 } 184 185 // TODO(b/297514518) Do not start capture if the app is in PIP, the bounds are 186 // inaccurate. 187 if (mContentRecordingSession.getContentToRecord() == RECORD_CONTENT_TASK) { 188 final Task capturedTask = mRecordedWindowContainer.asTask(); 189 if (capturedTask.inPinnedWindowingMode()) { 190 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 191 "Content Recording: Display %d was already recording, but " 192 + "pause capture since the task is in PIP", 193 mDisplayContent.getDisplayId()); 194 pauseRecording(); 195 return; 196 } 197 } 198 199 // Record updated windowing mode, if necessary. 200 int recordedContentWindowingMode = mRecordedWindowContainer.getWindowingMode(); 201 if (lastWindowingMode != recordedContentWindowingMode) { 202 mMediaProjectionManager.notifyWindowingModeChanged( 203 mContentRecordingSession.getContentToRecord(), 204 mContentRecordingSession.getTargetUid(), 205 recordedContentWindowingMode 206 ); 207 } 208 209 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 210 "Content Recording: Display %d was already recording, so apply " 211 + "transformations if necessary", 212 mDisplayContent.getDisplayId()); 213 // Retrieve the size of the region to record, and continue with the update 214 // if the bounds or orientation has changed. 215 final Rect recordedContentBounds = mRecordedWindowContainer.getBounds(); 216 @Configuration.Orientation int recordedContentOrientation = 217 mRecordedWindowContainer.getConfiguration().orientation; 218 final Point surfaceSize = fetchSurfaceSizeIfPresent(); 219 if (!mLastRecordedBounds.equals(recordedContentBounds) 220 || lastOrientation != recordedContentOrientation 221 || !mLastConsumingSurfaceSize.equals(surfaceSize)) { 222 if (surfaceSize != null) { 223 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 224 "Content Recording: Going ahead with updating recording for display " 225 + "%d to new bounds %s and/or orientation %d and/or surface " 226 + "size %s", 227 mDisplayContent.getDisplayId(), recordedContentBounds, 228 recordedContentOrientation, surfaceSize); 229 230 updateMirroredSurface(mRecordedWindowContainer.getSyncTransaction(), 231 recordedContentBounds, surfaceSize); 232 } else { 233 // If the surface removed, do nothing. We will handle this via onDisplayChanged 234 // (the display will be off if the surface is removed). 235 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 236 "Content Recording: Unable to update recording for display %d to new " 237 + "bounds %s and/or orientation %d and/or surface size %s, " 238 + "since the surface is not available.", 239 mDisplayContent.getDisplayId(), recordedContentBounds, 240 recordedContentOrientation, surfaceSize); 241 } 242 } 243 } 244 245 /** Called when the surface of display is changed to a different instance. */ resetRecordingDisplay(int displayId)246 void resetRecordingDisplay(int displayId) { 247 if (!isCurrentlyRecording() 248 || mContentRecordingSession.getDisplayToRecord() != displayId) { 249 return; 250 } 251 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 252 "Content Recording: Display %d changed surface so stop recording", displayId); 253 mDisplayContent.mWmService.mTransactionFactory.get().remove(mRecordedSurface).apply(); 254 mRecordedSurface = null; 255 // Do not un-set the token, in case new surface is ready and recording should begin again. 256 } 257 258 /** 259 * Pauses recording on this display content. Note the session does not need to be updated, 260 * since recording can be resumed still. 261 */ pauseRecording()262 void pauseRecording() { 263 if (mRecordedSurface == null) { 264 return; 265 } 266 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 267 "Content Recording: Display %d has content (%b) so pause recording", 268 mDisplayContent.getDisplayId(), mDisplayContent.getLastHasContent()); 269 // If the display is not on and it is a virtual display, then it no longer has an 270 // associated surface to write output to. 271 // If the display now has content, stop mirroring to it. 272 mDisplayContent.mWmService.mTransactionFactory.get() 273 // Remove the reference to mMirroredSurface, to clean up associated memory. 274 .remove(mRecordedSurface) 275 // Reparent the SurfaceControl of this DisplayContent back to mSurfaceControl, 276 // to allow content to be added to it. This allows this DisplayContent to stop 277 // mirroring and show content normally. 278 .reparent(mDisplayContent.getWindowingLayer(), mDisplayContent.getSurfaceControl()) 279 .reparent(mDisplayContent.getOverlayLayer(), mDisplayContent.getSurfaceControl()) 280 .apply(); 281 // Pause mirroring by destroying the reference to the mirrored layer. 282 mRecordedSurface = null; 283 // Do not un-set the token, in case content is removed and recording should begin again. 284 } 285 286 /** 287 * Stops recording on this DisplayContent, and updates the session details. 288 */ stopRecording()289 void stopRecording() { 290 unregisterListener(); 291 if (mRecordedSurface != null) { 292 // Do not wait for the mirrored surface to be garbage collected, but clean up 293 // immediately. 294 mDisplayContent.mWmService.mTransactionFactory.get().remove(mRecordedSurface).apply(); 295 mRecordedSurface = null; 296 clearContentRecordingSession(); 297 // Do not need to force remove the VirtualDisplay; this is handled by the media 298 // projection service when the display is removed. 299 } 300 } 301 isDisplayReadyForMirroring()302 private boolean isDisplayReadyForMirroring() { 303 return mDisplayContent.getDisplayInfo().type != Display.TYPE_EXTERNAL 304 || mDisplayContent.mWmService.mDisplayManagerInternal.isDisplayReadyForMirroring( 305 mDisplayContent.getDisplayId()); 306 } 307 308 /** 309 * Ensure recording does not fall back to the display stack; ensure the recording is stopped 310 * and the client notified by tearing down the virtual display. 311 */ stopMediaProjection(@topReason int stopReason)312 private void stopMediaProjection(@StopReason int stopReason) { 313 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 314 "Content Recording: Stop MediaProjection on virtual display %d", 315 mDisplayContent.getDisplayId()); 316 if (mMediaProjectionManager != null) { 317 mMediaProjectionManager.stopActiveProjection(stopReason); 318 } 319 } 320 321 /** 322 * Removes both the local cache and WM Service view of the current session, to stop the session 323 * on this display. 324 */ clearContentRecordingSession()325 private void clearContentRecordingSession() { 326 // Update the cached session state first, since updating the service will result in always 327 // returning to this instance to update recording state. 328 mContentRecordingSession = null; 329 mDisplayContent.mWmService.mContentRecordingController.setContentRecordingSessionLocked( 330 null, mDisplayContent.mWmService); 331 } 332 unregisterListener()333 private void unregisterListener() { 334 Task recordedTask = mRecordedWindowContainer != null 335 ? mRecordedWindowContainer.asTask() : null; 336 if (recordedTask == null || !isRecordingContentTask()) { 337 return; 338 } 339 recordedTask.unregisterWindowContainerListener(this); 340 mRecordedWindowContainer = null; 341 } 342 343 /** 344 * Start recording to this DisplayContent if it does not have its own content. Captures the 345 * content of a WindowContainer indicated by a WindowToken. If unable to start recording, falls 346 * back to original MediaProjection approach. 347 */ startRecordingIfNeeded()348 private void startRecordingIfNeeded() { 349 // Only record if this display does not have its own content, is not recording already, 350 // and if this display is on (it has a surface to write output to). 351 if (mDisplayContent.getLastHasContent() || isCurrentlyRecording() 352 || mDisplayContent.getDisplayInfo().state == Display.STATE_OFF 353 || mContentRecordingSession == null) { 354 return; 355 } 356 357 // Recording should not be started on displays that are eligible for hosting tasks. 358 // See android.view.Display#canHostTasks(). 359 if (DesktopExperienceFlags.ENABLE_DISPLAY_CONTENT_MODE_MANAGEMENT.isTrue() 360 && mDisplayContent.mDisplay.canHostTasks()) { 361 return; 362 } 363 364 if (mContentRecordingSession.isWaitingForConsent() || !isDisplayReadyForMirroring()) { 365 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, "Content Recording: waiting to record, so do " 366 + "nothing"); 367 return; 368 } 369 370 mRecordedWindowContainer = retrieveRecordedWindowContainer(); 371 if (mRecordedWindowContainer == null) { 372 // Either the token is missing, or the window associated with the token is missing. 373 // Error has already been handled, so just leave. 374 return; 375 } 376 377 final SurfaceControl sourceSurface = mRecordedWindowContainer.getSurfaceControl(); 378 if (sourceSurface == null || !sourceSurface.isValid()) { 379 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 380 "Content Recording: Unable to start recording for display %d since the " 381 + "surface is null or have been released.", 382 mDisplayContent.getDisplayId()); 383 return; 384 } 385 386 final int contentToRecord = mContentRecordingSession.getContentToRecord(); 387 388 // TODO(b/297514518) Do not start capture if the app is in PIP, the bounds are inaccurate. 389 if (contentToRecord == RECORD_CONTENT_TASK) { 390 if (mRecordedWindowContainer.asTask().inPinnedWindowingMode()) { 391 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 392 "Content Recording: Display %d should start recording, but " 393 + "don't yet since the task is in PIP", 394 mDisplayContent.getDisplayId()); 395 return; 396 } 397 } 398 399 final Point surfaceSize = fetchSurfaceSizeIfPresent(); 400 if (surfaceSize == null) { 401 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 402 "Content Recording: Unable to start recording for display %d since the " 403 + "surface is not available.", 404 mDisplayContent.getDisplayId()); 405 return; 406 } 407 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 408 "Content Recording: Display %d has no content and is on, so start recording for " 409 + "state %d", 410 mDisplayContent.getDisplayId(), mDisplayContent.getDisplayInfo().state); 411 412 // Create a mirrored hierarchy for the SurfaceControl of the DisplayArea to capture. 413 mRecordedSurface = SurfaceControl.mirrorSurface(sourceSurface); 414 SurfaceControl.Transaction transaction = 415 mDisplayContent.mWmService.mTransactionFactory.get() 416 // Set the mMirroredSurface's parent to the root SurfaceControl for this 417 // DisplayContent. This brings the new mirrored hierarchy under this 418 // DisplayContent, 419 // so SurfaceControl will write the layers of this hierarchy to the 420 // output surface 421 // provided by the app. 422 .reparent(mRecordedSurface, mDisplayContent.getSurfaceControl()) 423 // Reparent the SurfaceControl of this DisplayContent to null, to prevent 424 // content 425 // being added to it. This ensures that no app launched explicitly on the 426 // VirtualDisplay will show up as part of the mirrored content. 427 .reparent(mDisplayContent.getWindowingLayer(), null) 428 .reparent(mDisplayContent.getOverlayLayer(), null); 429 // Retrieve the size of the DisplayArea to mirror. 430 updateMirroredSurface(transaction, mRecordedWindowContainer.getBounds(), surfaceSize); 431 transaction.apply(); 432 433 // Notify the client about the visibility of the mirrored region, now that we have begun 434 // capture. 435 if (contentToRecord == RECORD_CONTENT_TASK) { 436 mMediaProjectionManager.notifyActiveProjectionCapturedContentVisibilityChanged( 437 mRecordedWindowContainer.asTask().isVisibleRequested()); 438 } else { 439 int currentDisplayState = 440 mRecordedWindowContainer.asDisplayContent().getDisplayInfo().state; 441 mMediaProjectionManager.notifyActiveProjectionCapturedContentVisibilityChanged( 442 currentDisplayState != DISPLAY_STATE_OFF); 443 } 444 445 // Record initial windowing mode after recording starts. 446 mMediaProjectionManager.notifyWindowingModeChanged( 447 contentToRecord, mContentRecordingSession.getTargetUid(), 448 mRecordedWindowContainer.getWindowConfiguration().getWindowingMode()); 449 450 // No need to clean up. In SurfaceFlinger, parents hold references to their children. The 451 // mirrored SurfaceControl is alive since the parent DisplayContent SurfaceControl is 452 // holding a reference to it. Therefore, the mirrored SurfaceControl will be cleaned up 453 // when the VirtualDisplay is destroyed - which will clean up this DisplayContent. 454 } 455 456 /** 457 * Retrieves the {@link WindowContainer} for the level of the hierarchy to start recording, 458 * indicated by the {@link #mContentRecordingSession}. Performs any error handling and state 459 * updates necessary if the {@link WindowContainer} could not be retrieved. 460 * {@link #mContentRecordingSession} must be non-null. 461 * 462 * @return a {@link WindowContainer} to record, or {@code null} if an error was encountered. The 463 * error is logged and any cleanup is handled. 464 */ 465 @Nullable retrieveRecordedWindowContainer()466 private WindowContainer retrieveRecordedWindowContainer() { 467 @RecordContent final int contentToRecord = mContentRecordingSession.getContentToRecord(); 468 final IBinder tokenToRecord = mContentRecordingSession.getTokenToRecord(); 469 switch (contentToRecord) { 470 case RECORD_CONTENT_DISPLAY: 471 // Given the id of the display to record, retrieve the associated DisplayContent. 472 final DisplayContent dc = 473 mDisplayContent.mWmService.mRoot.getDisplayContent( 474 mContentRecordingSession.getDisplayToRecord()); 475 if (dc == null) { 476 // Fall back to screenrecording using the data sent to DisplayManager 477 mDisplayContent.mWmService.mDisplayManagerInternal.setWindowManagerMirroring( 478 mDisplayContent.getDisplayId(), false); 479 handleStartRecordingFailed(); 480 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 481 "Unable to retrieve window container to start recording for " 482 + "display %d", mDisplayContent.getDisplayId()); 483 return null; 484 } 485 // TODO(206461622) Migrate to using the RootDisplayArea 486 return dc; 487 case RECORD_CONTENT_TASK: 488 // Given the WindowToken of the region to record, retrieve the associated 489 // SurfaceControl. 490 final WindowContainer wc = tokenToRecord != null 491 ? WindowContainer.fromBinder(tokenToRecord) : null; 492 if (wc == null) { 493 handleStartRecordingFailed(); 494 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 495 "Content Recording: Unable to start recording due to null token or " + 496 "null window container for " + "display %d", 497 mDisplayContent.getDisplayId()); 498 return null; 499 } 500 final Task taskToRecord = wc.asTask(); 501 if (taskToRecord == null || !taskToRecord.isAttached()) { 502 handleStartRecordingFailed(); 503 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 504 "Content Recording: Unable to retrieve task to start recording for " 505 + "display %d", 506 mDisplayContent.getDisplayId()); 507 } else { 508 taskToRecord.registerWindowContainerListener(this); 509 } 510 return taskToRecord; 511 default: 512 // Not a valid region, or recording is disabled, so fall back to Display stack 513 // capture for the entire display. 514 handleStartRecordingFailed(); 515 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 516 "Content Recording: Unable to start recording due to invalid region for " 517 + "display %d", 518 mDisplayContent.getDisplayId()); 519 return null; 520 } 521 } 522 523 /** 524 * Exit this recording session. 525 * <p> 526 * If this is a task session, stop the recording entirely, including the MediaProjection. 527 * Do not fall back to recording the entire display on the display stack; this would surprise 528 * the user given they selected task capture. 529 * </p><p> 530 * If this is a display session, just stop recording by layer mirroring. Fall back to recording 531 * from the display stack. 532 * </p> 533 */ handleStartRecordingFailed()534 private void handleStartRecordingFailed() { 535 final boolean shouldExitTaskRecording = isRecordingContentTask(); 536 unregisterListener(); 537 clearContentRecordingSession(); 538 if (shouldExitTaskRecording) { 539 // Clean up the cached session first to ensure recording doesn't re-start, since 540 // tearing down the display will generate display events which will trickle back here. 541 stopMediaProjection(StopReason.STOP_ERROR); 542 } 543 } 544 computeScaling(int inputSizeX, int inputSizeY, float inputDpiX, float inputDpiY, int outputSizeX, int outputSizeY, float outputDpiX, float outputDpiY, PointF scaleOut)545 private void computeScaling(int inputSizeX, int inputSizeY, 546 float inputDpiX, float inputDpiY, 547 int outputSizeX, int outputSizeY, 548 float outputDpiX, float outputDpiY, 549 PointF scaleOut) { 550 float relAnisotropy = (inputDpiY / inputDpiX) / (outputDpiY / outputDpiX); 551 if (!mCorrectForAnisotropicPixels 552 || (relAnisotropy > (1 - MAX_ANISOTROPY) && relAnisotropy < (1 + MAX_ANISOTROPY))) { 553 // Calculate the scale to apply to the root mirror SurfaceControl to fit the size of the 554 // output surface. 555 float scaleX = outputSizeX / (float) inputSizeX; 556 float scaleY = outputSizeY / (float) inputSizeY; 557 float scale = Math.min(scaleX, scaleY); 558 scaleOut.x = scale; 559 scaleOut.y = scale; 560 return; 561 } 562 563 float relDpiX = outputDpiX / inputDpiX; 564 float relDpiY = outputDpiY / inputDpiY; 565 566 float scale = Math.min(outputSizeX / relDpiX / inputSizeX, 567 outputSizeY / relDpiY / inputSizeY); 568 scaleOut.x = scale * relDpiX; 569 scaleOut.y = scale * relDpiY; 570 } 571 572 /** 573 * Apply transformations to the mirrored surface to ensure the captured contents are scaled to 574 * fit and centred in the output surface. 575 * 576 * @param transaction the transaction to include transformations of mMirroredSurface 577 * to. Transaction is not applied before returning. 578 * @param recordedContentBounds bounds of the content to record to the surface provided by 579 * the app. 580 * @param surfaceSize the default size of the surface to write the display area 581 * content to 582 */ updateMirroredSurface(SurfaceControl.Transaction transaction, Rect recordedContentBounds, Point surfaceSize)583 @VisibleForTesting void updateMirroredSurface(SurfaceControl.Transaction transaction, 584 Rect recordedContentBounds, Point surfaceSize) { 585 586 DisplayInfo inputDisplayInfo = mRecordedWindowContainer.mDisplayContent.getDisplayInfo(); 587 DisplayInfo outputDisplayInfo = mDisplayContent.getDisplayInfo(); 588 589 PointF scale = new PointF(); 590 computeScaling(recordedContentBounds.width(), recordedContentBounds.height(), 591 inputDisplayInfo.physicalXDpi, inputDisplayInfo.physicalYDpi, 592 surfaceSize.x, surfaceSize.y, 593 outputDisplayInfo.physicalXDpi, outputDisplayInfo.physicalYDpi, 594 scale); 595 596 int scaledWidth = Math.round(scale.x * (float) recordedContentBounds.width()); 597 int scaledHeight = Math.round(scale.y * (float) recordedContentBounds.height()); 598 599 // Calculate the shift to apply to the root mirror SurfaceControl to centre the mirrored 600 // contents in the output surface. 601 int shiftedX = 0; 602 if (scaledWidth != surfaceSize.x) { 603 shiftedX = (surfaceSize.x - scaledWidth) / 2; 604 } 605 int shiftedY = 0; 606 if (scaledHeight != surfaceSize.y) { 607 shiftedY = (surfaceSize.y - scaledHeight) / 2; 608 } 609 610 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 611 "Content Recording: Apply transformations of shift %d x %d, scale %f x %f, crop " 612 + "(aka recorded content size) %d x %d for display %d; display has size " 613 + "%d x %d; surface has size %d x %d", 614 shiftedX, shiftedY, scale.x, scale.y, recordedContentBounds.width(), 615 recordedContentBounds.height(), mDisplayContent.getDisplayId(), 616 mDisplayContent.getConfiguration().screenWidthDp, 617 mDisplayContent.getConfiguration().screenHeightDp, surfaceSize.x, surfaceSize.y); 618 619 transaction 620 // Crop the area to capture to exclude the 'extra' wallpaper that is used 621 // for parallax (b/189930234). 622 .setWindowCrop(mRecordedSurface, recordedContentBounds.width(), 623 recordedContentBounds.height()) 624 // Scale the root mirror SurfaceControl, based upon the size difference between the 625 // source (DisplayArea to capture) and output (surface the app reads images from). 626 .setMatrix(mRecordedSurface, scale.x, 0 /* dtdx */, 0 /* dtdy */, scale.y) 627 // Position needs to be updated when the mirrored DisplayArea has changed, since 628 // the content will no longer be centered in the output surface. 629 .setPosition(mRecordedSurface, shiftedX /* x */, shiftedY /* y */); 630 mLastRecordedBounds = new Rect(recordedContentBounds); 631 mLastConsumingSurfaceSize.x = surfaceSize.x; 632 mLastConsumingSurfaceSize.y = surfaceSize.y; 633 634 // Request to notify the client about the updated bounds. 635 mMediaProjectionManager.notifyCaptureBoundsChanged( 636 mContentRecordingSession.getContentToRecord(), 637 mContentRecordingSession.getTargetUid(), 638 mLastRecordedBounds 639 ); 640 } 641 642 /** 643 * Returns a non-null {@link Point} if the surface is present, or null otherwise 644 */ 645 @Nullable fetchSurfaceSizeIfPresent()646 private Point fetchSurfaceSizeIfPresent() { 647 // Retrieve the default size of the surface the app provided to 648 // MediaProjection#createVirtualDisplay. Note the app is the consumer of the surface, 649 // since it reads out buffers from the surface, and SurfaceFlinger is the producer since 650 // it writes the mirrored layers to the buffers. 651 Point surfaceSize = 652 mDisplayContent.mWmService.mDisplayManagerInternal.getDisplaySurfaceDefaultSize( 653 mDisplayContent.getDisplayId()); 654 if (surfaceSize == null) { 655 // Layer mirroring started with a null surface, so do not apply any transformations yet. 656 // State of virtual display will change to 'ON' when the surface is set. 657 // will get event DISPLAY_DEVICE_EVENT_CHANGED 658 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 659 "Content Recording: Provided surface for recording on display %d is not " 660 + "present, so do not update the surface", 661 mDisplayContent.getDisplayId()); 662 return null; 663 } 664 return surfaceSize; 665 } 666 667 // WindowContainerListener 668 @Override onRemoved()669 public void onRemoved() { 670 ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, 671 "Content Recording: Recorded task is removed, so stop recording on display %d", 672 mDisplayContent.getDisplayId()); 673 674 unregisterListener(); 675 // Stop mirroring and teardown. 676 clearContentRecordingSession(); 677 // Clean up the cached session first to ensure recording doesn't re-start, since 678 // tearing down the display will generate display events which will trickle back here. 679 stopMediaProjection(StopReason.STOP_TARGET_REMOVED); 680 } 681 682 // WindowContainerListener 683 @Override onMergedOverrideConfigurationChanged( Configuration mergedOverrideConfiguration)684 public void onMergedOverrideConfigurationChanged( 685 Configuration mergedOverrideConfiguration) { 686 WindowContainerListener.super.onMergedOverrideConfigurationChanged( 687 mergedOverrideConfiguration); 688 onConfigurationChanged(mLastOrientation, mLastWindowingMode); 689 mLastOrientation = mergedOverrideConfiguration.orientation; 690 mLastWindowingMode = mergedOverrideConfiguration.windowConfiguration.getWindowingMode(); 691 } 692 693 // WindowContainerListener 694 @Override onVisibleRequestedChanged(boolean isVisibleRequested)695 public void onVisibleRequestedChanged(boolean isVisibleRequested) { 696 // Check still recording just to be safe. 697 if (isCurrentlyRecording() && mLastRecordedBounds != null) { 698 mMediaProjectionManager.notifyActiveProjectionCapturedContentVisibilityChanged( 699 isVisibleRequested); 700 701 if (mContentRecordingSession.getContentToRecord() == RECORD_CONTENT_TASK) { 702 // If capturing a task, then the toggle visibility of the recorded surface to match 703 // visibility of the task, so we don't capture any mid-transition frames 704 mRecordedWindowContainer.getSyncTransaction() 705 .setVisibility(mRecordedSurface, isVisibleRequested); 706 mRecordedWindowContainer.scheduleAnimation(); 707 } 708 } 709 } 710 711 @VisibleForTesting interface MediaProjectionManagerWrapper { stopActiveProjection(@topReason int stopReason)712 void stopActiveProjection(@StopReason int stopReason); notifyActiveProjectionCapturedContentVisibilityChanged(boolean isVisible)713 void notifyActiveProjectionCapturedContentVisibilityChanged(boolean isVisible); notifyWindowingModeChanged(int contentToRecord, int targetUid, int windowingMode)714 void notifyWindowingModeChanged(int contentToRecord, int targetUid, int windowingMode); notifyCaptureBoundsChanged(int contentToRecord, int targetUid, Rect captureBounds)715 void notifyCaptureBoundsChanged(int contentToRecord, int targetUid, Rect captureBounds); 716 } 717 718 private static final class RemoteMediaProjectionManagerWrapper implements 719 MediaProjectionManagerWrapper { 720 721 private final int mDisplayId; 722 @Nullable private IMediaProjectionManager mIMediaProjectionManager = null; 723 RemoteMediaProjectionManagerWrapper(int displayId)724 RemoteMediaProjectionManagerWrapper(int displayId) { 725 mDisplayId = displayId; 726 } 727 728 @Override stopActiveProjection(@topReason int stopReason)729 public void stopActiveProjection(@StopReason int stopReason) { 730 fetchMediaProjectionManager(); 731 if (mIMediaProjectionManager == null) { 732 return; 733 } 734 try { 735 ProtoLog.e(WM_DEBUG_CONTENT_RECORDING, 736 "Content Recording: stopping active projection for display %d", 737 mDisplayId); 738 mIMediaProjectionManager.stopActiveProjection(stopReason); 739 } catch (RemoteException e) { 740 ProtoLog.e(WM_DEBUG_CONTENT_RECORDING, 741 "Content Recording: Unable to tell MediaProjectionManagerService to stop " 742 + "the active projection for display %d: %s", 743 mDisplayId, e); 744 } 745 } 746 747 @Override notifyActiveProjectionCapturedContentVisibilityChanged(boolean isVisible)748 public void notifyActiveProjectionCapturedContentVisibilityChanged(boolean isVisible) { 749 fetchMediaProjectionManager(); 750 if (mIMediaProjectionManager == null) { 751 return; 752 } 753 try { 754 mIMediaProjectionManager.notifyActiveProjectionCapturedContentVisibilityChanged( 755 isVisible); 756 } catch (RemoteException e) { 757 ProtoLog.e(WM_DEBUG_CONTENT_RECORDING, 758 "Content Recording: Unable to tell MediaProjectionManagerService about " 759 + "visibility change on the active projection: %s", 760 e); 761 } 762 } 763 764 @Override notifyWindowingModeChanged(int contentToRecord, int targetUid, int windowingMode)765 public void notifyWindowingModeChanged(int contentToRecord, int targetUid, 766 int windowingMode) { 767 fetchMediaProjectionManager(); 768 if (mIMediaProjectionManager == null) { 769 return; 770 } 771 try { 772 mIMediaProjectionManager.notifyWindowingModeChanged( 773 contentToRecord, targetUid, windowingMode); 774 } catch (RemoteException e) { 775 ProtoLog.e(WM_DEBUG_CONTENT_RECORDING, 776 "Content Recording: Unable to tell log windowing mode change: %s", e); 777 } 778 } 779 780 @Override notifyCaptureBoundsChanged(int contentToRecord, int targetUid, Rect captureBounds)781 public void notifyCaptureBoundsChanged(int contentToRecord, int targetUid, 782 Rect captureBounds) { 783 fetchMediaProjectionManager(); 784 if (mIMediaProjectionManager == null) { 785 return; 786 } 787 try { 788 mIMediaProjectionManager.notifyCaptureBoundsChanged( 789 contentToRecord, targetUid, captureBounds); 790 } catch (RemoteException e) { 791 ProtoLog.e(WM_DEBUG_CONTENT_RECORDING, 792 "Content Recording: Unable to tell log bounds change: %s", e); 793 } 794 } 795 fetchMediaProjectionManager()796 private void fetchMediaProjectionManager() { 797 if (mIMediaProjectionManager != null) { 798 return; 799 } 800 IBinder b = ServiceManager.getService(MEDIA_PROJECTION_SERVICE); 801 if (b == null) { 802 return; 803 } 804 mIMediaProjectionManager = IMediaProjectionManager.Stub.asInterface(b); 805 } 806 } 807 isRecordingContentTask()808 private boolean isRecordingContentTask() { 809 return mContentRecordingSession != null 810 && mContentRecordingSession.getContentToRecord() == RECORD_CONTENT_TASK; 811 } 812 } 813