1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.view.cts.surfacevalidator; 17 18 import static android.server.wm.BuildUtils.HW_TIMEOUT_MULTIPLIER; 19 import static android.server.wm.CtsWindowInfoUtils.waitForWindowVisible; 20 import static android.view.WindowInsets.Type.statusBars; 21 import static android.view.WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS; 22 23 import static org.junit.Assert.assertEquals; 24 import static org.junit.Assert.assertTrue; 25 import static org.junit.Assert.fail; 26 27 import android.Manifest; 28 import android.app.Activity; 29 import android.app.KeyguardManager; 30 import android.content.Context; 31 import android.content.Intent; 32 import android.content.pm.PackageManager; 33 import android.graphics.Bitmap; 34 import android.graphics.Insets; 35 import android.graphics.Point; 36 import android.graphics.Rect; 37 import android.hardware.display.DisplayManager; 38 import android.hardware.display.VirtualDisplay; 39 import android.media.projection.MediaProjection; 40 import android.media.projection.MediaProjectionManager; 41 import android.os.Bundle; 42 import android.os.Environment; 43 import android.os.Handler; 44 import android.os.Looper; 45 import android.os.Messenger; 46 import android.provider.Settings; 47 import android.server.wm.settings.SettingsSession; 48 import android.support.test.uiautomator.By; 49 import android.support.test.uiautomator.UiDevice; 50 import android.support.test.uiautomator.UiObject2; 51 import android.support.test.uiautomator.Until; 52 import android.util.DisplayMetrics; 53 import android.util.Log; 54 import android.util.SparseArray; 55 import android.view.PointerIcon; 56 import android.view.WindowInsets; 57 import android.view.WindowInsetsController; 58 import android.view.WindowManager; 59 import android.view.WindowMetrics; 60 import android.widget.FrameLayout; 61 62 import androidx.test.InstrumentationRegistry; 63 64 import com.android.compatibility.common.util.SystemUtil; 65 66 import org.junit.rules.TestName; 67 68 import java.io.File; 69 import java.io.FileOutputStream; 70 import java.io.IOException; 71 import java.util.concurrent.CountDownLatch; 72 import java.util.concurrent.TimeUnit; 73 import java.util.concurrent.atomic.AtomicBoolean; 74 75 public class CapturedActivity extends Activity { 76 public static class TestResult { 77 public int passFrames; 78 public int failFrames; 79 public final SparseArray<Bitmap> failures = new SparseArray<>(); 80 } 81 82 private static class ImmersiveConfirmationSetting extends SettingsSession<String> { ImmersiveConfirmationSetting()83 ImmersiveConfirmationSetting() { 84 super(Settings.Secure.getUriFor( 85 Settings.Secure.IMMERSIVE_MODE_CONFIRMATIONS), 86 Settings.Secure::getString, Settings.Secure::putString); 87 } 88 } 89 90 private ImmersiveConfirmationSetting mSettingsSession; 91 92 private static final String TAG = "CapturedActivity"; 93 private static final int PERMISSION_CODE = 1; 94 private MediaProjectionManager mProjectionManager; 95 private MediaProjection mMediaProjection; 96 private VirtualDisplay mVirtualDisplay; 97 98 private SurfacePixelValidator2 mSurfacePixelValidator; 99 100 private static final int PERMISSION_DIALOG_WAIT_MS = 1000; 101 private static final int RETRY_COUNT = 2; 102 103 private static final long START_CAPTURE_DELAY_MS = 4000; 104 105 private static final long WAIT_TIMEOUT_S = 5L * HW_TIMEOUT_MULTIPLIER; 106 107 private static final String ACCEPT_RESOURCE_ID = "android:id/button1"; 108 109 private final Handler mHandler = new Handler(Looper.getMainLooper()); 110 private volatile boolean mOnEmbedded; 111 private volatile boolean mOnWatch; 112 private CountDownLatch mMediaProjectionCreatedLatch; 113 114 private final Point mLogicalDisplaySize = new Point(); 115 private AtomicBoolean mIsSharingScreenDenied; 116 117 private int mResultCode; 118 private Intent mResultData; 119 120 private FrameLayout mParentLayout; 121 122 @Override onCreate(Bundle savedInstanceState)123 public void onCreate(Bundle savedInstanceState) { 124 super.onCreate(savedInstanceState); 125 mIsSharingScreenDenied = new AtomicBoolean(false); 126 final PackageManager packageManager = getPackageManager(); 127 mOnWatch = packageManager.hasSystemFeature(PackageManager.FEATURE_WATCH); 128 if (mOnWatch) { 129 // Don't try and set up test/capture infrastructure - they're not supported 130 return; 131 } 132 133 mParentLayout = new FrameLayout(this); 134 FrameLayout.LayoutParams layoutParams = new FrameLayout.LayoutParams( 135 FrameLayout.LayoutParams.MATCH_PARENT, 136 FrameLayout.LayoutParams.MATCH_PARENT); 137 setContentView(mParentLayout, layoutParams); 138 139 // Embedded devices are significantly slower, and are given 140 // longer duration to capture the expected number of frames 141 mOnEmbedded = packageManager.hasSystemFeature(PackageManager.FEATURE_EMBEDDED); 142 143 mSettingsSession = new ImmersiveConfirmationSetting(); 144 mSettingsSession.set("confirmed"); 145 146 WindowInsetsController windowInsetsController = getWindow().getInsetsController(); 147 windowInsetsController.hide( 148 WindowInsets.Type.navigationBars() | WindowInsets.Type.statusBars()); 149 WindowManager.LayoutParams params = getWindow().getAttributes(); 150 params.layoutInDisplayCutoutMode = LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS; 151 getWindow().setAttributes(params); 152 getWindow().setDecorFitsSystemWindows(false); 153 154 // Set the NULL pointer icon so that it won't obstruct the captured image. 155 getWindow().getDecorView().setPointerIcon( 156 PointerIcon.getSystemIcon(this, PointerIcon.TYPE_NULL)); 157 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); 158 159 160 mProjectionManager = 161 (MediaProjectionManager) getSystemService(Context.MEDIA_PROJECTION_SERVICE); 162 163 mMediaProjectionCreatedLatch = new CountDownLatch(1); 164 165 KeyguardManager keyguardManager = getSystemService(KeyguardManager.class); 166 if (keyguardManager != null) { 167 keyguardManager.requestDismissKeyguard(this, null); 168 } 169 170 startActivityForResult(mProjectionManager.createScreenCaptureIntent(), PERMISSION_CODE); 171 } 172 setLogicalDisplaySize(Point logicalDisplaySize)173 public void setLogicalDisplaySize(Point logicalDisplaySize) { 174 mLogicalDisplaySize.set(logicalDisplaySize.x, logicalDisplaySize.y); 175 } 176 dismissPermissionDialog()177 public boolean dismissPermissionDialog() { 178 // The permission dialog will be auto-opened by the activity - find it and accept 179 UiDevice uiDevice = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()); 180 UiObject2 acceptButton = uiDevice.wait(Until.findObject(By.res(ACCEPT_RESOURCE_ID)), 181 PERMISSION_DIALOG_WAIT_MS); 182 if (acceptButton != null) { 183 Log.d(TAG, "found permission dialog after searching all windows, clicked"); 184 acceptButton.click(); 185 return true; 186 } else { 187 Log.e(TAG, "Failed to find permission dialog"); 188 return false; 189 } 190 } 191 192 /** 193 * Request to start a foreground service with type "mediaProjection", 194 * it's free to run in either the same process or a different process in the package; 195 * passing a messenger object to send signal back when the foreground service is up. 196 */ startMediaProjectionService()197 private void startMediaProjectionService() { 198 final Messenger messenger = new Messenger(new Handler(Looper.getMainLooper(), msg -> { 199 switch (msg.what) { 200 case LocalMediaProjectionService.MSG_START_FOREGROUND_DONE: 201 createMediaProjection(); 202 return true; 203 } 204 Log.e(TAG, "Unknown message from the LocalMediaProjectionService: " + msg.what); 205 return false; 206 })); 207 final Intent intent = new Intent(this, LocalMediaProjectionService.class) 208 .putExtra(LocalMediaProjectionService.EXTRA_MESSENGER, messenger); 209 startForegroundService(intent); 210 } 211 212 @Override onDestroy()213 public void onDestroy() { 214 super.onDestroy(); 215 Log.d(TAG, "onDestroy"); 216 if (mMediaProjection != null) { 217 mMediaProjection.stop(); 218 mMediaProjection = null; 219 } 220 restoreSettings(); 221 } 222 223 @Override onActivityResult(int requestCode, int resultCode, Intent data)224 public void onActivityResult(int requestCode, int resultCode, Intent data) { 225 if (mOnWatch) return; 226 227 if (requestCode != PERMISSION_CODE) { 228 throw new IllegalStateException("Unknown request code: " + requestCode); 229 } 230 mIsSharingScreenDenied.set(resultCode != RESULT_OK); 231 if (mIsSharingScreenDenied.get()) { 232 Log.e(TAG, "Failed to start screenshare permission Activity result=" 233 + mIsSharingScreenDenied.get()); 234 235 return; 236 } 237 Log.d(TAG, "onActivityResult"); 238 mResultCode = resultCode; 239 mResultData = data; 240 startMediaProjectionService(); 241 } 242 createMediaProjection()243 private void createMediaProjection() { 244 mMediaProjection = mProjectionManager.getMediaProjection(mResultCode, mResultData); 245 mMediaProjection.registerCallback(new MediaProjectionCallback(), null); 246 mMediaProjectionCreatedLatch.countDown(); 247 } 248 getCaptureDurationMs()249 public long getCaptureDurationMs() { 250 return mOnEmbedded ? 100000 : 50000; 251 } 252 runTest(ISurfaceValidatorTestCase animationTestCase)253 public TestResult runTest(ISurfaceValidatorTestCase animationTestCase) throws Throwable { 254 TestResult testResult = new TestResult(); 255 Runnable cleanupRunnable = () -> { 256 Log.d(TAG, "Stopping capture and ending test case"); 257 if (mVirtualDisplay != null) { 258 mVirtualDisplay.release(); 259 mVirtualDisplay = null; 260 } 261 262 animationTestCase.end(); 263 FrameLayout contentLayout = findViewById(android.R.id.content); 264 contentLayout.removeAllViews(); 265 if (mSurfacePixelValidator != null) { 266 mSurfacePixelValidator.finish(testResult); 267 mSurfacePixelValidator = null; 268 } 269 }; 270 271 try { 272 if (mOnWatch) { 273 /** 274 * (TODO b/282204025): Legacy reasons why tests are disabled on wear. Investigate 275 * if enabling is now possible. 276 */ 277 Log.d(TAG, "Skipping test on watch."); 278 testResult.passFrames = 1000; 279 testResult.failFrames = 0; 280 return testResult; 281 } 282 283 final int numFramesRequired = animationTestCase.getNumFramesRequired(); 284 final long maxCapturedDuration = getCaptureDurationMs(); 285 286 int count = 0; 287 // Sometimes system decides to rotate the permission activity to another orientation 288 // right after showing it. This results in: uiautomation thinks that accept button 289 // appears, we successfully click it in terms of uiautomation, but nothing happens, 290 // because permission activity is already recreated. Thus, we try to click that 291 // button multiple times. 292 do { 293 // There are some cases where the consent dialog isn't shown because the process 294 // already has the additional permissions. In that case, we can skip waiting to 295 // dismiss the dialog. 296 if (mMediaProjectionCreatedLatch.getCount() == 0) { 297 break; 298 } 299 300 if (mIsSharingScreenDenied.get()) { 301 throw new IllegalStateException("User denied screen sharing permission."); 302 } 303 if (dismissPermissionDialog()) { 304 break; 305 } 306 count++; 307 Thread.sleep(1000); 308 } while (count <= RETRY_COUNT); 309 310 assertTrue("Failed to create mediaProjection", 311 mMediaProjectionCreatedLatch.await(20L * HW_TIMEOUT_MULTIPLIER, 312 TimeUnit.SECONDS)); 313 314 mHandler.post(() -> { 315 Log.d(TAG, "Setting up test case"); 316 317 // See b/216583939. On some devices, hiding system bars is disabled. In those cases, 318 // adjust the area that is rendering the test content to be outside the status bar 319 // margins to ensure capturing and comparing frames skips the status bar area. 320 Insets statusBarInsets = getWindow() 321 .getDecorView() 322 .getRootWindowInsets() 323 .getInsets(statusBars()); 324 FrameLayout.LayoutParams layoutParams = 325 (FrameLayout.LayoutParams) mParentLayout.getLayoutParams(); 326 layoutParams.setMargins(statusBarInsets.left, statusBarInsets.top, 327 statusBarInsets.right, statusBarInsets.bottom); 328 mParentLayout.setLayoutParams(layoutParams); 329 330 animationTestCase.start(getApplicationContext(), mParentLayout); 331 }); 332 333 assertTrue("Failed to wait for animation to start", animationTestCase.waitForReady()); 334 boolean[] success = new boolean[1]; 335 SystemUtil.runWithShellPermissionIdentity(() -> { 336 success[0] = waitForWindowVisible(mParentLayout); 337 }, Manifest.permission.ACCESS_SURFACE_FLINGER); 338 assertTrue("Failed to wait for test window to be visible", success[0]); 339 340 CountDownLatch setupLatch = new CountDownLatch(1); 341 mHandler.postDelayed(() -> { 342 WindowMetrics metrics = getWindowManager().getCurrentWindowMetrics(); 343 Log.d(TAG, "Starting capture: metrics=" + metrics); 344 345 int densityDpi = (int) (metrics.getDensity() * DisplayMetrics.DENSITY_DEFAULT); 346 347 int testAreaWidth = mParentLayout.getWidth(); 348 int testAreaHeight = mParentLayout.getHeight(); 349 350 Log.d(TAG, "testAreaWidth: " + testAreaWidth 351 + ", testAreaHeight: " + testAreaHeight); 352 353 Rect boundsToCheck = animationTestCase.getBoundsToCheck(mParentLayout); 354 355 if (boundsToCheck.width() < 40 || boundsToCheck.height() < 40) { 356 fail("capture bounds too small to be a fullscreen activity: " + boundsToCheck); 357 } 358 359 mSurfacePixelValidator = new SurfacePixelValidator2(mLogicalDisplaySize, 360 boundsToCheck, 361 animationTestCase.getChecker(), numFramesRequired); 362 Log.d("MediaProjection", "Size is " + mLogicalDisplaySize 363 + ", bounds are " + boundsToCheck.toShortString()); 364 mVirtualDisplay = mMediaProjection.createVirtualDisplay("CtsCapturedActivity", 365 mLogicalDisplaySize.x, mLogicalDisplaySize.y, 366 densityDpi, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, 367 mSurfacePixelValidator.getSurface(), 368 null /*Callbacks*/, 369 null /*Handler*/); 370 setupLatch.countDown(); 371 }, START_CAPTURE_DELAY_MS); 372 373 setupLatch.await(); 374 assertTrue("Failed to wait for required number of frames", 375 mSurfacePixelValidator.waitForAllFrames(maxCapturedDuration)); 376 final CountDownLatch testRunLatch = new CountDownLatch(1); 377 mHandler.post(() -> { 378 cleanupRunnable.run(); 379 testRunLatch.countDown(); 380 }); 381 382 assertTrue("Failed to wait for test to complete", 383 testRunLatch.await(WAIT_TIMEOUT_S, TimeUnit.SECONDS)); 384 385 Log.d(TAG, "Test finished, passFrames " + testResult.passFrames 386 + ", failFrames " + testResult.failFrames); 387 return testResult; 388 } catch (Throwable throwable) { 389 mHandler.post(cleanupRunnable); 390 Log.e(TAG, "Test Failed, passFrames " + testResult.passFrames + ", failFrames " 391 + testResult.failFrames); 392 throw throwable; 393 } 394 } 395 saveFailureCaptures(SparseArray<Bitmap> failFrames, TestName name)396 private void saveFailureCaptures(SparseArray<Bitmap> failFrames, TestName name) { 397 if (failFrames.size() == 0) return; 398 399 String directoryName = Environment.getExternalStorageDirectory() 400 + "/" + getClass().getSimpleName() 401 + "/" + name.getMethodName(); 402 File testDirectory = new File(directoryName); 403 if (testDirectory.exists()) { 404 String[] children = testDirectory.list(); 405 if (children == null) { 406 return; 407 } 408 for (String file : children) { 409 new File(testDirectory, file).delete(); 410 } 411 } else { 412 testDirectory.mkdirs(); 413 } 414 415 for (int i = 0; i < failFrames.size(); i++) { 416 int frameNr = failFrames.keyAt(i); 417 Bitmap bitmap = failFrames.valueAt(i); 418 419 String bitmapName = "frame_" + frameNr + ".png"; 420 Log.d(TAG, "Saving file : " + bitmapName + " in directory : " + directoryName); 421 422 File file = new File(directoryName, bitmapName); 423 try (FileOutputStream fileStream = new FileOutputStream(file)) { 424 bitmap.compress(Bitmap.CompressFormat.PNG, 85, fileStream); 425 fileStream.flush(); 426 } catch (IOException e) { 427 e.printStackTrace(); 428 } 429 } 430 } 431 verifyTest(ISurfaceValidatorTestCase testCase, TestName name)432 public void verifyTest(ISurfaceValidatorTestCase testCase, TestName name) throws Throwable { 433 if (mIsSharingScreenDenied.get()) { 434 throw new IllegalStateException("User denied screen sharing permission."); 435 } 436 437 CapturedActivity.TestResult result = runTest(testCase); 438 saveFailureCaptures(result.failures, name); 439 440 float failRatio = 1.0f * result.failFrames / (result.failFrames + result.passFrames); 441 assertTrue("Error: " + failRatio + " fail ratio - extremely high, is activity obstructed?", 442 failRatio < 0.95f); 443 assertEquals("Error: " + result.failFrames 444 + " incorrect frames observed - incorrect positioning", 0, result.failFrames); 445 } 446 447 private class MediaProjectionCallback extends MediaProjection.Callback { 448 @Override onStop()449 public void onStop() { 450 Log.d(TAG, "MediaProjectionCallback#onStop"); 451 if (mVirtualDisplay != null) { 452 mVirtualDisplay.release(); 453 mVirtualDisplay = null; 454 } 455 } 456 } 457 restoreSettings()458 public void restoreSettings() { 459 // Adding try/catch due to bug with UiAutomation crashing the test b/272370325 460 try { 461 if (mSettingsSession != null) { 462 mSettingsSession.close(); 463 mSettingsSession = null; 464 } 465 } catch (Exception e) { 466 Log.e(TAG, "Crash occurred when closing settings session. See b/272370325", e); 467 } 468 } 469 isOnWatch()470 public boolean isOnWatch() { 471 return mOnWatch; 472 } 473 474 } 475