1 /*
<lambda>null2 * Copyright 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package platform.test.screenshot
18
19 import android.annotation.ColorInt
20 import android.annotation.SuppressLint
21 import android.graphics.Bitmap
22 import android.graphics.BitmapFactory
23 import android.graphics.Color
24 import android.graphics.Rect
25 import android.platform.uiautomatorhelpers.DeviceHelpers.shell
26 import android.provider.Settings.System
27 import androidx.annotation.VisibleForTesting
28 import androidx.test.platform.app.InstrumentationRegistry
29 import androidx.test.runner.screenshot.Screenshot
30 import com.android.internal.app.SimpleIconFactory
31 import java.io.FileNotFoundException
32 import org.junit.rules.TestRule
33 import org.junit.runner.Description
34 import org.junit.runners.model.Statement
35 import platform.test.screenshot.matchers.BitmapMatcher
36 import platform.test.screenshot.matchers.MSSIMMatcher
37 import platform.test.screenshot.matchers.MatchResult
38 import platform.test.screenshot.matchers.PixelPerfectMatcher
39 import platform.test.screenshot.parity.ParityStatsCollector
40 import platform.test.screenshot.proto.ScreenshotResultProto
41 import platform.test.screenshot.report.DiffResultExportStrategy
42
43 /**
44 * Rule to be added to a test to facilitate screenshot testing.
45 *
46 * This rule records current test name and when instructed it will perform the given bitmap
47 * comparison against the given golden. All the results (including result proto file) are stored
48 * into the device to be retrieved later.
49 *
50 * @see Bitmap.assertAgainstGolden
51 */
52 @SuppressLint("SyntheticAccessor")
53 open class ScreenshotTestRule
54 @VisibleForTesting
55 internal constructor(
56 val goldenPathManager: GoldenPathManager,
57 /** Strategy to report diffs to external systems. */
58 private val diffEscrowStrategy: DiffResultExportStrategy,
59 private val disableIconPool: Boolean = true,
60 ) : TestRule, BitmapDiffer, ScreenshotAsserterFactory {
61
62 @JvmOverloads
63 constructor(
64 goldenPathManager: GoldenPathManager,
65 disableIconPool: Boolean = true,
66 ) : this(
67 goldenPathManager,
68 DiffResultExportStrategy.createDefaultStrategy(goldenPathManager),
69 disableIconPool,
70 )
71
72 private val doesCollectScreenshotParityStats =
73 "yes"
74 .equals(
75 java.lang.System.getProperty("screenshot.collectScreenshotParityStats"),
76 ignoreCase = true,
77 )
78 private lateinit var testIdentifier: String
79
80 companion object {
81 private val parityStatsCollector = ParityStatsCollector()
82 }
83
84 override fun apply(base: Statement, description: Description): Statement =
85 object : Statement() {
86 override fun evaluate() {
87 try {
88 testIdentifier = getTestIdentifier(description)
89 if (disableIconPool) {
90 // Disables pooling of SimpleIconFactory objects as it caches
91 // density, which when updating the screen configuration in runtime
92 // sometimes it does not get updated in the icon renderer.
93 SimpleIconFactory.setPoolEnabled(false)
94 }
95 base.evaluate()
96 } finally {
97 if (disableIconPool) {
98 SimpleIconFactory.setPoolEnabled(true)
99 }
100 }
101 }
102 }
103
104 open fun getTestIdentifier(description: Description): String =
105 "${description.className}_${description.methodName}"
106
107 open fun fetchExpectedImage(goldenIdentifier: String): Bitmap? {
108 val instrument = InstrumentationRegistry.getInstrumentation()
109 return listOf(instrument.targetContext.applicationContext, instrument.context)
110 .map { context ->
111 try {
112 context.assets
113 .open(goldenPathManager.goldenImageIdentifierResolver(goldenIdentifier))
114 .use {
115 return@use BitmapFactory.decodeStream(it)
116 }
117 } catch (e: FileNotFoundException) {
118 return@map null
119 }
120 }
121 .filterNotNull()
122 .firstOrNull()
123 }
124
125 /**
126 * Asserts the given bitmap against the golden identified by the given name.
127 *
128 * Note: The golden identifier should be unique per your test module (unless you want multiple
129 * tests to match the same golden). The name must not contain extension. You should also avoid
130 * adding strings like "golden", "image" and instead describe what is the golder referring to.
131 *
132 * @param actual The bitmap captured during the test.
133 * @param goldenIdentifier Name of the golden. Allowed characters: 'A-Za-z0-9_-'
134 * @param matcher The algorithm to be used to perform the matching.
135 * @throws IllegalArgumentException If the golden identifier contains forbidden characters or is
136 * empty.
137 * @see MSSIMMatcher
138 * @see PixelPerfectMatcher
139 * @see Bitmap.assertAgainstGolden
140 */
141 @Deprecated("use BitmapDiffer or ScreenshotAsserterFactory interfaces")
142 fun assertBitmapAgainstGolden(
143 actual: Bitmap,
144 goldenIdentifier: String,
145 matcher: BitmapMatcher,
146 ) {
147 try {
148 assertBitmapAgainstGolden(
149 actual = actual,
150 goldenIdentifier = goldenIdentifier,
151 matcher = matcher,
152 regions = emptyList<Rect>(),
153 )
154 } finally {
155 actual.recycle()
156 }
157 }
158
159 /**
160 * Asserts the given bitmap against the golden identified by the given name.
161 *
162 * Note: The golden identifier should be unique per your test module (unless you want multiple
163 * tests to match the same golden). The name must not contain extension. You should also avoid
164 * adding strings like "golden", "image" and instead describe what is the golder referring to.
165 *
166 * @param actual The bitmap captured during the test.
167 * @param goldenIdentifier Name of the golden. Allowed characters: 'A-Za-z0-9_-'
168 * @param matcher The algorithm to be used to perform the matching.
169 * @param regions An optional array of interesting regions for partial screenshot diff.
170 * @throws IllegalArgumentException If the golden identifier contains forbidden characters or is
171 * empty.
172 * @see MSSIMMatcher
173 * @see PixelPerfectMatcher
174 * @see Bitmap.assertAgainstGolden
175 */
176 @Deprecated("use BitmapDiffer or ScreenshotAsserterFactory interfaces")
177 override fun assertBitmapAgainstGolden(
178 actual: Bitmap,
179 goldenIdentifier: String,
180 matcher: BitmapMatcher,
181 regions: List<Rect>,
182 ) {
183 if (!goldenIdentifier.matches("^[A-Za-z0-9_-]+$".toRegex())) {
184 throw IllegalArgumentException(
185 "The given golden identifier '$goldenIdentifier' does not satisfy the naming " +
186 "requirement. Allowed characters are: '[A-Za-z0-9_-]'"
187 )
188 }
189
190 val expected = fetchExpectedImage(goldenIdentifier)
191 if (expected == null) {
192 diffEscrowStrategy.reportResult(
193 testIdentifier = testIdentifier,
194 goldenIdentifier = goldenIdentifier,
195 status = ScreenshotResultProto.DiffResult.Status.MISSING_REFERENCE,
196 actual = actual,
197 )
198 throw AssertionError(
199 "Missing golden image " +
200 "'${goldenPathManager.goldenImageIdentifierResolver(goldenIdentifier)}'. " +
201 "Did you mean to check in a new image?"
202 )
203 }
204
205 if (expected.sameAs(actual)) {
206 if (doesCollectScreenshotParityStats) {
207 val stats =
208 ScreenshotResultProto.DiffResult.ComparisonStatistics.newBuilder()
209 .setNumberPixelsCompared(actual.width * actual.height)
210 .setNumberPixelsIdentical(actual.width * actual.height)
211 .setNumberPixelsDifferent(0)
212 .setNumberPixelsIgnored(0)
213 .build()
214 parityStatsCollector.collectTestStats(
215 testIdentifier,
216 MatchResult(matches = true, diff = null, comparisonStatistics = stats),
217 )
218 parityStatsCollector.report()
219 }
220 expected.recycle()
221 return
222 }
223
224 if (actual.width != expected.width || actual.height != expected.height) {
225 val comparisonResult =
226 matcher.compareBitmaps(
227 expected = expected.toIntArray(),
228 given = actual.toIntArray(),
229 expectedWidth = expected.width,
230 expectedHeight = expected.height,
231 actualWidth = actual.width,
232 actualHeight = actual.height,
233 )
234 diffEscrowStrategy.reportResult(
235 testIdentifier = testIdentifier,
236 goldenIdentifier = goldenIdentifier,
237 status = ScreenshotResultProto.DiffResult.Status.FAILED,
238 actual = actual,
239 comparisonStatistics = comparisonResult.comparisonStatistics,
240 expected = expected,
241 diff = comparisonResult.diff,
242 )
243 if (doesCollectScreenshotParityStats) {
244 parityStatsCollector.collectTestStats(testIdentifier, comparisonResult)
245 parityStatsCollector.report()
246 }
247
248 val expectedWidth = expected.width
249 val expectedHeight = expected.height
250 expected.recycle()
251
252 throw AssertionError(
253 "Sizes are different! Expected: [$expectedWidth, $expectedHeight], Actual: [${
254 actual.width}, ${actual.height}]. " +
255 "Force aligned at (0, 0). Comparison stats: '${comparisonResult
256 .comparisonStatistics}'"
257 )
258 }
259
260 val comparisonResult =
261 matcher.compareBitmaps(
262 expected = expected.toIntArray(),
263 given = actual.toIntArray(),
264 width = actual.width,
265 height = actual.height,
266 regions = regions,
267 )
268 if (doesCollectScreenshotParityStats) {
269 parityStatsCollector.collectTestStats(testIdentifier, comparisonResult)
270 parityStatsCollector.report()
271 }
272
273 val status =
274 if (comparisonResult.matches) {
275 ScreenshotResultProto.DiffResult.Status.PASSED
276 } else {
277 ScreenshotResultProto.DiffResult.Status.FAILED
278 }
279
280 if (!comparisonResult.matches) {
281 val expectedWithHighlight = highlightedBitmap(expected, regions)
282 diffEscrowStrategy.reportResult(
283 testIdentifier = testIdentifier,
284 goldenIdentifier = goldenIdentifier,
285 status = status,
286 actual = actual,
287 comparisonStatistics = comparisonResult.comparisonStatistics,
288 expected = expectedWithHighlight,
289 diff = comparisonResult.diff,
290 )
291
292 expectedWithHighlight.recycle()
293 expected.recycle()
294
295 throw AssertionError(
296 "Image mismatch! Comparison stats: '${comparisonResult.comparisonStatistics}'"
297 )
298 }
299
300 expected.recycle()
301 }
302
303 override fun createScreenshotAsserter(config: ScreenshotAsserterConfig): ScreenshotAsserter {
304 return ScreenshotRuleAsserter.Builder(this)
305 .withMatcher(config.matcher)
306 .setOnBeforeScreenshot(config.beforeScreenshot)
307 .setOnAfterScreenshot(config.afterScreenshot)
308 .setScreenshotProvider(config.captureStrategy)
309 .build()
310 }
311
312 /** This will create a new Bitmap with the output (not modifying the [original] Bitmap */
313 private fun highlightedBitmap(original: Bitmap, regions: List<Rect>): Bitmap {
314 if (regions.isEmpty()) return original
315
316 val outputBitmap = original.copy(original.config!!, true)
317 val imageRect = Rect(0, 0, original.width, original.height)
318 val regionLineWidth = 2
319 for (region in regions) {
320 val regionToDraw =
321 Rect(region).apply {
322 inset(-regionLineWidth, -regionLineWidth)
323 intersect(imageRect)
324 }
325
326 repeat(regionLineWidth) {
327 drawRectOnBitmap(outputBitmap, regionToDraw, Color.RED)
328 regionToDraw.inset(1, 1)
329 regionToDraw.intersect(imageRect)
330 }
331 }
332 return outputBitmap
333 }
334
335 private fun drawRectOnBitmap(bitmap: Bitmap, rect: Rect, @ColorInt color: Int) {
336 // Draw top and bottom edges
337 for (x in rect.left until rect.right) {
338 bitmap.setPixel(x, rect.top, color)
339 bitmap.setPixel(x, rect.bottom - 1, color)
340 }
341 // Draw left and right edge
342 for (y in rect.top until rect.bottom) {
343 bitmap.setPixel(rect.left, y, color)
344 bitmap.setPixel(rect.right - 1, y, color)
345 }
346 }
347 }
348
349 typealias BitmapSupplier = () -> Bitmap
350
351 /** Implements a screenshot asserter based on the ScreenshotRule */
352 class ScreenshotRuleAsserter private constructor(private val rule: ScreenshotTestRule) :
353 ScreenshotAsserter {
354 // use the most constraining matcher as default
355 private var matcher: BitmapMatcher = PixelPerfectMatcher()
356 private var beforeScreenshot: Runnable? = null
357 private var afterScreenshot: Runnable? = null
358
359 // use the instrumentation screenshot as default
<lambda>null360 private var screenShotter: BitmapSupplier = { Screenshot.capture().bitmap }
361
362 private var pointerLocationSetting: Int
363 get() = shell("settings get system ${System.POINTER_LOCATION}").trim().toIntOrNull() ?: 0
364 set(value) {
365 shell("settings put system ${System.POINTER_LOCATION} $value")
366 }
367
368 private var showTouchesSetting
369 get() = shell("settings get system ${System.SHOW_TOUCHES}").trim().toIntOrNull() ?: 0
370 set(value) {
371 shell("settings put system ${System.SHOW_TOUCHES} $value")
372 }
373
374 private var prevPointerLocationSetting: Int? = null
375 private var prevShowTouchesSetting: Int? = null
376
377 @Suppress("DEPRECATION")
assertGoldenImagenull378 override fun assertGoldenImage(goldenId: String) {
379 runBeforeScreenshot()
380 var actual: Bitmap? = null
381 try {
382 actual = screenShotter()
383 rule.assertBitmapAgainstGolden(actual, goldenId, matcher)
384 } finally {
385 actual?.recycle()
386 runAfterScreenshot()
387 }
388 }
389
390 @Suppress("DEPRECATION")
assertGoldenImagenull391 override fun assertGoldenImage(goldenId: String, areas: List<Rect>) {
392 runBeforeScreenshot()
393 var actual: Bitmap? = null
394 try {
395 actual = screenShotter()
396 rule.assertBitmapAgainstGolden(actual, goldenId, matcher, areas)
397 } finally {
398 actual?.recycle()
399 runAfterScreenshot()
400 }
401 }
402
runBeforeScreenshotnull403 private fun runBeforeScreenshot() {
404 // Disable Sensitive Content Redaction
405 shell("settings put secure disable_secure_windows 1")
406
407 prevPointerLocationSetting = pointerLocationSetting
408 prevShowTouchesSetting = showTouchesSetting
409
410 if (prevPointerLocationSetting != 0) pointerLocationSetting = 0
411 if (prevShowTouchesSetting != 0) showTouchesSetting = 0
412
413 beforeScreenshot?.run()
414 }
415
runAfterScreenshotnull416 private fun runAfterScreenshot() {
417 // Enable Sensitive Content Redaction
418 shell("settings put secure disable_secure_windows 0")
419 afterScreenshot?.run()
420
421 prevPointerLocationSetting?.let { pointerLocationSetting = it }
422 prevShowTouchesSetting?.let { showTouchesSetting = it }
423 }
424
425 @Deprecated("Use ScreenshotAsserterFactory instead")
426 class Builder(private val rule: ScreenshotTestRule) {
427 private var asserter = ScreenshotRuleAsserter(rule)
428
<lambda>null429 fun withMatcher(matcher: BitmapMatcher): Builder = apply { asserter.matcher = matcher }
430
431 /**
432 * The [Bitmap] produced by [screenshotProvider] will be recycled immediately after
433 * assertions are completed. Therefore, do not retain references to created [Bitmap]s.
434 */
<lambda>null435 fun setScreenshotProvider(screenshotProvider: BitmapSupplier): Builder = apply {
436 asserter.screenShotter = screenshotProvider
437 }
438
setOnBeforeScreenshotnull439 fun setOnBeforeScreenshot(run: Runnable): Builder = apply {
440 asserter.beforeScreenshot = run
441 }
442
<lambda>null443 fun setOnAfterScreenshot(run: Runnable): Builder = apply { asserter.afterScreenshot = run }
444
<lambda>null445 fun build(): ScreenshotAsserter = asserter.also { asserter = ScreenshotRuleAsserter(rule) }
446 }
447 }
448
toIntArraynull449 internal fun Bitmap.toIntArray(): IntArray {
450 val bitmapArray = IntArray(width * height)
451 getPixels(bitmapArray, 0, width, 0, 0, width, height)
452 return bitmapArray
453 }
454
455 /**
456 * Asserts this bitmap against the golden identified by the given name.
457 *
458 * Note: The golden identifier should be unique per your test module (unless you want multiple tests
459 * to match the same golden). The name must not contain extension. You should also avoid adding
460 * strings like "golden", "image" and instead describe what is the golder referring to.
461 *
462 * @param bitmapDiffer The screenshot test rule that provides the comparison and reporting.
463 * @param goldenIdentifier Name of the golden. Allowed characters: 'A-Za-z0-9_-'
464 * @param matcher The algorithm to be used to perform the matching. By default [MSSIMMatcher] is
465 * used.
466 * @see MSSIMMatcher
467 * @see PixelPerfectMatcher
468 */
assertAgainstGoldennull469 fun Bitmap.assertAgainstGolden(
470 bitmapDiffer: BitmapDiffer,
471 goldenIdentifier: String,
472 matcher: BitmapMatcher = MSSIMMatcher(),
473 regions: List<Rect> = emptyList(),
474 ) {
475 bitmapDiffer.assertBitmapAgainstGolden(
476 this,
477 goldenIdentifier,
478 matcher = matcher,
479 regions = regions,
480 )
481 }
482