1 /* <lambda>null2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.wm.flicker.rules 18 19 import androidx.test.platform.app.InstrumentationRegistry 20 import com.android.server.wm.flicker.FlickerTestParameter 21 import com.android.server.wm.flicker.dsl.AssertionTag 22 import com.android.server.wm.flicker.service.FlickerService 23 import com.android.server.wm.flicker.service.assertors.AssertionConfigParser 24 import com.android.server.wm.flicker.service.assertors.AssertionData 25 import com.android.server.wm.flicker.traces.layers.LayersTraceSubject 26 import com.android.server.wm.flicker.traces.windowmanager.WindowManagerTraceSubject 27 import com.android.server.wm.traces.common.errors.ErrorTrace 28 import org.junit.rules.TestWatcher 29 import org.junit.runner.Description 30 import java.util.regex.Pattern 31 32 /** 33 * A test rule reusing flicker data from [FlickerTestParameter], and fetching the traces 34 * to call the WM Flicker Service after the test and report metrics from the results. 35 */ 36 @Deprecated("This test rule should be only used with legacy flicker tests. " + 37 "For new tests use WMFlickerServiceRule instead") 38 class WMFlickerServiceRuleForTestSpec( 39 private val testSpec: FlickerTestParameter 40 ) : TestWatcher() { 41 private val flickerResultsCollector = FlickerResultsCollector() 42 43 init { 44 flickerResultsCollector.instrumentation = InstrumentationRegistry.getInstrumentation() 45 } 46 47 override fun starting(description: Description?) { 48 val runParameterString = extractRunParameterStringFromMethodName(description?.methodName) 49 val cuj = if (runParameterString.isNullOrBlank()) { 50 description?.className 51 } else { 52 "${description?.className}[$runParameterString]" 53 } 54 flickerResultsCollector.setCriticalUserJourneyName(cuj) 55 flickerResultsCollector.testStarted(description) 56 } 57 58 private fun extractRunParameterStringFromMethodName(methodName: String?): String? { 59 if (methodName.isNullOrBlank()) { 60 return null 61 } 62 63 val pattern = Pattern.compile("\\[(.+)\\]") 64 val matcher = pattern.matcher(methodName) 65 return if (matcher.find()) { 66 matcher.group(1) 67 } else { 68 null 69 } 70 } 71 72 override fun finished(description: Description?) { 73 flickerResultsCollector.testFinished(description) 74 } 75 76 fun getMetrics(): Map<String, Int> { 77 return flickerResultsCollector.getMetrics() 78 } 79 80 private fun checkFlicker(category: String): List<ErrorTrace> { 81 // run flicker if it was not executed before 82 testSpec.result ?: testSpec.assertWm { isNotEmpty() } 83 84 val errors = mutableListOf<ErrorTrace>() 85 val result = testSpec.result ?: error("No flicker results for $testSpec") 86 val assertions = AssertionData.readConfiguration().filter { it.category == category } 87 val flickerService = FlickerService(assertions) 88 89 result.successfulRuns 90 .filter { it.assertionTag == AssertionTag.ALL } 91 .filter { 92 val hasWmTrace = it.wmSubject?.let { true } ?: false 93 val hasLayersTrace = it.layersSubject?.let { true } ?: false 94 hasWmTrace || hasLayersTrace 95 } 96 .forEach { run -> 97 val wmSubject = run.wmSubject as WindowManagerTraceSubject 98 val layersSubject = run.layersSubject as LayersTraceSubject 99 100 val outputDir = run.mTraceFile?.traceFile?.parent 101 ?: error("Output dir not detected") 102 103 val wmTrace = wmSubject.trace 104 val layersTrace = layersSubject.trace 105 val (errorTrace, assertionsResults) = 106 flickerService.process(wmTrace, layersTrace, outputDir) 107 errors.add(errorTrace) 108 flickerResultsCollector.postRunResults(assertionsResults) 109 } 110 111 return errors 112 } 113 114 /** 115 * @return true if all assertions pass, false otherwise 116 */ 117 @JvmOverloads 118 fun checkPresubmitAssertions(failOnAssertionFailure: Boolean = true): Boolean { 119 val errors = checkFlicker(AssertionConfigParser.PRESUBMIT_KEY) 120 return handleErrors(errors, failOnAssertionFailure) 121 } 122 123 /** 124 * @return true if all assertions pass, false otherwise 125 */ 126 fun checkPostsubmitAssertions(failOnAssertionFailure: Boolean = true): Boolean { 127 val errors = checkFlicker(AssertionConfigParser.POSTSUBMIT_KEY) 128 return handleErrors(errors, failOnAssertionFailure) 129 } 130 131 /** 132 * @return true if all assertions pass, false otherwise 133 */ 134 fun checkFlakyAssertions(failOnAssertionFailure: Boolean = true): Boolean { 135 val errors = checkFlicker(AssertionConfigParser.FLAKY_KEY) 136 return handleErrors(errors, failOnAssertionFailure) 137 } 138 139 private fun handleErrors(errors: List<ErrorTrace>, failOnAssertionFailure: Boolean): Boolean { 140 return if (failOnAssertionFailure) { 141 failIfAnyError(errors) 142 } else { 143 !hasErrors(errors) 144 } 145 } 146 147 /** 148 * @return true if there were no errors 149 */ 150 private fun failIfAnyError(errors: List<ErrorTrace>): Boolean { 151 val errorMsg = errors.joinToString("\n") { runs -> 152 runs.entries.joinToString { state -> 153 state.errors.joinToString { "${it.assertionName}\n${it.message}" } 154 } 155 } 156 if (errorMsg.isNotEmpty()) { 157 error(errorMsg) 158 } 159 return true 160 } 161 162 private fun hasErrors(errors: List<ErrorTrace>): Boolean { 163 return errors.any { runs -> 164 runs.entries.any { state -> 165 state.errors.any { error -> 166 error.assertionName.isNotEmpty() 167 } 168 } 169 } 170 } 171 } 172