1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.tools.flicker 18 19 import android.tools.Rotation 20 import android.tools.Timestamps 21 import android.tools.flicker.assertions.AssertionResult 22 import android.tools.flicker.assertions.FlickerTest 23 import android.tools.flicker.assertors.AssertionTemplate 24 import android.tools.flicker.config.FlickerConfigEntry 25 import android.tools.flicker.config.ScenarioId 26 import android.tools.flicker.extractors.ScenarioExtractor 27 import android.tools.flicker.extractors.TraceSlice 28 import android.tools.flicker.subject.exceptions.SimpleFlickerAssertionError 29 import android.tools.getTraceReaderFromScenario 30 import android.tools.io.Reader 31 import com.google.common.truth.Truth 32 import org.junit.Test 33 34 class ScenarioInstanceTest { 35 @Test willReportFlickerAssertionsnull36 fun willReportFlickerAssertions() { 37 val errorMessage = "My Error" 38 val scenarioInstance = 39 ScenarioInstanceImpl( 40 config = 41 FlickerConfigEntry( 42 scenarioId = ScenarioId("MY_CUSTOM_SCENARIO"), 43 extractor = 44 object : ScenarioExtractor { 45 override fun extract(reader: Reader): List<TraceSlice> { 46 return listOf(TraceSlice(Timestamps.min(), Timestamps.max())) 47 } 48 }, 49 assertions = 50 mapOf( 51 object : AssertionTemplate("myAssertionSingle") { 52 override fun doEvaluate( 53 scenarioInstance: ScenarioInstance, 54 flicker: FlickerTest 55 ) { 56 flicker.assertLayers { 57 throw SimpleFlickerAssertionError(errorMessage) 58 } 59 } 60 } to AssertionInvocationGroup.BLOCKING, 61 object : AssertionTemplate("myAssertionMultiple") { 62 override fun doEvaluate( 63 scenarioInstance: ScenarioInstance, 64 flicker: FlickerTest 65 ) { 66 flicker.assertLayers { 67 // No errors 68 } 69 flicker.assertLayers { 70 throw SimpleFlickerAssertionError(errorMessage) 71 } 72 flicker.assertWmStart { 73 throw SimpleFlickerAssertionError(errorMessage) 74 } 75 flicker.assertWm { 76 // No errors 77 } 78 } 79 } to AssertionInvocationGroup.BLOCKING 80 ), 81 enabled = true 82 ), 83 startRotation = Rotation.ROTATION_0, 84 endRotation = Rotation.ROTATION_90, 85 startTimestamp = Timestamps.min(), 86 endTimestamp = Timestamps.max(), 87 reader = getTraceReaderFromScenario("AppLaunch"), 88 ) 89 90 val assertions = scenarioInstance.generateAssertions() 91 Truth.assertThat(assertions).hasSize(2) 92 93 val results = assertions.map { it.execute() } 94 Truth.assertThat(results.map { it.name }).contains("MY_CUSTOM_SCENARIO::myAssertionSingle") 95 Truth.assertThat(results.map { it.name }) 96 .contains("MY_CUSTOM_SCENARIO::myAssertionMultiple") 97 98 val singleAssertionResult = 99 results.first { it.name == "MY_CUSTOM_SCENARIO::myAssertionSingle" } 100 val multipleAssertionResult = 101 results.first { it.name == "MY_CUSTOM_SCENARIO::myAssertionMultiple" } 102 103 Truth.assertThat(singleAssertionResult.status).isEqualTo(AssertionResult.Status.FAIL) 104 Truth.assertThat(singleAssertionResult.assertionErrors).hasSize(1) 105 Truth.assertThat(singleAssertionResult.assertionErrors.first()) 106 .hasMessageThat() 107 .startsWith(errorMessage) 108 109 Truth.assertThat(multipleAssertionResult.status).isEqualTo(AssertionResult.Status.FAIL) 110 Truth.assertThat(multipleAssertionResult.assertionErrors).hasSize(2) 111 Truth.assertThat(multipleAssertionResult.assertionErrors.first()) 112 .hasMessageThat() 113 .startsWith(errorMessage) 114 Truth.assertThat(multipleAssertionResult.assertionErrors.last()) 115 .hasMessageThat() 116 .startsWith(errorMessage) 117 } 118 119 @Test willReportMainBlockAssertionsnull120 fun willReportMainBlockAssertions() { 121 val errorMessage = "My Error" 122 val scenarioInstance = 123 ScenarioInstanceImpl( 124 config = 125 FlickerConfigEntry( 126 scenarioId = ScenarioId("MY_CUSTOM_SCENARIO"), 127 extractor = 128 object : ScenarioExtractor { 129 override fun extract(reader: Reader): List<TraceSlice> { 130 return listOf(TraceSlice(Timestamps.min(), Timestamps.max())) 131 } 132 }, 133 assertions = 134 mapOf( 135 object : AssertionTemplate("myAssertion1") { 136 override fun doEvaluate( 137 scenarioInstance: ScenarioInstance, 138 flicker: FlickerTest 139 ) { 140 throw SimpleFlickerAssertionError(errorMessage) 141 } 142 } to AssertionInvocationGroup.BLOCKING, 143 object : AssertionTemplate("myAssertion2") { 144 override fun doEvaluate( 145 scenarioInstance: ScenarioInstance, 146 flicker: FlickerTest 147 ) { 148 flicker.assertLayers { 149 throw SimpleFlickerAssertionError("Some flicker error") 150 } 151 throw SimpleFlickerAssertionError(errorMessage) 152 } 153 } to AssertionInvocationGroup.BLOCKING 154 ), 155 enabled = true 156 ), 157 startRotation = Rotation.ROTATION_0, 158 endRotation = Rotation.ROTATION_90, 159 startTimestamp = Timestamps.min(), 160 endTimestamp = Timestamps.max(), 161 reader = getTraceReaderFromScenario("AppLaunch"), 162 ) 163 164 val assertions = scenarioInstance.generateAssertions() 165 Truth.assertThat(assertions).hasSize(2) 166 167 val results = assertions.map { it.execute() } 168 Truth.assertThat(results.map { it.name }).contains("MY_CUSTOM_SCENARIO::myAssertion1") 169 Truth.assertThat(results.map { it.name }).contains("MY_CUSTOM_SCENARIO::myAssertion2") 170 171 val assertion1Result = results.first { it.name == "MY_CUSTOM_SCENARIO::myAssertion1" } 172 Truth.assertThat(assertion1Result.status).isEqualTo(AssertionResult.Status.FAIL) 173 Truth.assertThat(assertion1Result.assertionErrors).hasSize(1) 174 Truth.assertThat(assertion1Result.assertionErrors.first()) 175 .hasMessageThat() 176 .startsWith(errorMessage) 177 178 val assertion2Result = results.first { it.name == "MY_CUSTOM_SCENARIO::myAssertion2" } 179 Truth.assertThat(assertion2Result.status).isEqualTo(AssertionResult.Status.FAIL) 180 Truth.assertThat(assertion2Result.assertionErrors).hasSize(2) 181 Truth.assertThat(assertion2Result.assertionErrors.first().message) 182 .contains("Some flicker error") 183 Truth.assertThat(assertion2Result.assertionErrors.drop(1).first().message) 184 .contains(errorMessage) 185 } 186 } 187