• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
<lambda>null2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.tools.device.flicker
18 
19 import android.app.Instrumentation
20 import android.device.collectors.BaseMetricListener
21 import android.device.collectors.DataRecord
22 import android.tools.common.FLICKER_TAG
23 import android.tools.common.Logger
24 import android.tools.common.ScenarioBuilder
25 import android.tools.common.flicker.AssertionInvocationGroup
26 import android.tools.common.flicker.FlickerConfig
27 import android.tools.common.flicker.FlickerService
28 import android.tools.common.flicker.TracesCollector
29 import android.tools.common.flicker.assertions.AssertionResult
30 import android.tools.common.flicker.config.FlickerServiceConfig
31 import android.tools.common.flicker.config.ScenarioId
32 import android.tools.common.io.RunStatus
33 import androidx.test.platform.app.InstrumentationRegistry
34 import com.android.internal.annotations.VisibleForTesting
35 import org.junit.runner.Description
36 import org.junit.runner.Result
37 import org.junit.runner.notification.Failure
38 
39 /**
40  * Collects all the Flicker Service's metrics which are then uploaded for analysis and monitoring to
41  * the CrystalBall database.
42  */
43 class FlickerServiceResultsCollector(
44     private val tracesCollector: TracesCollector,
45     private val flickerService: FlickerService =
46         FlickerService(FlickerConfig().use(FlickerServiceConfig.DEFAULT)),
47     instrumentation: Instrumentation = InstrumentationRegistry.getInstrumentation(),
48     private val collectMetricsPerTest: Boolean = true,
49     private val reportOnlyForPassingTests: Boolean = true
50 ) : BaseMetricListener(), IFlickerServiceResultsCollector {
51     private var hasFailedTest = false
52     private var testSkipped = false
53 
54     private val _executionErrors = mutableListOf<Throwable>()
55     override val executionErrors
56         get() = _executionErrors
57 
58     @VisibleForTesting val assertionResults = mutableListOf<AssertionResult>()
59     @VisibleForTesting
60     val assertionResultsByTest = mutableMapOf<Description, Collection<AssertionResult>>()
61     @VisibleForTesting
62     val detectedScenariosByTest = mutableMapOf<Description, Collection<ScenarioId>>()
63 
64     init {
65         setInstrumentation(instrumentation)
66     }
67 
68     override fun onTestRunStart(runData: DataRecord, description: Description) {
69         errorReportingBlock {
70             tracesCollector.cleanup() // Cleanup any trace archives from previous runs
71 
72             Logger.i(LOG_TAG, "onTestRunStart :: collectMetricsPerTest = $collectMetricsPerTest")
73             if (!collectMetricsPerTest) {
74                 hasFailedTest = false
75                 val scenario =
76                     ScenarioBuilder()
77                         .forClass(description.testClass.canonicalName)
78                         .withDescriptionOverride("")
79                         .build()
80                 tracesCollector.start(scenario)
81             }
82         }
83     }
84 
85     override fun onTestStart(testData: DataRecord, description: Description) {
86         errorReportingBlock {
87             Logger.i(LOG_TAG, "onTestStart :: collectMetricsPerTest = $collectMetricsPerTest")
88             if (collectMetricsPerTest) {
89                 hasFailedTest = false
90                 val scenario =
91                     ScenarioBuilder()
92                         .forClass(
93                             "${description.testClass.canonicalName}#${description.methodName}"
94                         )
95                         .withDescriptionOverride("")
96                         .build()
97                 tracesCollector.start(scenario)
98             }
99             testSkipped = false
100         }
101     }
102 
103     override fun onTestFail(testData: DataRecord, description: Description, failure: Failure) {
104         errorReportingBlock {
105             Logger.i(LOG_TAG, "onTestFail")
106             hasFailedTest = true
107         }
108     }
109 
110     override fun testSkipped(description: Description) {
111         errorReportingBlock {
112             Logger.i(LOG_TAG, "testSkipped")
113             testSkipped = true
114         }
115     }
116 
117     override fun onTestEnd(testData: DataRecord, description: Description) {
118         errorReportingBlock {
119             Logger.i(LOG_TAG, "onTestEnd :: collectMetricsPerTest = $collectMetricsPerTest")
120             if (collectMetricsPerTest && !testSkipped) {
121                 stopTracingAndCollectFlickerMetrics(testData, description)
122             }
123         }
124     }
125 
126     override fun onTestRunEnd(runData: DataRecord, result: Result) {
127         errorReportingBlock {
128             Logger.i(LOG_TAG, "onTestRunEnd :: collectMetricsPerTest = $collectMetricsPerTest")
129             if (!collectMetricsPerTest) {
130                 stopTracingAndCollectFlickerMetrics(runData)
131             }
132         }
133     }
134 
135     private fun stopTracingAndCollectFlickerMetrics(
136         dataRecord: DataRecord,
137         description: Description? = null
138     ) {
139         errorReportingBlock {
140             Logger.i(LOG_TAG, "Stopping trace collection")
141             val reader = tracesCollector.stop()
142             Logger.i(LOG_TAG, "Stopped trace collection")
143             if (reportOnlyForPassingTests && hasFailedTest) {
144                 return@errorReportingBlock
145             }
146 
147             try {
148                 Logger.i(LOG_TAG, "Processing traces")
149                 val scenarios = flickerService.detectScenarios(reader)
150                 val results = scenarios.flatMap { it.generateAssertions() }.map { it.execute() }
151                 reader.artifact.updateStatus(RunStatus.RUN_EXECUTED)
152                 Logger.i(LOG_TAG, "Got ${results.size} results")
153                 assertionResults.addAll(results)
154                 if (description != null) {
155                     require(assertionResultsByTest[description] == null) {
156                         "Test description already contains flicker assertion results."
157                     }
158                     require(detectedScenariosByTest[description] == null) {
159                         "Test description already contains detected scenarios."
160                     }
161                     assertionResultsByTest[description] = results
162                     detectedScenariosByTest[description] = scenarios.map { it.type }.distinct()
163                 }
164                 if (results.any { it.failed }) {
165                     reader.artifact.updateStatus(RunStatus.ASSERTION_FAILED)
166                 } else {
167                     reader.artifact.updateStatus(RunStatus.ASSERTION_SUCCESS)
168                 }
169 
170                 Logger.v(LOG_TAG, "Adding metric $FLICKER_ASSERTIONS_COUNT_KEY = ${results.size}")
171                 dataRecord.addStringMetric(FLICKER_ASSERTIONS_COUNT_KEY, "${results.size}")
172 
173                 val aggregatedResults = processFlickerResults(results)
174                 collectMetrics(dataRecord, aggregatedResults)
175             } finally {
176                 Logger.v(LOG_TAG, "Adding metric $WINSCOPE_FILE_PATH_KEY = ${reader.artifactPath}")
177                 dataRecord.addStringMetric(WINSCOPE_FILE_PATH_KEY, reader.artifactPath)
178             }
179         }
180     }
181 
182     private fun processFlickerResults(
183         results: Collection<AssertionResult>
184     ): Map<String, AggregatedFlickerResult> {
185         val aggregatedResults = mutableMapOf<String, AggregatedFlickerResult>()
186         for (result in results) {
187             val key = getKeyForAssertionResult(result)
188             if (!aggregatedResults.containsKey(key)) {
189                 aggregatedResults[key] = AggregatedFlickerResult()
190             }
191             aggregatedResults[key]!!.addResult(result)
192         }
193         return aggregatedResults
194     }
195 
196     private fun collectMetrics(
197         data: DataRecord,
198         aggregatedResults: Map<String, AggregatedFlickerResult>
199     ) {
200         val it = aggregatedResults.entries.iterator()
201 
202         while (it.hasNext()) {
203             val (key, aggregatedResult) = it.next()
204             aggregatedResult.results.forEachIndexed { index, result ->
205                 val resultStatus = if (result.passed) 0 else 1
206                 Logger.v(LOG_TAG, "Adding metric ${key}_$index = $resultStatus")
207                 data.addStringMetric("${key}_$index", "$resultStatus")
208             }
209         }
210     }
211 
212     private fun errorReportingBlock(function: () -> Unit) {
213         try {
214             function()
215         } catch (e: Throwable) {
216             Logger.e(FLICKER_TAG, "Error executing in FlickerServiceResultsCollector", e)
217             _executionErrors.add(e)
218         }
219     }
220 
221     override fun resultsForTest(description: Description): Collection<AssertionResult> {
222         val resultsForTest = assertionResultsByTest[description]
223         requireNotNull(resultsForTest) { "No results set for test $description" }
224         return resultsForTest
225     }
226 
227     override fun detectedScenariosForTest(description: Description): Collection<ScenarioId> {
228         val scenariosForTest = detectedScenariosByTest[description]
229         requireNotNull(scenariosForTest) { "No detected scenarios set for test $description" }
230         return scenariosForTest
231     }
232 
233     companion object {
234         // Unique prefix to add to all FaaS metrics to identify them
235         const val FAAS_METRICS_PREFIX = "FAAS"
236         private const val LOG_TAG = "$FLICKER_TAG-Collector"
237         const val WINSCOPE_FILE_PATH_KEY = "winscope_file_path"
238         const val FLICKER_ASSERTIONS_COUNT_KEY = "flicker_assertions_count"
239 
240         fun getKeyForAssertionResult(result: AssertionResult): String {
241             return "$FAAS_METRICS_PREFIX::${result.name}"
242         }
243 
244         class AggregatedFlickerResult {
245             val results = mutableListOf<AssertionResult>()
246             var failures = 0
247             var passes = 0
248             val errors = mutableListOf<String>()
249             var invocationGroup: AssertionInvocationGroup? = null
250 
251             fun addResult(result: AssertionResult) {
252                 results.add(result)
253 
254                 if (result.failed) {
255                     failures++
256                     result.assertionErrors.forEach {
257                         errors.add(it.message ?: "FAILURE WITHOUT ERROR MESSAGE...")
258                     }
259                 } else {
260                     passes++
261                 }
262 
263                 if (invocationGroup == null) {
264                     invocationGroup = result.stabilityGroup
265                 }
266 
267                 if (invocationGroup != result.stabilityGroup) {
268                     error("Unexpected assertion group mismatch")
269                 }
270             }
271         }
272     }
273 }
274