1 /*
<lambda>null2  * Copyright 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.benchmark
18 
19 import android.os.Build
20 import android.os.Bundle
21 import android.util.Log
22 import androidx.annotation.RestrictTo
23 import androidx.annotation.VisibleForTesting
24 import androidx.test.platform.app.InstrumentationRegistry
25 
26 /** This allows tests to override arguments from code */
27 @RestrictTo(RestrictTo.Scope.LIBRARY)
28 @get:RestrictTo(RestrictTo.Scope.LIBRARY)
29 @set:RestrictTo(RestrictTo.Scope.LIBRARY)
30 @VisibleForTesting
31 var argumentSource: Bundle? = null
32 
33 @Suppress("NullableBooleanElvis") // suggestion makes boolean argument defaults less clear
34 @RestrictTo(RestrictTo.Scope.LIBRARY_GROUP)
35 object Arguments {
36     // public properties are shared by micro + macro benchmarks
37     val suppressedErrors: Set<String>
38 
39     /**
40      * Set to true to enable androidx.tracing.perfetto tracepoints (such as composition tracing)
41      *
42      * Note that when StartupMode.COLD is used, additional work must be performed during target app
43      * startup to initialize tracing.
44      */
45     val perfettoSdkTracingEnable: Boolean
46         get() = perfettoSdkTracingEnableOverride ?: _perfettoSdkTracingEnable
47 
48     private val _perfettoSdkTracingEnable: Boolean
49     @VisibleForTesting var perfettoSdkTracingEnableOverride: Boolean? = null
50 
51     /**
52      * Base URL for help articles for Startup Insights.
53      *
54      * This property should only be used while the Startup Insights feature is under development. It
55      * can be overridden for testing purposes using [startupInsightsHelpUrlBaseOverride].
56      */
57     val startupInsightsHelpUrlBase: String?
58         get() = startupInsightsHelpUrlBaseOverride ?: _startupInsightsHelpUrlBase
59 
60     private val _startupInsightsHelpUrlBase: String?
61     @VisibleForTesting var startupInsightsHelpUrlBaseOverride: String? = null
62 
63     val enabledRules: Set<RuleType>
64 
65     enum class RuleType {
66         Microbenchmark,
67         Macrobenchmark,
68         BaselineProfile
69     }
70 
71     val enableCompilation: Boolean
72     val killProcessDelayMillis: Long
73     val dryRunMode: Boolean
74     val dropShadersEnable: Boolean
75     val dropShadersThrowOnFailure: Boolean
76     val skipBenchmarksOnEmulator: Boolean
77     val saveProfileWaitMillis: Long
78     val killExistingPerfettoRecordings: Boolean
79 
80     // internal properties are microbenchmark only
81     internal val outputEnable: Boolean
82     internal val startupMode: Boolean
83     internal val iterations: Int?
84     internal val profiler: Profiler?
85     internal val profilerDefault: Boolean
86     internal val profilerSampleFrequencyHz: Int
87     internal val profilerSampleDurationSeconds: Long
88     internal val profilerSkipWhenDurationRisksAnr: Boolean
89     internal val profilerPerfCompareEnable: Boolean
90     internal val thermalThrottleSleepDurationSeconds: Long
91     val cpuEventCounterEnable: Boolean // non-internal, checked in CpuEventCounterBenchmark
92     internal val cpuEventCounterMask: Int
93     internal val requireAot: Boolean
94     internal val requireJitDisabledIfRooted: Boolean
95     val throwOnMainThreadMeasureRepeated: Boolean // non-internal, used in BenchmarkRule
96     val measureRepeatedOnMainThrowOnDeadline: Boolean // non-internal, used in BenchmarkRule
97 
98     internal var error: String? = null
99     internal val additionalTestOutputDir: String?
100 
101     private val targetPackageName: String?
102 
103     val payload: Map<String, String>
104 
105     private const val prefix = "androidx.benchmark."
106 
107     private fun Bundle.getBenchmarkArgument(key: String, defaultValue: String? = null) =
108         getString(prefix + key, defaultValue)
109 
110     private fun Bundle.getBenchmarkArgumentsWithPrefix(key: String): Map<String, String> {
111         val combinedPrefix = "$prefix$key."
112         val bundle = this
113         return buildMap {
114             bundle
115                 .keySet()
116                 .filter { it.startsWith(combinedPrefix) }
117                 .forEach { put(it.substringAfter(combinedPrefix), getString(it, null)) }
118         }
119     }
120 
121     private fun Bundle.getProfiler(outputIsEnabled: Boolean): Pair<Profiler?, Boolean> {
122         val argumentName = "profiling.mode"
123         val argumentValue = getBenchmarkArgument(argumentName, "DEFAULT_VAL")
124         if (argumentValue == "DEFAULT_VAL") {
125             return if (Build.VERSION.SDK_INT <= 21) {
126                 // Have observed stack corruption on API 21, we haven't spent the time to find out
127                 // why, or if it's better on other low API levels. See b/300658578
128                 // TODO: consider adding warning here
129                 null to true
130             } else if (DeviceInfo.methodTracingAffectsMeasurements) {
131                 // We warn here instead of in Errors since this doesn't affect all measurements -
132                 // BenchmarkState throws rather than measuring incorrectly, and the first benchmark
133                 // can still measure with a trace safely
134                 InstrumentationResults.scheduleIdeWarningOnNextReport(
135                     """
136                     NOTE: Your device is running a version of ART where method tracing is known to
137                     affect performance measurement after trace capture, so method tracing is
138                     off by default.
139 
140                     To use method tracing, either flash this device, use a different device, or
141                     enable method tracing with MicrobenchmarkConfig / instrumentation argument, and
142                     only run one test at a time.
143 
144                     For more information, see https://issuetracker.google.com/issues/316174880
145                     """
146                         .trimIndent()
147                 )
148                 null to true
149             } else MethodTracing to true
150         }
151 
152         val profiler = Profiler.getByName(argumentValue)
153         if (
154             profiler == null &&
155                 argumentValue.isNotEmpty() &&
156                 // 'none' is documented as noop (and works better in gradle than
157                 // an empty string, if a value must be specified)
158                 argumentValue.trim().lowercase() != "none"
159         ) {
160             error = "Could not parse $prefix$argumentName=$argumentValue"
161             return null to false
162         }
163         if (profiler?.requiresLibraryOutputDir == true && !outputIsEnabled) {
164             error = "Output is not enabled, so cannot profile with mode $argumentValue"
165             return null to false
166         }
167         return profiler to false
168     }
169 
170     // note: initialization may happen at any time
171     init {
172         val arguments = argumentSource ?: InstrumentationRegistry.getArguments()
173 
174         dryRunMode = arguments.getBenchmarkArgument("dryRunMode.enable")?.toBoolean() ?: false
175 
176         startupMode =
177             !dryRunMode &&
178                 (arguments.getBenchmarkArgument("startupMode.enable")?.toBoolean() ?: false)
179 
180         outputEnable =
181             !dryRunMode && (arguments.getBenchmarkArgument("output.enable")?.toBoolean() ?: true)
182 
183         iterations = arguments.getBenchmarkArgument("iterations")?.toInt()
184 
185         targetPackageName = arguments.getBenchmarkArgument("targetPackageName", defaultValue = null)
186 
187         _perfettoSdkTracingEnable =
188             arguments.getBenchmarkArgument("perfettoSdkTracing.enable")?.toBoolean()
189                 // fullTracing.enable is the legacy/compat name
190                 ?: arguments.getBenchmarkArgument("fullTracing.enable")?.toBoolean()
191                 ?: false
192 
193         _startupInsightsHelpUrlBase =
194             arguments.getBenchmarkArgument("startupInsights.helpUrlBase", defaultValue = null)
195 
196         // Transform comma-delimited list into set of suppressed errors
197         // E.g. "DEBUGGABLE, UNLOCKED" -> setOf("DEBUGGABLE", "UNLOCKED")
198         suppressedErrors =
199             arguments
200                 .getBenchmarkArgument("suppressErrors", "")
201                 .split(',')
202                 .map { it.trim() }
203                 .filter { it.isNotEmpty() }
204                 .toSet()
205 
206         skipBenchmarksOnEmulator =
207             arguments.getBenchmarkArgument("skipBenchmarksOnEmulator")?.toBoolean() ?: false
208 
209         enabledRules =
210             arguments
211                 .getBenchmarkArgument(
212                     key = "enabledRules",
213                     defaultValue = RuleType.values().joinToString(separator = ",") { it.toString() }
214                 )
215                 .run {
216                     if (this.lowercase() == "none") {
217                         emptySet()
218                     } else {
219                         // parse comma-delimited list
220                         try {
221                             this.split(',')
222                                 .map { it.trim() }
223                                 .filter { it.isNotEmpty() }
224                                 .map { arg ->
225                                     RuleType.values().find {
226                                         arg.lowercase() == it.toString().lowercase()
227                                     } ?: throw Throwable("unable to find $arg")
228                                 }
229                                 .toSet()
230                         } catch (e: Throwable) {
231                             // defer parse error, so it doesn't show up as a missing class
232                             val allRules = RuleType.values()
233                             val allRulesString = allRules.joinToString(",") { it.toString() }
234                             error =
235                                 "unable to parse enabledRules='$this', should be 'None' or" +
236                                     " comma-separated list of supported ruletypes: $allRulesString"
237                             allRules
238                                 .toSet() // don't filter tests, so we have an opportunity to throw
239                         }
240                     }
241                 }
242 
243         // compilation defaults to disabled if dryRunMode is on
244         enableCompilation =
245             arguments.getBenchmarkArgument("compilation.enabled")?.toBoolean() ?: !dryRunMode
246 
247         val profilerState = arguments.getProfiler(outputEnable)
248         profiler = profilerState.first
249         profilerDefault = profilerState.second
250         profilerSampleFrequencyHz =
251             arguments.getBenchmarkArgument("profiling.sampleFrequency")?.ifBlank { null }?.toInt()
252                 ?: 1000
253         profilerSampleDurationSeconds =
254             arguments
255                 .getBenchmarkArgument("profiling.sampleDurationSeconds")
256                 ?.ifBlank { null }
257                 ?.toLong() ?: 5
258         profilerSkipWhenDurationRisksAnr =
259             arguments.getBenchmarkArgument("profiling.skipWhenDurationRisksAnr")?.toBoolean()
260                 ?: true
261         profilerPerfCompareEnable =
262             arguments.getBenchmarkArgument("profiling.perfCompare.enable")?.toBoolean() ?: false
263         if (profiler != null) {
264             Log.d(
265                 BenchmarkState.TAG,
266                 "Profiler ${profiler.javaClass.simpleName}, freq " +
267                     "$profilerSampleFrequencyHz, duration $profilerSampleDurationSeconds"
268             )
269         }
270 
271         val cpuEventsDesired =
272             arguments.getBenchmarkArgument("cpuEventCounter.enable")?.toBoolean() ?: false
273         cpuEventCounterEnable =
274             when {
275                 !cpuEventsDesired -> {
276                     false // not attempting to use
277                 }
278                 dryRunMode -> {
279                     Log.d(
280                         BenchmarkState.TAG,
281                         "Ignoring request for cpuEventCounter due to dryRunMode=true"
282                     )
283                     false
284                 }
285                 !DeviceInfo.supportsCpuEventCounters -> {
286                     Log.d(
287                         BenchmarkState.TAG,
288                         "Ignoring request for cpuEventCounter due to unrooted device"
289                     )
290                     false
291                 }
292                 else -> true
293             }
294         cpuEventCounterMask =
295             if (cpuEventCounterEnable) {
296                 arguments
297                     .getBenchmarkArgument(
298                         "cpuEventCounter.events",
299                         "Instructions,CpuCycles,BranchMisses"
300                     )
301                     .split(",")
302                     .map { eventName -> CpuEventCounter.Event.valueOf(eventName) }
303                     .getFlags()
304             } else {
305                 0x0
306             }
307         if (cpuEventCounterEnable && cpuEventCounterMask == 0x0) {
308             error =
309                 "Must set a cpu event counters mask to use counters." +
310                     " See CpuEventCounters.Event for flag definitions."
311         }
312 
313         thermalThrottleSleepDurationSeconds =
314             arguments
315                 .getBenchmarkArgument("thermalThrottle.sleepDurationSeconds")
316                 ?.ifBlank { null }
317                 ?.toLong() ?: 90
318 
319         additionalTestOutputDir = arguments.getString("additionalTestOutputDir")
320         Log.d(BenchmarkState.TAG, "additionalTestOutputDir=$additionalTestOutputDir")
321 
322         killProcessDelayMillis =
323             arguments.getBenchmarkArgument("killProcessDelayMillis")?.toLong() ?: 0L
324 
325         saveProfileWaitMillis =
326             arguments.getBenchmarkArgument("saveProfileWaitMillis")?.toLong() ?: 1_000L
327 
328         dropShadersEnable =
329             arguments.getBenchmarkArgument("dropShaders.enable")?.toBoolean() ?: true
330         dropShadersThrowOnFailure =
331             arguments.getBenchmarkArgument("dropShaders.throwOnFailure")?.toBoolean() ?: true
332 
333         measureRepeatedOnMainThrowOnDeadline =
334             arguments
335                 .getBenchmarkArgument("measureRepeatedOnMainThread.throwOnDeadline")
336                 ?.toBoolean() ?: true
337 
338         requireAot = arguments.getBenchmarkArgument("requireAot")?.toBoolean() ?: false
339         requireJitDisabledIfRooted =
340             arguments.getBenchmarkArgument("requireJitDisabledIfRooted")?.toBoolean() ?: false
341 
342         throwOnMainThreadMeasureRepeated =
343             arguments.getBenchmarkArgument("throwOnMainThreadMeasureRepeated")?.toBoolean() ?: false
344 
345         killExistingPerfettoRecordings =
346             arguments.getBenchmarkArgument("killExistingPerfettoRecordings")?.toBoolean()
347                 // below is a temporary workaround for compat, see b/399818365
348                 ?: arguments.getString("killExistingPerfettoRecordings")?.toBoolean()
349                 ?: true
350 
351         if (arguments.getString("orchestratorService") != null) {
352             InstrumentationResults.scheduleIdeWarningOnNextReport(
353                 """
354                     AndroidX Benchmark does not support running with the AndroidX Test Orchestrator.
355 
356                     AndroidX benchmarks (micro and macro) produce one JSON file per test module,
357                     which together with Test Orchestrator restarting the process frequently causes
358                     benchmark output JSON files to be repeatedly overwritten during the test.
359                     """
360                     .trimIndent()
361             )
362         }
363         payload = arguments.getBenchmarkArgumentsWithPrefix("output.payload")
364     }
365 
366     fun macrobenchMethodTracingEnabled(): Boolean {
367         return when {
368             dryRunMode -> false
369             profilerDefault -> false // don't enable tracing by default in macrobench
370             else -> profiler == MethodTracing
371         }
372     }
373 
374     fun throwIfError() {
375         if (error != null) {
376             throw AssertionError(error)
377         }
378     }
379 
380     /**
381      * Retrieves the target app package name from the instrumentation runner arguments. Note that
382      * this is supported only when MacrobenchmarkRule and BaselineProfileRule are used with the
383      * baseline profile gradle plugin. This feature requires AGP 8.3.0-alpha10 as minimum version.
384      */
385     fun getTargetPackageNameOrThrow(): String =
386         targetPackageName
387             ?: throw IllegalArgumentException(
388                 """
389         Can't retrieve the target package name from instrumentation arguments.
390         This feature requires the baseline profile gradle plugin with minimum version 1.3.0-alpha01
391         and the Android Gradle Plugin minimum version 8.3.0-alpha10.
392         Please ensure your project has the correct versions in order to use this feature.
393     """
394                     .trimIndent()
395             )
396 }
397