1 /*
2  * Copyright 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.benchmark.macro.junit4
18 
19 import androidx.annotation.IntRange
20 import androidx.benchmark.Arguments
21 import androidx.benchmark.ExperimentalBenchmarkConfigApi
22 import androidx.benchmark.ExperimentalConfig
23 import androidx.benchmark.macro.CompilationMode
24 import androidx.benchmark.macro.MacrobenchmarkScope
25 import androidx.benchmark.macro.Metric
26 import androidx.benchmark.macro.StartupMode
27 import androidx.benchmark.macro.macrobenchmarkWithStartupMode
28 import androidx.benchmark.perfetto.PerfettoConfig
29 import org.junit.Assume.assumeTrue
30 import org.junit.rules.TestRule
31 import org.junit.runner.Description
32 import org.junit.runners.model.Statement
33 
34 /**
35  * JUnit rule for benchmarking large app operations like startup, scrolling, or animations.
36  *
37  * See the [Macrobenchmark Guide](https://developer.android.com/studio/profile/macrobenchmark) for
38  * more information on macrobenchmarks.
39  *
40  * @sample androidx.benchmark.samples.macrobenchmarkRuleSample
41  */
42 class MacrobenchmarkRule : TestRule {
43     private lateinit var currentDescription: Description
44 
45     /**
46      * Measure behavior of the specified [packageName] given a set of [metrics].
47      *
48      * This performs a macrobenchmark with the below control flow:
49      * ```
50      *     resetAppCompilation()
51      *     compile(compilationMode)
52      *     repeat(iterations) {
53      *         setupBlock()
54      *         captureTraceAndMetrics {
55      *             measureBlock()
56      *         }
57      *     }
58      * ```
59      *
60      * @param packageName ApplicationId / Application manifest package name of the app for which
61      *   profiles are generated.
62      * @param metrics List of metrics to measure.
63      * @param compilationMode Mode of compilation used before capturing measurement, such as
64      *   [CompilationMode.Partial], defaults to [CompilationMode.DEFAULT].
65      * @param startupMode Optional mode to force app launches performed with
66      *   [MacrobenchmarkScope.startActivityAndWait] (and similar variants) to be of the assigned
67      *   type. For example, `COLD` launches kill the process before the measureBlock, to ensure
68      *   startups will go through full process creation. Generally, leave as null for non-startup
69      *   benchmarks.
70      * @param iterations Number of times the [measureBlock] will be run during measurement. Note
71      *   that total iteration count may not match, due to warmup iterations needed for the
72      *   [compilationMode].
73      * @param setupBlock The block performing app actions each iteration, prior to the
74      *   [measureBlock]. For example, navigating to a UI where scrolling will be measured.
75      * @param measureBlock The block performing app actions to benchmark each iteration.
76      */
77     @JvmOverloads
measureRepeatednull78     fun measureRepeated(
79         packageName: String,
80         metrics: List<Metric>,
81         compilationMode: CompilationMode = CompilationMode.DEFAULT,
82         startupMode: StartupMode? = null,
83         @IntRange(from = 1) iterations: Int,
84         setupBlock: MacrobenchmarkScope.() -> Unit = {},
85         measureBlock: MacrobenchmarkScope.() -> Unit
86     ) {
87         macrobenchmarkWithStartupMode(
88             uniqueName = currentDescription.toUniqueName(),
89             className = currentDescription.className,
90             testName = currentDescription.methodName,
91             packageName = packageName,
92             metrics = metrics,
93             compilationMode = compilationMode,
94             iterations = iterations,
95             startupMode = startupMode,
96             experimentalConfig = null,
97             setupBlock = setupBlock,
98             measureBlock = measureBlock
99         )
100     }
101 
102     /**
103      * Measure behavior of the specified [packageName] given a set of [metrics], with a custom
104      * [PerfettoConfig].
105      *
106      * This performs a macrobenchmark with the below control flow:
107      * ```
108      *     resetAppCompilation()
109      *     compile(compilationMode)
110      *     repeat(iterations) {
111      *         setupBlock()
112      *         captureTraceAndMetrics {
113      *             measureBlock()
114      *         }
115      *     }
116      * ```
117      *
118      * Note that a custom [PerfettoConfig]s may result in built-in [Metric]s not working.
119      *
120      * You can see the PerfettoConfig used by a trace (as a text proto) by opening the trace in
121      * [ui.perfetto.dev](http://ui.perfetto.dev), and selecting `Info and Stats` view on the left
122      * panel. You can also generate a custom text proto config by selecting `Record new trace` on
123      * the same panel, selecting recording options, and then clicking `Recording command` to access
124      * the generated text proto.
125      *
126      * @param packageName ApplicationId / Application manifest package name of the app for which
127      *   profiles are generated.
128      * @param metrics List of metrics to measure.
129      * @param compilationMode Mode of compilation used before capturing measurement, such as
130      *   [CompilationMode.Partial], defaults to [CompilationMode.DEFAULT].
131      * @param startupMode Optional mode to force app launches performed with
132      *   [MacrobenchmarkScope.startActivityAndWait] (and similar variants) to be of the assigned
133      *   type. For example, `COLD` launches kill the process before the measureBlock, to ensure
134      *   startups will go through full process creation. Generally, leave as null for non-startup
135      *   benchmarks.
136      * @param iterations Number of times the [measureBlock] will be run during measurement. Note
137      *   that total iteration count may not match, due to warmup iterations needed for the
138      *   [compilationMode].
139      * @param experimentalConfig Configuration for experimental features.
140      * @param setupBlock The block performing app actions each iteration, prior to the
141      *   [measureBlock]. For example, navigating to a UI where scrolling will be measured.
142      * @param measureBlock The block performing app actions to benchmark each iteration.
143      */
144     @ExperimentalBenchmarkConfigApi
145     @JvmOverloads
measureRepeatednull146     fun measureRepeated(
147         packageName: String,
148         metrics: List<Metric>,
149         @IntRange(from = 1) iterations: Int,
150         experimentalConfig: ExperimentalConfig,
151         compilationMode: CompilationMode = CompilationMode.DEFAULT,
152         startupMode: StartupMode? = null,
153         setupBlock: MacrobenchmarkScope.() -> Unit = {},
154         measureBlock: MacrobenchmarkScope.() -> Unit,
155     ) {
156         macrobenchmarkWithStartupMode(
157             uniqueName = currentDescription.toUniqueName(),
158             className = currentDescription.className,
159             testName = currentDescription.methodName,
160             packageName = packageName,
161             metrics = metrics,
162             compilationMode = compilationMode,
163             iterations = iterations,
164             experimentalConfig = experimentalConfig,
165             startupMode = startupMode,
166             setupBlock = setupBlock,
167             measureBlock = measureBlock
168         )
169     }
170 
171     @ExperimentalBenchmarkConfigApi
172     @JvmOverloads
173     @Deprecated(
174         "Deprecated in favour of a variant that accepts experimental config",
175         replaceWith =
176             ReplaceWith(
177                 "measureRepeated(packageName, metrics, iterations,experimentalConfig, " +
178                     "compilationMode, startupMode, setupBlock, measureBlock)"
179             ),
180         level = DeprecationLevel.WARNING
181     )
182     /**
183      * @param perfettoConfig Configuration for Perfetto trace capture during each iteration. Note
184      *   that insufficient or invalid configs may result in built-in [Metric]s not working.
185      */
measureRepeatednull186     fun measureRepeated(
187         packageName: String,
188         metrics: List<Metric>,
189         @IntRange(from = 1) iterations: Int,
190         perfettoConfig: PerfettoConfig,
191         compilationMode: CompilationMode = CompilationMode.DEFAULT,
192         startupMode: StartupMode? = null,
193         setupBlock: MacrobenchmarkScope.() -> Unit = {},
194         measureBlock: MacrobenchmarkScope.() -> Unit,
195     ) =
196         measureRepeated(
197             packageName,
198             metrics,
199             iterations,
200             ExperimentalConfig(perfettoConfig = perfettoConfig),
201             compilationMode,
202             startupMode,
203             setupBlock,
204             measureBlock
205         )
206 
applynull207     override fun apply(base: Statement, description: Description): Statement =
208         object : Statement() {
209             override fun evaluate() {
210                 assumeTrue(Arguments.RuleType.Macrobenchmark in Arguments.enabledRules)
211                 currentDescription = description
212                 base.evaluate()
213             }
214         }
215 
Descriptionnull216     private fun Description.toUniqueName() = testClass.simpleName + "_" + methodName
217 }
218