1 /*
<lambda>null2 * Copyright 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package androidx.benchmark.junit4
18
19 import android.Manifest
20 import android.os.Looper
21 import androidx.annotation.RestrictTo
22 import androidx.benchmark.Arguments
23 import androidx.benchmark.BenchmarkState
24 import androidx.benchmark.DeviceInfo
25 import androidx.benchmark.ExperimentalBenchmarkConfigApi
26 import androidx.benchmark.MicrobenchmarkConfig
27 import androidx.benchmark.MicrobenchmarkRunningState
28 import androidx.benchmark.MicrobenchmarkScope
29 import androidx.benchmark.TestDefinition
30 import androidx.benchmark.measureRepeatedImplWithTracing
31 import androidx.test.rule.GrantPermissionRule
32 import org.junit.Assume.assumeFalse
33 import org.junit.Assume.assumeTrue
34 import org.junit.rules.RuleChain
35 import org.junit.rules.TestRule
36 import org.junit.runner.Description
37 import org.junit.runners.model.Statement
38
39 /**
40 * JUnit rule for benchmarking code on an Android device.
41 *
42 * In Kotlin, benchmark with [measureRepeated]. In Java, use [getState].
43 *
44 * Benchmark results will be output:
45 * - Summary in Android Studio in the test log
46 * - In JSON format, on the host
47 * - In simple form in Logcat with the tag "Benchmark"
48 *
49 * Every test in the Class using this @Rule must contain a single benchmark.
50 *
51 * See the [Benchmark Guide](https://developer.android.com/studio/profile/benchmark) for more
52 * information on writing Benchmarks.
53 *
54 * @sample androidx.benchmark.samples.benchmarkRuleSample
55 */
56 class BenchmarkRule
57 @ExperimentalBenchmarkConfigApi
58 constructor(
59 val config: MicrobenchmarkConfig,
60 ) : TestRule {
61 constructor() : this(config = MicrobenchmarkConfig())
62
63 @PublishedApi
64 internal // synthetic access
65 var testDefinition: TestDefinition? = null
66 get() {
67 throwIfNotApplied()
68 return field
69 }
70
71 internal // synthetic access
72 var internalState: BenchmarkState? = null
73
74 internal fun throwIfNotApplied() {
75 if (!applied) {
76 throw IllegalStateException(
77 "Cannot get state before BenchmarkRule is applied to a test. Check that your " +
78 "BenchmarkRule is annotated correctly (@Rule in Java, @get:Rule in Kotlin)."
79 )
80 }
81 }
82
83 /**
84 * Object used for benchmarking in Java.
85 *
86 * ```java
87 * @Rule
88 * public BenchmarkRule benchmarkRule = new BenchmarkRule();
89 *
90 * @Test
91 * public void myBenchmark() {
92 * ...
93 * BenchmarkState state = benchmarkRule.getBenchmarkState();
94 * while (state.keepRunning()) {
95 * doSomeWork();
96 * }
97 * ...
98 * }
99 * ```
100 *
101 * @throws [IllegalStateException] if the BenchmarkRule isn't correctly applied to a test.
102 */
103 fun getState(): BenchmarkState {
104 // Note: this is an explicit method instead of an accessor to help convey it's only for Java
105 // Kotlin users should call the [measureRepeated] method.
106 throwIfNotApplied()
107 return internalState!!
108 }
109
110 internal // synthetic access
111 var applied = false
112
113 // can we avoid published API here?
114 @RestrictTo(RestrictTo.Scope.LIBRARY_GROUP)
115 @get:RestrictTo(RestrictTo.Scope.LIBRARY_GROUP)
116 val scopeFactory: (MicrobenchmarkRunningState) -> MicrobenchmarkScope = { runningState ->
117 Scope(runningState)
118 }
119
120 /** Handle used for controlling measurement during [measureRepeated]. */
121 inner class Scope internal constructor(internal val state: MicrobenchmarkRunningState) :
122
123 /*
124 * Ideally, the microbenchmark scope concept would live entirely in benchmark-common so that
125 * we can define it in common code, without a dependence on benchmark-junit / JUnit.
126 *
127 * To preserve compatibility though, we have to preserve this copy, which causes the
128 * following layering compromises:
129 *
130 * 1. The top level `measureRepeated` function accepts a `scopeFactory` function to let it
131 * construct a BenchmarkRule.Scope object, even though it's a trivial wrapper around
132 * MicrobenchmarkScope.
133 *
134 * 2. To let scope-ish calls go from BenchmarkState -> MicrobenchmarkScope ->
135 * MicrobenchmarkRunningState, BenchmarkState has a LIBRARY_GROUP mutable var, so both
136 * legacy BenchmarkState.keepRunning() and modern BenchmarkRule.measureRepeated have to
137 * separately set BenchmarkState.scope after scope is constructed. This stinks, but it's the
138 * price of compat. Both are needed only because it was valid to pause timing in a pure
139 * Kotlin benchmark with rule.getState().pauseTiming()
140 */
141 MicrobenchmarkScope(state) {
142
143 /**
144 * Disable measurement for a block of code.
145 *
146 * Used for disabling timing/measurement for work that isn't part of the benchmark:
147 * - When constructing per-loop randomized inputs for operations with caching,
148 * - Controlling which parts of multi-stage work are measured (e.g. View measure/layout)
149 * - Per-loop verification
150 *
151 * @sample androidx.benchmark.samples.runWithMeasurementDisabledSample
152 */
153 @Deprecated(
154 "Renamed to runWithMeasurementDisabled to clarify all measurements are paused",
155 replaceWith = ReplaceWith("runWithMeasurementDisabled")
156 )
157 inline fun <T> runWithTimingDisabled(block: () -> T): T {
158 return runWithMeasurementDisabled(block)
159 }
160
161 /**
162 * Allows the inline function [runWithTimingDisabled] to be called outside of this scope for
163 * compat with compiled code using old versions of the library.
164 */
165 @Suppress("unused")
166 @PublishedApi
167 internal fun getOuterState(): BenchmarkState {
168 return getState()
169 }
170 }
171
172 override fun apply(base: Statement, description: Description): Statement {
173 return RuleChain.outerRule(
174 GrantPermissionRule.grant(Manifest.permission.WRITE_EXTERNAL_STORAGE)
175 )
176 .around(::applyInternal)
177 .apply(base, description)
178 }
179
180 private fun applyInternal(base: Statement, description: Description) = Statement {
181 applied = true
182
183 assumeTrue(Arguments.RuleType.Microbenchmark in Arguments.enabledRules)
184
185 // When running on emulator and argument `skipOnEmulator` is passed,
186 // the test is skipped.
187 if (Arguments.skipBenchmarksOnEmulator) {
188 assumeFalse(
189 "Skipping test because it's running on emulator and `skipOnEmulator` is enabled",
190 DeviceInfo.isEmulator
191 )
192 }
193
194 testDefinition =
195 TestDefinition(
196 fullClassName = description.className,
197 simpleClassName = description.testClass.simpleName,
198 methodName = description.methodName
199 )
200
201 // only used with legacy getState() API, which is intended to be deprecated in the future,
202 // to be replaced by Java variant of measureRepeated
203 internalState = BenchmarkState(testDefinition!!, config)
204
205 base.evaluate()
206 }
207 }
208
209 /**
210 * Benchmark a block of code.
211 *
212 * @param block The block of code to benchmark.
213 * @sample androidx.benchmark.samples.benchmarkRuleSample
214 */
measureRepeatednull215 public inline fun BenchmarkRule.measureRepeated(crossinline block: BenchmarkRule.Scope.() -> Unit) {
216 // Note: this is an extension function to discourage calling from Java.
217 if (Arguments.throwOnMainThreadMeasureRepeated) {
218 check(Looper.myLooper() != Looper.getMainLooper()) {
219 "Cannot invoke measureRepeated from the main thread. Instead use" +
220 " measureRepeatedOnMainThread()"
221 }
222 }
223 measureRepeatedImplWithTracing(
224 postToMainThread = false,
225 definition = testDefinition!!,
226 config = config,
227 scopeFactory = scopeFactory, // inflate custom Scope object to respect/maintain public API
228 loopedMeasurementBlock = { scope, iterations ->
229 val ruleScope = scope as BenchmarkRule.Scope // cast back to outer scope type
230 getState().scope = scope
231 var remainingIterations = iterations
232 do {
233 block.invoke(ruleScope)
234 remainingIterations--
235 } while (remainingIterations > 0)
236 }
237 )
238 }
239
240 /**
241 * Benchmark a block of code, which runs on the main thread, and can safely interact with UI.
242 *
243 * While `@UiThreadRule` works for a standard test, it doesn't work for benchmarks of arbitrary
244 * duration, as they may run for much more than 5 seconds and suffer ANRs, especially in continuous
245 * runs.
246 *
247 * @param block The block of code to benchmark.
248 * @throws java.lang.Throwable when an exception is thrown on the main thread.
249 * @throws IllegalStateException if a hard deadline is exceeded while the block is running on the
250 * main thread.
251 * @sample androidx.benchmark.samples.measureRepeatedOnMainThreadSample
252 */
measureRepeatedOnMainThreadnull253 inline fun BenchmarkRule.measureRepeatedOnMainThread(
254 crossinline block: BenchmarkRule.Scope.() -> Unit
255 ) {
256 check(Looper.myLooper() != Looper.getMainLooper()) {
257 "Cannot invoke measureRepeatedOnMainThread from the main thread"
258 }
259
260 measureRepeatedImplWithTracing(
261 postToMainThread = true,
262 definition = testDefinition!!,
263 config = config,
264 scopeFactory = scopeFactory, // inflate custom Scope object to respect/maintain public API
265 loopedMeasurementBlock = { scope, iterations ->
266 val ruleScope = scope as BenchmarkRule.Scope // cast back to outer scope type
267 getState().scope = scope
268 var remainingIterations = iterations
269 do {
270 block.invoke(ruleScope)
271 remainingIterations--
272 } while (remainingIterations > 0)
273 }
274 )
275 }
276
Statementnull277 internal inline fun Statement(crossinline evaluate: () -> Unit) =
278 object : Statement() {
279 override fun evaluate() = evaluate()
280 }
281