1 /* <lambda>null2 * Copyright 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package androidx.benchmark.json 18 19 import androidx.benchmark.Arguments 20 import androidx.benchmark.CpuInfo 21 import androidx.benchmark.DeviceInfo 22 import androidx.benchmark.IsolationActivity 23 import androidx.benchmark.MemInfo 24 import androidx.benchmark.PackageInfo 25 import androidx.benchmark.Profiler 26 import androidx.benchmark.ResultWriter 27 import com.squareup.moshi.JsonClass 28 29 /** 30 * Top level json object for benchmark output for a multi-test run 31 * 32 * Corresponds to <packagename>BenchmarkData.json file output. 33 * 34 * Must be public, restrict to for usage from macrobench. We avoid @RestrictTo on these objects, and 35 * rely on package-info instead, as that works for adapters as well, which fail to be detected by 36 * metalava: b/331978183. 37 */ 38 @JsonClass(generateAdapter = true) 39 data class BenchmarkData(val context: Context, val benchmarks: List<TestResult>) { 40 /** Device & OS information */ 41 @JsonClass(generateAdapter = true) 42 data class Context( 43 val build: Build, 44 val cpuCoreCount: Int, 45 @Suppress("GetterSetterNames") // 1.0 JSON compat 46 @get:Suppress("GetterSetterNames") // 1.0 JSON compat 47 val cpuLocked: Boolean, 48 val cpuMaxFreqHz: Long, 49 val memTotalBytes: Long, 50 @Suppress("GetterSetterNames") // 1.0 JSON compat 51 @get:Suppress("GetterSetterNames") // 1.0 JSON compat 52 val sustainedPerformanceModeEnabled: Boolean, 53 val artMainlineVersion: Long, // -1 if not found 54 val osCodenameAbbreviated: String, 55 val compilationMode: String, 56 // additional data that can be passed from instrumentation arguments and copied into 57 // the json output. 58 val payload: Map<String, String> = emptyMap() // need default value for backwards compat 59 // Note: Convention is to add new entries at bottom 60 ) { 61 /** Default constructor populates with current run state */ 62 constructor() : 63 this( 64 build = Build(), 65 cpuCoreCount = CpuInfo.coreDirs.size, 66 cpuLocked = CpuInfo.locked, 67 cpuMaxFreqHz = CpuInfo.maxFreqHz, 68 memTotalBytes = MemInfo.memTotalBytes, 69 sustainedPerformanceModeEnabled = IsolationActivity.sustainedPerformanceModeInUse, 70 artMainlineVersion = DeviceInfo.artMainlineVersion, 71 osCodenameAbbreviated = 72 if ( 73 android.os.Build.VERSION.SDK_INT >= 35 && 74 android.os.Build.VERSION.CODENAME == "REL" 75 ) { 76 "REL" // OS doesn't support codename letters anymore 77 } else { 78 if (android.os.Build.VERSION.CODENAME != "REL") { 79 // non-release build, use codename 80 android.os.Build.VERSION.CODENAME 81 } else { 82 // release build, use start of build ID 83 android.os.Build.ID 84 } 85 .substring(0, 1) 86 }, 87 compilationMode = PackageInfo.compilationMode, 88 payload = Arguments.payload 89 ) 90 91 /** 92 * Device & OS information, corresponds to `android.os.Build` 93 * 94 * Anything that doesn't correspond exactly to `android.os.Build` should be in context 95 * instead 96 */ 97 @JsonClass(generateAdapter = true) 98 data class Build( 99 val brand: String, 100 val device: String, 101 val fingerprint: String, 102 val id: String, 103 val model: String, 104 val type: String, 105 val version: Version 106 // Note: Convention is alphabetical 107 ) { 108 /** Default constructor which populates values from `android.os.BUILD` */ 109 constructor() : 110 this( 111 brand = android.os.Build.BRAND, 112 device = android.os.Build.DEVICE, 113 fingerprint = android.os.Build.FINGERPRINT, 114 id = android.os.Build.ID, 115 model = android.os.Build.MODEL, 116 type = android.os.Build.TYPE, 117 version = 118 Version( 119 codename = android.os.Build.VERSION.CODENAME, 120 sdk = android.os.Build.VERSION.SDK_INT, 121 ), 122 ) 123 124 @JsonClass(generateAdapter = true) 125 data class Version( 126 val codename: String, 127 val sdk: Int, 128 ) 129 } 130 } 131 132 /** 133 * Measurements corresponding to a single test's invocation. 134 * 135 * Note that one parameterized test in code can produce more than one test result. 136 */ 137 @JsonClass(generateAdapter = true) 138 data class TestResult( 139 val name: String, 140 val params: Map<String, String>, 141 val className: String, 142 @Suppress("MethodNameUnits") @get:Suppress("MethodNameUnits") val totalRunTimeNs: Long, 143 val metrics: Map<String, SingleMetricResult>, 144 val sampledMetrics: Map<String, SampledMetricResult>, 145 val warmupIterations: Int?, 146 val repeatIterations: Int?, 147 val thermalThrottleSleepSeconds: Long?, 148 val profilerOutputs: List<ProfilerOutput>?, 149 ) { 150 init { 151 profilerOutputs?.let { profilerOutput -> 152 val labels = profilerOutput.map { it.label } 153 require(labels.toSet().size == profilerOutput.size) { 154 "Each profilerOutput must have a distinct label. Labels seen: " + 155 labels.joinToString() 156 } 157 } 158 } 159 160 constructor( 161 name: String, 162 className: String, 163 totalRunTimeNs: Long, 164 metrics: List<androidx.benchmark.MetricResult>, 165 warmupIterations: Int, 166 repeatIterations: Int, 167 thermalThrottleSleepSeconds: Long, 168 profilerOutputs: List<ProfilerOutput>? 169 ) : this( 170 name = name, 171 params = ResultWriter.getParams(name), 172 className = className, 173 totalRunTimeNs = totalRunTimeNs, 174 metrics = 175 metrics 176 .filter { 177 it.iterationData == null // single metrics only 178 } 179 .associate { it.name to SingleMetricResult(it) }, 180 sampledMetrics = 181 metrics 182 .filter { 183 it.iterationData != null // single metrics only 184 } 185 .associate { it.name to SampledMetricResult(it) }, 186 warmupIterations = warmupIterations, 187 repeatIterations = repeatIterations, 188 thermalThrottleSleepSeconds = thermalThrottleSleepSeconds, 189 profilerOutputs = profilerOutputs, 190 ) 191 192 @JsonClass(generateAdapter = true) 193 data class ProfilerOutput( 194 /** 195 * Type of trace. 196 * 197 * Note that multiple data formats may use the same type here, like simpleperf vs art 198 * stack sampling traces. 199 * 200 * This isn't meant to be a specific data format, but more conceptual category. 201 */ 202 val type: Type, 203 /** 204 * User facing label for the profiler output. 205 * 206 * If more than one profiler output has the same type, this label gives context 207 * explaining the distinction. 208 */ 209 val label: String, 210 /** Filename of trace file. */ 211 val filename: String 212 ) { 213 constructor( 214 profilerResult: Profiler.ResultFile 215 ) : this( 216 type = profilerResult.type, 217 label = profilerResult.label, 218 filename = profilerResult.outputRelativePath, 219 ) 220 221 enum class Type { 222 MethodTrace, 223 PerfettoTrace, 224 StackSamplingTrace 225 } 226 } 227 228 sealed class MetricResult 229 230 @JsonClass(generateAdapter = true) 231 data class SingleMetricResult( 232 val minimum: Double, 233 val maximum: Double, 234 val median: Double, 235 val coefficientOfVariation: Double, 236 val runs: List<Double> 237 ) : MetricResult() { 238 constructor( 239 metricResult: androidx.benchmark.MetricResult 240 ) : this( 241 minimum = metricResult.min, 242 maximum = metricResult.max, 243 median = metricResult.median, 244 coefficientOfVariation = metricResult.coefficientOfVariation, 245 runs = metricResult.data 246 ) 247 } 248 249 @JsonClass(generateAdapter = true) 250 data class SampledMetricResult( 251 @Suppress("PropertyName") val P50: Double, 252 @Suppress("PropertyName") val P90: Double, 253 @Suppress("PropertyName") val P95: Double, 254 @Suppress("PropertyName") val P99: Double, 255 val runs: List<List<Double>> 256 ) : MetricResult() { 257 constructor( 258 metricResult: androidx.benchmark.MetricResult 259 ) : this( 260 P50 = metricResult.p50, 261 P90 = metricResult.p90, 262 P95 = metricResult.p95, 263 P99 = metricResult.p99, 264 runs = metricResult.iterationData!! 265 ) 266 } 267 } 268 } 269